From c5c493396f23d1d762e127bd19b1ad0b62293571 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:48:11 +0000 Subject: [PATCH 01/38] feat(emails): send transactional email and read its status Add PostHog.Emails.send/2 for a template made in PostHog (template_id, template_data) or content the app rendered (template_key, subject, html and/or text), and PostHog.Emails.get/2 for an email's status. Both use the project secret key against the app host, with one idempotency key across retries. New config options: app_host, email_request_timeout_ms, email_request_max_retries and migrated_from. Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-send.md | 5 + README.md | 1 + lib/posthog/config.ex | 18 +- lib/posthog/config_schema.ex | 49 +- lib/posthog/emails.ex | 821 +++++++++++++++++++++++++++++++ lib/posthog/emails/error.ex | 85 ++++ lib/posthog/emails/results.ex | 68 +++ lib/posthog/registry.ex | 10 + lib/posthog/supervisor.ex | 2 +- mix.exs | 7 + public_api.snapshot | 38 ++ test/posthog/emails_test.exs | 746 ++++++++++++++++++++++++++++ 12 files changed, 1842 insertions(+), 8 deletions(-) create mode 100644 .sampo/changesets/emails-send.md create mode 100644 lib/posthog/emails.ex create mode 100644 lib/posthog/emails/error.ex create mode 100644 lib/posthog/emails/results.ex create mode 100644 test/posthog/emails_test.exs diff --git a/.sampo/changesets/emails-send.md b/.sampo/changesets/emails-send.md new file mode 100644 index 00000000..176c978c --- /dev/null +++ b/.sampo/changesets/emails-send.md @@ -0,0 +1,5 @@ +--- +hex/posthog: minor +--- + +Add `PostHog.Emails` to send transactional email and read its status. `PostHog.Emails.send/2` queues one email from a template made in PostHog (`template_id` and `template_data`) or from content your code rendered (`template_key`, `subject`, and `html` and/or `text`). `PostHog.Emails.get/2` reads what happened to it. Both need a project secret API key in `secret_key`. New config options: `app_host`, `email_request_timeout_ms`, `email_request_max_retries` and `migrated_from`. diff --git a/README.md b/README.md index 1f0df978..e2c1eb8d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ SDK usage examples and code snippets live in the official documentation so they - [Elixir library docs](https://posthog.com/docs/libraries/elixir) - [Phoenix framework docs](https://posthog.com/docs/libraries/phoenix) +- [Transactional email (HexDocs)](https://hexdocs.pm/posthog/PostHog.Emails.html) - [OpenTelemetry span context (HexDocs)](https://hexdocs.pm/posthog/PostHog.OpenTelemetry.ContextSpanProcessor.html) - [API reference (HexDocs)](https://hexdocs.pm/posthog) diff --git a/lib/posthog/config.ex b/lib/posthog/config.ex index f82062d7..22f99710 100644 --- a/lib/posthog/config.ex +++ b/lib/posthog/config.ex @@ -118,8 +118,9 @@ defmodule PostHog.Config do ## Remarks - String `:api_key`, `:secret_key`, and `:api_host` values are trimmed before validation. A blank - `:api_host` falls back to the default PostHog US ingestion host. + String `:api_key`, `:secret_key`, `:app_host`, and `:api_host` values are trimmed before + validation. A blank `:api_host` falls back to the default PostHog US ingestion host, and a blank + `:app_host` counts as unset. """ @spec validate(options()) :: {:ok, config()} | {:error, NimbleOptions.ValidationError.t()} @@ -179,7 +180,14 @@ defmodule PostHog.Config do end) |> then(fn normalized_options -> if Keyword.has_key?(normalized_options, :secret_key) do - Keyword.update!(normalized_options, :secret_key, &normalize_secret_key/1) + Keyword.update!(normalized_options, :secret_key, &normalize_optional_string/1) + else + normalized_options + end + end) + |> then(fn normalized_options -> + if Keyword.has_key?(normalized_options, :app_host) do + Keyword.update!(normalized_options, :app_host, &normalize_optional_string/1) else normalized_options end @@ -197,14 +205,14 @@ defmodule PostHog.Config do defp normalize_api_key(nil), do: "" defp normalize_api_key(api_key), do: api_key - defp normalize_secret_key(secret_key) when is_binary(secret_key) do + defp normalize_optional_string(secret_key) when is_binary(secret_key) do case String.trim(secret_key) do "" -> nil value -> value end end - defp normalize_secret_key(secret_key), do: secret_key + defp normalize_optional_string(secret_key), do: secret_key defp normalize_api_host(api_host) when is_binary(api_host) do api_host diff --git a/lib/posthog/config_schema.ex b/lib/posthog/config_schema.ex index a1c742b5..0889281d 100644 --- a/lib/posthog/config_schema.ex +++ b/lib/posthog/config_schema.ex @@ -43,9 +43,43 @@ defmodule PostHog.Config.Schema do type: {:or, [:string, nil]}, default: nil, doc: """ - A privileged project secret (`phs_`) or appropriately scoped personal API key (`phx_`) - required only to fetch local feature flag definitions directly from PostHog. A configured + A privileged project secret (`phs_`) or appropriately scoped personal API key (`phx_`). + Fetches local feature flag definitions directly from PostHog. A configured `flag_definition_cache_provider` can supply definitions without this key. + `PostHog.Emails` needs a project secret API key with the `hog_flow:write` scope to send + email, or `hog_flow:read` to read an email's status. + """ + ], + app_host: [ + type: {:or, [:string, nil]}, + default: nil, + doc: """ + The PostHog app host that `PostHog.Emails` sends to, such as `https://posthog.example.com` + for a self-hosted instance. Without it, a PostHog Cloud `api_host` maps to its region's app + host, and any other `api_host` makes email calls fail with `missing_app_host`, because a + reverse proxy for ingestion must not receive the secret key. + """ + ], + email_request_timeout_ms: [ + type: :pos_integer, + default: 15_000, + doc: "Maximum time in milliseconds to wait for each `PostHog.Emails` request." + ], + email_request_max_retries: [ + type: :non_neg_integer, + default: 2, + doc: """ + Number of retries for a `PostHog.Emails` request after a network error, a rate limit, an + idempotency key still in flight, or an unavailable service. Set to 0 when a job runner + already retries the send. + """ + ], + migrated_from: [ + type: {:or, [{:custom, __MODULE__, :header_token, []}, nil]}, + default: nil, + doc: """ + The email provider you moved from, such as `"resend"`. Sent with every `PostHog.Emails` + request as `X-PostHog-Migrated-From`. """ ], enable_local_evaluation: [ @@ -151,6 +185,17 @@ defmodule PostHog.Config.Schema do ] ++ shared() end + @doc false + def header_token(value) when is_binary(value) do + if value =~ ~r/\A[\x21-\x7e]+\z/ do + {:ok, value} + else + {:error, "expected visible ASCII characters without spaces, got: #{inspect(value)}"} + end + end + + def header_token(value), do: {:error, "expected a string, got: #{inspect(value)}"} + def convenience do [ enable: [ diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex new file mode 100644 index 00000000..f0b6d3bf --- /dev/null +++ b/lib/posthog/emails.ex @@ -0,0 +1,821 @@ +defmodule PostHog.Emails do + @moduledoc """ + Send transactional email, such as a password reset or a receipt, and read its status. + + An email goes out in one of two forms: + + - **A template made in PostHog**, sent by its ID with `:template_id` and `:template_data`. + - **Content your code rendered**, named by `:template_key`, with `:subject` and `:html`, + `:text` or both. To send a React email, render it to HTML in your app, for example with + `@react-email/render`, and pass the result as `:html`. The SDK adds no rendering. + + ## Setup + + 1. In PostHog, open the email template under Workflows, make it transactional, and copy its + ID. `:template_id` is that UUID. The code form needs no template. + 2. Create a project secret API key (`phs_...`) with Workflow write access (`hog_flow:write`) + in your project settings. The project API key (`phc_...`) captures events but cannot send + email. + 3. Set it as `:secret_key` in your PostHog config: + + config :posthog, + api_host: "https://eu.i.posthog.com", + api_key: "phc_...", + secret_key: "phs_..." + + ## Examples + + Send a template made in PostHog: + + {:ok, %PostHog.Emails.SendResult{id: message_id}} = + PostHog.Emails.send( + to: "user@example.com", + template_id: "0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", + template_data: %{name: "Ada"}, + distinct_id: "user_123", + idempotency_key: "welcome:user_123" + ) + + Send content your code rendered: + + PostHog.Emails.send( + to: "user@example.com", + template_key: "password-reset", + subject: "Reset your password", + html: rendered_html, + distinct_id: "user_123", + idempotency_key: "password-reset:\#{reset.id}" + ) + + Read what happened to it: + + {:ok, %PostHog.Emails.Status{status: status}} = PostHog.Emails.get(message_id) + + Always pass `:distinct_id`. Without it, the email's events go to a person whose distinct ID + is the email address, not to your user. The SDK never takes it from `PostHog.set_context/2`. + + ## What a result means + + `{:ok, result}` means PostHog queued the email. Delivery happens later: read it with `get/2`, + or in the template's workflow in PostHog. + + ## Where requests go + + A client on PostHog Cloud sends to its region's app host, `https://us.posthog.com` or + `https://eu.posthog.com`. For any other `:api_host`, such as a reverse proxy, set `:app_host` + in the config, because the proxy must not receive the secret key. Without it, a send returns + the error `missing_app_host`. + + ## Timeouts, retries and idempotency + + Each request times out after `:email_request_timeout_ms` (15 seconds by default). A send + retries network errors, rate limits, an idempotency key still in flight and an unavailable + service `:email_request_max_retries` times (2 by default), waiting about 1 and then 2 seconds, + or the server's `Retry-After` when it is longer. A `Retry-After` over 30 seconds fails at once + with `:retry_after_ms` set. A monthly quota is never retried. + + Every attempt sends the same idempotency key, so a retry never sends twice. Without + `:idempotency_key`, the SDK makes a UUID for the call. When a job runner such as Oban + already retries the send, pass `max_retries: 0` and an `:idempotency_key` that names the + email, such as `"welcome:user_123"`. A generated key doesn't survive a job retry. + + Sends and status reads run in the calling process. The SDK keeps no email queue. + + ## Errors + + Every failure returns `{:error, %PostHog.Emails.Error{}}`. Branch on `:code`. The ones a + setup usually hits: + + - `missing_secret_key`: the config has no `:secret_key`. + - `missing_app_host`: the `:api_host` is not PostHog Cloud. Set `:app_host`. + - `authentication_failed` (401) or `permission_denied` (403): the key is wrong, or it lacks + `hog_flow:write`. + - `not_found` (404): the template ID is wrong, or transactional email is not enabled for the + project. + - `client_disabled`: the PostHog instance is not running or has no `:api_key`, so it sends + nothing. + + An option the SDK refuses returns an error with an `invalid_*` code, such as + `invalid_template_id`, and the name `:validation_error`, before any request. + + ## Testing + + Requests go through the configured `:api_client_module`, so a `Mox` mock of + `PostHog.API.Client` sees every email request without network calls. + """ + + import Kernel, except: [send: 2] + + require Logger + + alias PostHog.Emails.Error + alias PostHog.Emails.SendResult + alias PostHog.Emails.Status + alias PostHog.Emails.Warning + + @send_path "/api/projects/@current/messaging/send/email/" + @status_path "/api/projects/@current/messaging/emails/" + @us_app_host "https://us.posthog.com" + @eu_app_host "https://eu.posthog.com" + @cloud_app_hosts %{ + "https://us.i.posthog.com" => @us_app_host, + "https://us.posthog.com" => @us_app_host, + "https://app.posthog.com" => @us_app_host, + "https://eu.i.posthog.com" => @eu_app_host, + "https://eu.posthog.com" => @eu_app_host + } + @max_retry_delay_ms 30_000 + @default_timeout_ms 15_000 + @default_max_retries 2 + @log_metadata [posthog_skip_capture: true] + + @shared_fields [:to, :distinct_id, :idempotency_key, :max_retries] + @template_fields @shared_fields ++ [:template_id, :template_data] + @code_fields @shared_fields ++ + [:template_key, :subject, :html, :text, :from, :reply_to, :capture_engagement] + + @typedoc """ + Options for `send/2`. Pass `:template_id` or `:template_key`, never both. + + Every send takes: + + - `:to` - one recipient email address. Required. + - `:distinct_id` - the recipient's distinct ID in your product. The email's events go to + this person. + - `:idempotency_key` - 1 to 255 visible ASCII characters. A send with a key PostHog already + accepted queues nothing and returns the first send's result. Template sends and code sends + share one set of keys: a key already used for another email returns + `idempotency_key_reused`. + - `:max_retries` - overrides the config's `:email_request_max_retries` for this send. `0` + turns retries off. + + A template made in PostHog takes: + + - `:template_id` - the UUID of a template made transactional in PostHog. + - `:template_data` - a map of values for the template's `{{ variables.* }}` placeholders. + + Content your code rendered takes: + + - `:template_key` - names the email, such as `"password-reset"`, so its metrics, logs and + status group under it in PostHog. Letters, digits, `-` and `_`, at most 400. + - `:subject` - the subject line. Required. + - `:html` and `:text` - the bodies. Give one or both. Without `:text`, PostHog derives it from + `:html`. + - `:from` - a verified sender, such as `"Acme "`. Without it, the + project's default sender. + - `:reply_to` - one address or a list of addresses. + - `:capture_engagement` - whether PostHog captures the email's opens and clicks as events of + the recipient. Defaults to `true`. A project whose admin turned engagement capture off + captures nothing, whatever the send asks. + + An option set to `nil` counts as left out. + """ + @type send_option() :: + {:to, String.t()} + | {:distinct_id, String.t() | nil} + | {:idempotency_key, String.t() | nil} + | {:max_retries, non_neg_integer() | nil} + | {:template_id, String.t()} + | {:template_data, map() | nil} + | {:template_key, String.t()} + | {:subject, String.t()} + | {:html, String.t() | nil} + | {:text, String.t() | nil} + | {:from, String.t() | nil} + | {:reply_to, String.t() | [String.t()] | nil} + | {:capture_engagement, boolean() | nil} + + @doc """ + Queues one email, from a template made in PostHog or from content your code rendered. + + Returns `{:ok, %PostHog.Emails.SendResult{}}` once PostHog queued the email, or + `{:error, %PostHog.Emails.Error{}}`. See `t:send_option/0` for the options. `name` is the + PostHog instance to send through, `PostHog` by default. + + ## Examples + + PostHog.Emails.send( + to: "user@example.com", + template_id: "0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", + distinct_id: "user_123" + ) + + PostHog.Emails.send(MyPostHog, + to: "user@example.com", + template_key: "receipt", + subject: "Your receipt", + html: html, + distinct_id: "user_123" + ) + """ + @spec send(PostHog.supervisor_name(), [send_option()] | map()) :: + {:ok, SendResult.t()} | {:error, Error.t()} + def send(name \\ PostHog, options) do + with {:ok, prepared} <- prepare_send(options), + {:ok, connection} <- connect(name, :send), + {:ok, result} <- deliver(connection, prepared) do + log_warnings(result) + {:ok, result} + end + rescue + exception -> {:error, unexpected(exception, "Sending the email failed unexpectedly.")} + end + + @doc """ + Reads the status of an email that `send/2` queued, for 30 days after the send. + + Returns `{:ok, %PostHog.Emails.Status{}}` or `{:error, %PostHog.Emails.Error{}}`. The error + code is `not_found` when no email of this project has the ID, when the email is older than 30 + days, or for a few seconds after the send. A key with only `hog_flow:read` can read statuses + but not send. Network errors, rate limits and an unavailable service are retried. + + ## Examples + + {:ok, %PostHog.Emails.Status{status: "bounced", reason: reason}} = + PostHog.Emails.get(message_id) + """ + @spec get(PostHog.supervisor_name(), String.t()) :: {:ok, Status.t()} | {:error, Error.t()} + def get(name \\ PostHog, message_id) do + with {:ok, message_id} <- check_message_id(message_id), + {:ok, connection} <- connect(name, :get) do + path = @status_path <> message_id <> "/" + + with_retries(connection, connection.max_retries, fn -> + connection + |> request(:get, path, headers(connection), nil) + |> parse_status_reply() + end) + end + rescue + exception -> {:error, unexpected(exception, "Reading the email status failed unexpectedly.")} + end + + defp deliver(connection, prepared) do + headers = [ + {"content-type", "application/json"}, + {"idempotency-key", prepared.idempotency_key} + | headers(connection) + ] + + with_retries(connection, prepared.max_retries || connection.max_retries, fn -> + connection + |> request(:post, @send_path, headers, prepared.body) + |> parse_send_reply() + end) + end + + # Options + + defp prepare_send(options) when is_list(options) or is_map(options) do + if is_map(options) or Keyword.keyword?(options) do + options |> Enum.reject(fn {_key, value} -> is_nil(value) end) |> Map.new() |> prepare() + else + invalid_options() + end + end + + defp prepare_send(_options), do: invalid_options() + + defp invalid_options, + do: {:error, Error.new("invalid_form", "send takes a keyword list or a map of options.")} + + defp prepare(options) do + with :ok <- check_recipient(options), + {:ok, content} <- content(options), + {:ok, idempotency_key} <- idempotency_key(options), + {:ok, max_retries} <- max_retries(options), + {:ok, body} <- encode(content, options) do + {:ok, %{body: body, idempotency_key: idempotency_key, max_retries: max_retries}} + end + end + + defp check_recipient(options) do + cond do + not nonblank?(options[:to]) -> + invalid("invalid_recipient", "to must be one email address as a nonempty string.", "to") + + Map.has_key?(options, :distinct_id) and not nonempty?(options.distinct_id) -> + invalid("invalid_distinct_id", "distinct_id must be a nonempty string.", "distinct_id") + + true -> + :ok + end + end + + defp content(%{template_id: _, template_key: _}), do: invalid_form() + defp content(%{template_id: _} = options), do: template_content(options) + defp content(%{template_key: _} = options), do: code_content(options) + defp content(_options), do: invalid_form() + + defp invalid_form, + do: + {:error, + Error.new( + "invalid_form", + "Send either template_id, or template_key with subject and html or text." + )} + + defp template_content(%{template_id: template_id} = options) do + template_data = Map.get(options, :template_data, %{}) + + cond do + not uuid?(template_id) -> + invalid("invalid_template_id", "template_id must be the template's UUID.", "template_id") + + not is_map(template_data) -> + invalid("invalid_template_data", "template_data must be a map.", "template_data") + + true -> + with :ok <- refuse_fields_outside(options, :template) do + {:ok, + options + |> Map.take([:to, :distinct_id]) + |> Map.merge(%{template_id: String.downcase(template_id), message_data: template_data})} + end + end + end + + defp code_content(%{template_key: template_key} = options) do + with :ok <- check_template_key(template_key), + :ok <- refuse_fields_outside(options, :code), + :ok <- check_code_fields(options) do + {:ok, + options + |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) + |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) + |> put_reply_to(options)} + end + end + + defp check_template_key(template_key) do + if is_binary(template_key) and template_key =~ ~r/\A[a-zA-Z0-9_-]{1,400}\z/ do + :ok + else + invalid( + "invalid_template_key", + "template_key can only have letters, digits, hyphens (-) and underscores (_), at most 400.", + "template_key" + ) + end + end + + defp check_code_fields(options) do + with :ok <- check_content(options) do + check_sender(options) + end + end + + defp check_content(options) do + cond do + not is_binary(options[:subject]) -> + invalid("invalid_form", "A send with template_key needs a subject.", "subject") + + field = Enum.find([:html, :text], ¬_string?(options, &1)) -> + invalid("invalid_form", "#{field} must be a string.", to_string(field)) + + not Map.has_key?(options, :html) and not Map.has_key?(options, :text) -> + invalid("invalid_form", "A send with template_key needs html, text or both.", "html") + + true -> + :ok + end + end + + defp check_sender(options) do + cond do + not_string?(options, :from) -> + invalid("invalid_form", "from must be a string.", "from") + + Map.has_key?(options, :reply_to) and not addresses?(options.reply_to) -> + invalid( + "invalid_form", + "reply_to must be one address or a list of addresses.", + "reply_to" + ) + + not is_boolean(Map.get(options, :capture_engagement, true)) -> + invalid("invalid_form", "capture_engagement must be true or false.", "capture_engagement") + + true -> + :ok + end + end + + defp not_string?(options, field), + do: Map.has_key?(options, field) and not is_binary(Map.fetch!(options, field)) + + defp put_reply_to(content, %{reply_to: reply_to}) when is_binary(reply_to), + do: Map.put(content, :reply_to, [reply_to]) + + defp put_reply_to(content, %{reply_to: reply_to}), do: Map.put(content, :reply_to, reply_to) + defp put_reply_to(content, _options), do: content + + defp addresses?(address) when is_binary(address), do: nonblank?(address) + defp addresses?(addresses) when is_list(addresses), do: Enum.all?(addresses, &nonblank?/1) + defp addresses?(_addresses), do: false + + defp refuse_fields_outside(options, form) do + {fields, other_fields} = + case form do + :template -> {@template_fields, @code_fields} + :code -> {@code_fields, @template_fields} + end + + case Enum.find(Map.keys(options), &(&1 not in fields)) do + nil -> + :ok + + field -> + {:error, field_outside_error(form, field, field in other_fields)} + end + end + + defp field_outside_error(form, field, other_form_takes_it?) do + form_field = if form == :template, do: "template_id", else: "template_key" + message = "A send with #{form_field} doesn't take #{field}, so no email was sent." + + if other_form_takes_it? do + {why, how} = + case form do + :template -> + {"A template made in PostHog sets its own sender, subject and content, and follows the project's engagement setting.", + "send content from your code with template_key"} + + :code -> + {"A send with template_key takes its content from your code.", + "send a template made in PostHog with template_id"} + end + + Error.new("invalid_form", message, + field: to_string(field), + why: why, + fix: "Remove #{field}, or #{how}." + ) + else + Error.new("invalid_form", message, + field: to_string(field), + fix: "Remove #{field} and send again." + ) + end + end + + defp idempotency_key(%{idempotency_key: key}) do + if is_binary(key) and key =~ ~r/\A[\x21-\x7e]{1,255}\z/ do + {:ok, key} + else + invalid( + "invalid_idempotency_key", + "idempotency_key must be 1 to 255 visible ASCII characters without spaces.", + "idempotency_key" + ) + end + end + + defp idempotency_key(_options), do: {:ok, uuid4()} + + defp max_retries(%{max_retries: retries}) when is_integer(retries) and retries >= 0, + do: {:ok, retries} + + defp max_retries(%{max_retries: _retries}), + do: + invalid("invalid_max_retries", "max_retries must be a nonnegative integer.", "max_retries") + + defp max_retries(_options), do: {:ok, nil} + + defp encode(content, options) do + case Jason.encode(content) do + {:ok, body} -> + {:ok, body} + + {:error, _reason} when is_map_key(options, :template_id) -> + invalid("invalid_template_data", "template_data must encode as JSON.", "template_data") + + {:error, _reason} -> + {:error, Error.new("invalid_form", "Every string option must be valid UTF-8.")} + end + end + + defp check_message_id(message_id) do + if uuid?(message_id) do + {:ok, String.downcase(message_id)} + else + invalid( + "invalid_message_id", + "message_id must be the message ID that send/2 returned.", + "message_id" + ) + end + end + + defp uuid?(value), + do: + is_binary(value) and + value =~ ~r/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i + + defp nonempty?(value), do: is_binary(value) and value != "" + defp nonblank?(value), do: is_binary(value) and String.trim(value) != "" + + defp invalid(code, message, field), do: {:error, Error.new(code, message, field: field)} + + defp uuid4 do + <> = :crypto.strong_rand_bytes(16) + + <> + |> Base.encode16(case: :lower) + |> then(fn <> -> + Enum.join([p1, p2, p3, p4, p5], "-") + end) + end + + # Connection + + defp connect(name, operation) do + nothing_done = + if operation == :send, do: "no email was sent", else: "no email status was read" + + config = PostHog.Registry.config(name) + + with :ok <- check_enabled(config, name, nothing_done), + {:ok, app_host} <- app_host(config), + {:ok, secret_key} <- secret_key(name, operation) do + {:ok, + %{ + api_client: config.api_client, + app_host: app_host, + secret_key: secret_key, + migrated_from: Map.get(config, :migrated_from), + timeout_ms: Map.get(config, :email_request_timeout_ms, @default_timeout_ms), + max_retries: Map.get(config, :email_request_max_retries, @default_max_retries), + sleep: Map.get(config, :email_sleep, &Process.sleep/1) + }} + end + end + + defp check_enabled(%{enabled: true, api_client: %{}}, _name, _nothing_done), do: :ok + + defp check_enabled(_config, name, nothing_done) do + {:error, + Error.new( + "client_disabled", + "The PostHog instance #{inspect(name)} is not running or has no api_key, so #{nothing_done}." + )} + end + + # A custom host is often an ingestion proxy, which must not receive the secret key. + defp app_host(config) do + case Map.get(config, :app_host) do + app_host when is_binary(app_host) -> + app_host = String.trim_trailing(app_host, "/") + {:ok, Map.get(@cloud_app_hosts, app_host, app_host)} + + nil -> + case Map.fetch(@cloud_app_hosts, String.trim_trailing(config.api_host, "/")) do + {:ok, app_host} -> + {:ok, app_host} + + :error -> + {:error, + Error.new( + "missing_app_host", + "The config's api_host is not PostHog Cloud, so the SDK can't tell where to send email. Set app_host to your PostHog app host." + )} + end + end + end + + defp secret_key(name, operation) do + case PostHog.Registry.secret_key(name) do + %PostHog.Config.Secret{} = secret -> + {:ok, PostHog.Config.Secret.reveal(secret)} + + nil -> + message = + if operation == :send, + do: + "Sending email needs secret_key, a project secret API key with the hog_flow:write scope.", + else: + "Reading an email status needs secret_key, a project secret API key with the hog_flow:read or hog_flow:write scope." + + {:error, Error.new("missing_secret_key", message)} + end + end + + defp headers(connection) do + headers = [ + {"authorization", "Bearer #{connection.secret_key}"}, + {"accept", "application/json"}, + {"user-agent", PostHog.API.Client.user_agent()} + ] + + case connection.migrated_from do + nil -> headers + migrated_from -> [{"x-posthog-migrated-from", migrated_from} | headers] + end + end + + # Requests + + defp request(connection, method, path, headers, body) do + options = + [ + headers: headers, + retry: false, + redirect: false, + compress_body: false, + receive_timeout: connection.timeout_ms + ] ++ if(body, do: [body: body], else: []) + + %{module: module, client: client} = connection.api_client + + case module.request(client, method, connection.app_host <> path, options) do + {:ok, %{status: status} = response} when is_integer(status) -> + {:ok, status, fields(Map.get(response, :body)), + retry_after_ms(Map.get(response, :headers))} + + {:error, _reason} -> + {:error, Error.new("network_error", "The request to PostHog failed.")} + end + end + + defp fields(body) when is_map(body), do: body + + defp fields(body) when is_binary(body) do + case Jason.decode(body) do + {:ok, %{} = fields} -> fields + _other -> %{} + end + end + + defp fields(_body), do: %{} + + defp with_retries(connection, max_retries, attempt_once, attempt \\ 0) do + case attempt_once.() do + {:ok, result} -> + {:ok, result} + + {:error, %Error{} = error} -> + case retry_delay_ms(error, attempt, max_retries) do + nil -> + {:error, error} + + delay_ms -> + connection.sleep.(delay_ms) + with_retries(connection, max_retries, attempt_once, attempt + 1) + end + end + end + + # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. + defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" + defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" + # Waiting doesn't fix a monthly quota. + defp retryable?(%Error{status_code: 429, code: code}), do: code != "quota_limited" + defp retryable?(%Error{status_code: status}), do: status == 503 + + defp retry_delay_ms(error, attempt, max_retries) do + cond do + attempt >= max_retries or not retryable?(error) -> + nil + + # Retrying before a longer Retry-After would break the server's rate limit. + (error.retry_after_ms || 0) > @max_retry_delay_ms -> + nil + + true -> + backoff = + min(@max_retry_delay_ms, round(1000 * 2 ** attempt * (0.8 + :rand.uniform() * 0.4))) + + max(backoff, error.retry_after_ms || 0) + end + end + + # Replies + + defp parse_send_reply({:ok, status, fields, _retry_after}) when status in 200..299 do + case text(fields["message_id"]) do + nil -> + {:error, + Error.new("api_error", "PostHog answered without a message ID.", status_code: status)} + + id -> + {:ok, + %SendResult{ + id: id, + replayed: fields["replayed"] == true, + warnings: warnings(fields["warnings"]) + }} + end + end + + defp parse_send_reply(reply), do: reply_error(reply) + + defp warnings(warnings) when is_list(warnings) do + for %{} = warning <- warnings do + %Warning{ + code: text(warning["code"]) || "warning", + message: text(warning["detail"]) || text(warning["message"]) || "", + fix: text(warning["fix"]) + } + end + end + + defp warnings(_warnings), do: [] + + defp log_warnings(%SendResult{warnings: warnings}) do + for warning <- warnings do + Logger.warning("PostHog email warning #{warning.code}: #{warning.message}", @log_metadata) + end + end + + defp parse_status_reply({:ok, status, fields, _retry_after}) when status in 200..299 do + with id when is_binary(id) <- text(fields["id"]), + delivery_status when is_binary(delivery_status) <- text(fields["status"]), + {:ok, created_at, _offset} <- DateTime.from_iso8601(text(fields["created_at"]) || ""), + {:ok, status_at, _offset} <- DateTime.from_iso8601(text(fields["status_at"]) || "") do + {:ok, + %Status{ + id: id, + status: delivery_status, + created_at: created_at, + status_at: status_at, + reason: text(fields["reason"]) + }} + else + _missing -> + {:error, + Error.new("api_error", "PostHog answered without an email status.", status_code: status)} + end + end + + defp parse_status_reply(reply), do: reply_error(reply) + + defp reply_error({:error, error}), do: {:error, error} + + defp reply_error({:ok, status, fields, retry_after_ms}) do + {:error, + Error.new( + text(fields["code"]) || "api_error", + text(fields["detail"]) || "PostHog answered HTTP #{status}.", + status_code: status, + why: text(fields["why"]), + fix: text(fields["fix"]), + docs_url: text(fields["docs_url"]), + field: text(fields["attr"]), + retry_after_ms: retry_after_ms + )} + end + + defp text(value) when is_binary(value) and value != "", do: value + defp text(_value), do: nil + + defp unexpected(exception, message) do + Logger.error("#{message} #{inspect(exception.__struct__)}", @log_metadata) + Error.new("unexpected_error", message) + end + + # Retry-After: delay-seconds or an HTTP date. + + defp retry_after_ms(headers) do + case header_value(headers, "retry-after") do + value when is_binary(value) -> parse_retry_after(String.trim(value)) + _missing -> nil + end + end + + defp header_value(headers, name) when is_map(headers) do + Enum.find_value(headers, fn {key, value} -> + if String.downcase(to_string(key)) == name, do: List.wrap(value) |> List.first() + end) + end + + defp header_value(headers, name) when is_list(headers) do + Enum.find_value(headers, fn + {key, value} -> if String.downcase(to_string(key)) == name, do: value + _other -> nil + end) + end + + defp header_value(_headers, _name), do: nil + + defp parse_retry_after(value) do + case Integer.parse(value) do + {seconds, ""} when seconds >= 0 -> seconds * 1000 + _other -> parse_http_date(value) + end + end + + @months ~w(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec) + + defp parse_http_date(value) do + with [_, day, month, year, time] <- + Regex.run(~r/\A\w{3}, (\d{2}) (\w{3}) (\d{4}) (\d{2}:\d{2}:\d{2}) GMT\z/, value), + month when is_integer(month) <- Enum.find_index(@months, &(&1 == month)), + {:ok, date} <- Date.new(String.to_integer(year), month + 1, String.to_integer(day)), + {:ok, time} <- Time.from_iso8601(time), + {:ok, at} <- DateTime.new(date, time) do + max(0, DateTime.diff(at, DateTime.utc_now(), :millisecond)) + else + _invalid -> nil + end + end +end diff --git a/lib/posthog/emails/error.ex b/lib/posthog/emails/error.ex new file mode 100644 index 00000000..7ecc6002 --- /dev/null +++ b/lib/posthog/emails/error.ex @@ -0,0 +1,85 @@ +defmodule PostHog.Emails.Error do + @moduledoc """ + PostHog did not queue the email or return its status, or the SDK could not tell whether it did. + + Branch on `:code`, PostHog's stable error code, or on `:name`, a coarse category of the + failure. The message never contains the secret key or the request body. + + ## Fields + + - `:name` - the category, one of `t:name/0`. + - `:code` - PostHog's error code, such as `"quota_limited"` or `"recipient_suppressed"`, or an + SDK code: `"missing_secret_key"`, `"missing_app_host"`, `"client_disabled"`, + `"network_error"`, `"api_error"` (an answer without a code), `"unexpected_error"`, or an + `"invalid_*"` code for an option the SDK refused before any request. + - `:message` - human-readable message. + - `:status_code` - the HTTP status, when PostHog answered. + - `:why` and `:fix` - why PostHog refused the email and how to fix it, when it said. + - `:docs_url` - a link to the docs for the error, when PostHog sent one. + - `:field` - the option the error is about, such as `"to"`. + - `:retry_after_ms` - how long PostHog asked the client to wait before it sends again. + """ + + @typedoc "A coarse category of the failure." + @type name() :: + :validation_error + | :missing_api_key + | :invalid_permission + | :not_found + | :concurrent_idempotent_requests + | :invalid_idempotent_request + | :rate_limit_exceeded + | :monthly_quota_exceeded + | :service_unavailable + | :application_error + + @typedoc "An email request that failed." + @type t() :: %__MODULE__{ + name: name(), + code: String.t(), + message: String.t(), + status_code: pos_integer() | nil, + why: String.t() | nil, + fix: String.t() | nil, + docs_url: String.t() | nil, + field: String.t() | nil, + retry_after_ms: non_neg_integer() | nil + } + + defexception [ + :name, + :code, + :message, + :status_code, + :why, + :fix, + :docs_url, + :field, + :retry_after_ms + ] + + @doc false + @spec new(String.t(), String.t(), keyword()) :: t() + def new(code, message, fields \\ []) do + status_code = fields[:status_code] + struct!(%__MODULE__{name: name(status_code, code), code: code, message: message}, fields) + end + + defp name(_status, "missing_secret_key"), do: :missing_api_key + defp name(401, _code), do: :missing_api_key + defp name(nil, "network_error"), do: :service_unavailable + defp name(nil, "invalid_" <> _), do: :validation_error + defp name(nil, _code), do: :application_error + defp name(400, _code), do: :validation_error + defp name(403, _code), do: :invalid_permission + defp name(404, _code), do: :not_found + defp name(409, "template_outdated"), do: :validation_error + defp name(409, _code), do: :concurrent_idempotent_requests + # The template send route also answers 422 for send checks such as recipient_suppressed. + defp name(422, "idempotency_key_reused"), do: :invalid_idempotent_request + defp name(422, _code), do: :validation_error + defp name(429, "quota_limited"), do: :monthly_quota_exceeded + defp name(429, _code), do: :rate_limit_exceeded + defp name(503, _code), do: :service_unavailable + defp name(_status, _code), do: :application_error +end diff --git a/lib/posthog/emails/results.ex b/lib/posthog/emails/results.ex new file mode 100644 index 00000000..948f670d --- /dev/null +++ b/lib/posthog/emails/results.ex @@ -0,0 +1,68 @@ +defmodule PostHog.Emails.Warning do + @moduledoc """ + A warning PostHog returned with a queued email. `PostHog.Emails.send/2` also logs it. + """ + + @typedoc "A warning about a queued email." + @type t() :: %__MODULE__{code: String.t(), message: String.t(), fix: String.t() | nil} + + @enforce_keys [:code, :message] + defstruct [:code, :message, :fix] +end + +defmodule PostHog.Emails.SendResult do + @moduledoc """ + The queued email. A result means PostHog queued the email, not that it was delivered. + + ## Fields + + - `:id` - the queued email's message ID. Read its status with `PostHog.Emails.get/2`. + - `:replayed` - `true` when PostHog had already accepted this idempotency key and returned the + first send's result. + - `:warnings` - the warnings PostHog returned with the email. + """ + + @typedoc "A queued email." + @type t() :: %__MODULE__{ + id: String.t(), + replayed: boolean(), + warnings: [PostHog.Emails.Warning.t()] + } + + @enforce_keys [:id] + defstruct [:id, replayed: false, warnings: []] +end + +defmodule PostHog.Emails.Status do + @moduledoc """ + What happened to an email that `PostHog.Emails.send/2` queued. + + ## Fields + + - `:id` - the message ID that `send/2` returned. + - `:status` - one of `"queued"`, `"sent"`, `"failed"`, `"delivered"`, `"bounced"` or + `"complained"`. Later versions of PostHog can return values this list does not have yet. + - `"queued"`: the email waits to go out. + - `"sent"`: PostHog handed the email to the email provider. + - `"failed"`: the email was not sent, and `:reason` says why. + - `"delivered"`: the recipient's mail server accepted the email. + - `"bounced"` and `"complained"`: the recipient's mail provider reported them after delivery. + - `:created_at` - when PostHog queued the email. + - `:status_at` - when the email reached its current status. + - `:reason` - why a failed email was not sent, such as `"provider_error"` or + `"recipient_suppressed"`, or the bounce type of a bounced email: `"hard"`, `"soft"` or + `"unknown"`. `nil` otherwise. + """ + + @typedoc "The status of a queued email." + @type t() :: %__MODULE__{ + id: String.t(), + status: String.t(), + created_at: DateTime.t(), + status_at: DateTime.t(), + reason: String.t() | nil + } + + @enforce_keys [:id, :status, :created_at, :status_at] + defstruct [:id, :status, :created_at, :status_at, :reason] +end diff --git a/lib/posthog/registry.ex b/lib/posthog/registry.ex index 90e22cd1..a634db55 100644 --- a/lib/posthog/registry.ex +++ b/lib/posthog/registry.ex @@ -15,6 +15,16 @@ defmodule PostHog.Registry do ArgumentError -> disabled_config(supervisor_name) end + # The secret key stays out of `config/1`, so it never shows up where the config is read or logged. + def secret_key(supervisor_name) do + case Registry.meta(registry_name(supervisor_name), :secret_key) do + {:ok, secret_key} -> secret_key + :error -> nil + end + rescue + ArgumentError -> nil + end + def disabled_config(supervisor_name) do %{ supervisor_name: supervisor_name, diff --git a/lib/posthog/supervisor.ex b/lib/posthog/supervisor.ex index 47bc2674..d34fe7d1 100644 --- a/lib/posthog/supervisor.ex +++ b/lib/posthog/supervisor.ex @@ -59,7 +59,7 @@ defmodule PostHog.Supervisor do {Registry, keys: :unique, name: PostHog.Registry.registry_name(config.supervisor_name), - meta: [config: public_config]}, + meta: [config: public_config, secret_key: config.secret_key]}, {PostHog.FeatureFlags.CalledCache, supervisor_name: config.supervisor_name} ] ++ definition_loader(config, callers) ++ senders(config) ++ sources(config) diff --git a/mix.exs b/mix.exs index d4590f8d..7b1fc95d 100644 --- a/mix.exs +++ b/mix.exs @@ -54,6 +54,13 @@ defmodule PostHog.MixProject do "CONTRIBUTING.md" ], groups_for_modules: [ + Emails: [ + PostHog.Emails, + PostHog.Emails.Error, + PostHog.Emails.SendResult, + PostHog.Emails.Status, + PostHog.Emails.Warning + ], Integrations: [PostHog.Integrations.Plug, PostHog.Integrations.LLMAnalytics.Req], Testing: [PostHog.Test] ], diff --git a/public_api.snapshot b/public_api.snapshot index 2c3302c2..2e070215 100644 --- a/public_api.snapshot +++ b/public_api.snapshot @@ -105,6 +105,44 @@ PostHog.Config macros: (none) +PostHog.Emails + module_doc: documented + functions: + get/1 doc: none + get/2 doc: documented + send/1 doc: none + send/2 doc: documented + macros: + (none) + +PostHog.Emails.Error + module_doc: documented + functions: + new/2 doc: none + macros: + (none) + +PostHog.Emails.SendResult + module_doc: documented + functions: + (none) + macros: + (none) + +PostHog.Emails.Status + module_doc: documented + functions: + (none) + macros: + (none) + +PostHog.Emails.Warning + module_doc: documented + functions: + (none) + macros: + (none) + PostHog.Error module_doc: documented functions: diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs new file mode 100644 index 00000000..6abfd6f1 --- /dev/null +++ b/test/posthog/emails_test.exs @@ -0,0 +1,746 @@ +defmodule PostHog.EmailsTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureLog + import Mox + + alias PostHog.API + alias PostHog.Emails + alias PostHog.Emails.Error + + setup :verify_on_exit! + + @send_url "https://us.posthog.com/api/projects/@current/messaging/send/email/" + @template_id "0190F6D2-7C1E-7B3A-9F5E-1C2D3E4F5A6B" + @message_id "01a18356-c56c-0001-e494-7666bc915188" + + setup context do + Mox.stub_with(API.Mock, API.Stub) + test_pid = self() + + config = + [ + api_host: "https://us.i.posthog.com", + api_key: "phc_project_key", + secret_key: " phs_secret_key ", + api_client_module: API.Mock, + supervisor_name: context.test, + enable_local_evaluation: false + ] + |> Keyword.merge(context[:config] || []) + |> PostHog.Config.validate!() + |> Map.put(:email_sleep, fn ms -> send(test_pid, {:slept, ms}) end) + + start_link_supervised!({PostHog.Supervisor, config}) + %{name: context.test} + end + + defp expect_reply(times \\ 1, reply) do + test_pid = self() + + expect(API.Mock, :request, times, fn _client, method, url, opts -> + send(test_pid, {:request, method, url, opts}) + reply + end) + end + + defp accepted(body \\ %{"message_id" => @message_id}), + do: {:ok, %{status: 202, headers: %{}, body: body}} + + defp refused(status, body, headers \\ %{}), + do: {:ok, %{status: status, headers: headers, body: body}} + + defp sent_body do + assert_received {:request, :post, @send_url, opts} + Jason.decode!(opts[:body]) + end + + defp header(opts, name) do + Enum.find_value(opts[:headers], fn {key, value} -> if key == name, do: value end) + end + + defp refute_requested, do: refute_received({:request, _, _, _}) + + describe "send/2 with a template made in PostHog" do + test "posts the flat template body to the app host", %{name: name} do + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id, replayed: false, warnings: []}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + template_data: %{name: "Ada"}, + distinct_id: "user_123" + ) + + assert sent_body() == %{ + "template_id" => "0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", + "to" => "user@example.com", + "message_data" => %{"name" => "Ada"}, + "distinct_id" => "user_123" + } + end + + test "sends empty message data and no distinct ID when left out", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert sent_body() == %{ + "template_id" => "0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", + "to" => "user@example.com", + "message_data" => %{} + } + end + + test "never takes the distinct ID from the process context", %{name: name} do + PostHog.set_context(name, %{distinct_id: "context_user"}) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + refute Map.has_key?(sent_body(), "distinct_id") + end + + test "refuses a code form field", %{name: name} do + assert {:error, + %Error{code: "invalid_form", field: "subject", name: :validation_error} = error} = + Emails.send(name, to: "user@example.com", template_id: @template_id, subject: "Hi") + + assert error.fix =~ "template_key" + assert error.why =~ "template made in PostHog" + refute_requested() + end + + test "refuses a template ID that is not a UUID", %{name: name} do + assert {:error, %Error{code: "invalid_template_id", field: "template_id"}} = + Emails.send(name, to: "user@example.com", template_id: "welcome") + + refute_requested() + end + + test "refuses template data that is not a map", %{name: name} do + assert {:error, %Error{code: "invalid_template_data"}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + template_data: [name: "Ada"] + ) + + refute_requested() + end + + test "refuses template data that can't be encoded as JSON", %{name: name} do + assert {:error, %Error{code: "invalid_template_data"}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + template_data: %{pid: self()} + ) + + refute_requested() + end + end + + describe "send/2 with content from code" do + test "posts the rendered content with engagement capture on by default", %{name: name} do + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send(name, + to: "user@example.com", + distinct_id: "user_123", + template_key: "password-reset", + subject: "Reset your password", + html: "

Reset

", + text: "Reset", + from: "Acme ", + reply_to: "support@example.com" + ) + + assert sent_body() == %{ + "to" => "user@example.com", + "distinct_id" => "user_123", + "template_key" => "password-reset", + "subject" => "Reset your password", + "html" => "

Reset

", + "text" => "Reset", + "from" => "Acme ", + "reply_to" => ["support@example.com"], + "capture_engagement" => true + } + end + + test "sends only the fields given, and capture_engagement false", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send(name, + to: "user@example.com", + template_key: "receipt", + subject: "Your receipt", + text: "Thanks", + reply_to: ["a@example.com", "b@example.com"], + capture_engagement: false, + html: nil + ) + + assert sent_body() == %{ + "to" => "user@example.com", + "template_key" => "receipt", + "subject" => "Your receipt", + "text" => "Thanks", + "reply_to" => ["a@example.com", "b@example.com"], + "capture_engagement" => false + } + end + + test "needs html, text or both", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "html"}} = + Emails.send(name, to: "user@example.com", template_key: "receipt", subject: "Hi") + + refute_requested() + end + + test "needs a subject", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "subject"}} = + Emails.send(name, to: "user@example.com", template_key: "receipt", html: "

") + + refute_requested() + end + + test "refuses template data on the code form", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "template_data"} = error} = + Emails.send(name, + to: "user@example.com", + template_key: "receipt", + subject: "Hi", + html: "

", + template_data: %{a: 1} + ) + + assert error.fix =~ "template_id" + refute_requested() + end + + test "refuses a template key with other characters", %{name: name} do + assert {:error, %Error{code: "invalid_template_key", name: :validation_error}} = + Emails.send(name, + to: "user@example.com", + template_key: "password reset", + subject: "Hi", + html: "

" + ) + + refute_requested() + end + + test "refuses a reply_to that is not addresses", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "reply_to"}} = + Emails.send(name, + to: "user@example.com", + template_key: "receipt", + subject: "Hi", + html: "

", + reply_to: [:nope] + ) + + refute_requested() + end + + test "refuses a capture_engagement that is not a boolean", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "capture_engagement"}} = + Emails.send(name, + to: "user@example.com", + template_key: "receipt", + subject: "Hi", + html: "

", + capture_engagement: "yes" + ) + + refute_requested() + end + end + + describe "send/2 argument checks" do + test "needs exactly one form", %{name: name} do + assert {:error, %Error{code: "invalid_form"}} = Emails.send(name, to: "user@example.com") + + assert {:error, %Error{code: "invalid_form"}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + template_key: "receipt" + ) + + refute_requested() + end + + test "refuses an option neither form takes", %{name: name} do + assert {:error, %Error{code: "invalid_form", field: "cc", fix: "Remove cc and send again."}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + cc: "x@example.com" + ) + + refute_requested() + end + + test "refuses options that are not a keyword list or map", %{name: name} do + assert {:error, %Error{code: "invalid_form"}} = Emails.send(name, ["user@example.com"]) + refute_requested() + end + + test "takes a map of options", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, %{to: "user@example.com", template_id: @template_id}) + end + + test "checks to, distinct_id, idempotency_key and max_retries", %{name: name} do + base = [to: "user@example.com", template_id: @template_id] + + for {opts, code} <- [ + {[to: " "], "invalid_recipient"}, + {[to: ["a@example.com"]], "invalid_recipient"}, + {[distinct_id: ""], "invalid_distinct_id"}, + {[idempotency_key: "has space"], "invalid_idempotency_key"}, + {[idempotency_key: String.duplicate("a", 256)], "invalid_idempotency_key"}, + {[max_retries: -1], "invalid_max_retries"}, + {[max_retries: 1.0], "invalid_max_retries"} + ] do + assert {:error, %Error{code: ^code, name: :validation_error}} = + Emails.send(name, Keyword.merge(base, opts)) + end + + refute_requested() + end + end + + describe "send/2 headers and hosts" do + test "authenticates with the trimmed secret key and sends the SDK headers", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + idempotency_key: "welcome:user_123" + ) + + assert_received {:request, :post, @send_url, opts} + assert header(opts, "authorization") == "Bearer phs_secret_key" + assert header(opts, "content-type") == "application/json" + assert header(opts, "accept") == "application/json" + assert header(opts, "idempotency-key") == "welcome:user_123" + assert header(opts, "user-agent") == PostHog.API.Client.user_agent() + assert header(opts, "x-posthog-migrated-from") == nil + assert opts[:retry] == false + assert opts[:redirect] == false + assert opts[:compress_body] == false + assert opts[:receive_timeout] == 15_000 + refute Keyword.has_key?(opts, :json) + end + + @tag config: [migrated_from: "resend", email_request_timeout_ms: 5_000] + test "sends the migration source and the request timeout", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:request, :post, @send_url, opts} + assert header(opts, "x-posthog-migrated-from") == "resend" + assert opts[:receive_timeout] == 5_000 + end + + @tag config: [api_host: "https://eu.i.posthog.com"] + test "maps an EU ingestion host to the EU app host", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:request, :post, + "https://eu.posthog.com/api/projects/@current/messaging/send/email/", _} + end + + @tag config: [api_host: "https://proxy.example.com"] + test "refuses a custom host without an app host", %{name: name} do + assert {:error, %Error{code: "missing_app_host", name: :application_error}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + refute_requested() + end + + @tag config: [ + api_host: "https://proxy.example.com", + app_host: " https://posthog.example.com/ " + ] + test "sends to the app host the client sets", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:request, :post, + "https://posthog.example.com/api/projects/@current/messaging/send/email/", + _} + end + + @tag config: [app_host: "https://eu.i.posthog.com"] + test "maps a cloud ingestion host set as the app host", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:request, :post, + "https://eu.posthog.com/api/projects/@current/messaging/send/email/", _} + end + + @tag config: [secret_key: nil] + test "refuses to send without a secret key", %{name: name} do + assert {:error, %Error{code: "missing_secret_key", name: :missing_api_key} = error} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert error.message =~ "hog_flow:write" + refute_requested() + end + + test "a client that is not running sends nothing" do + assert {:error, %Error{code: "client_disabled", name: :application_error}} = + Emails.send(:not_started, to: "user@example.com", template_id: @template_id) + + refute_requested() + end + + test "never puts the secret key or the body in an error", %{name: name} do + expect_reply(refused(400, %{"code" => "invalid_request", "detail" => "Bad field."})) + + assert {:error, error} = + Emails.send(name, + to: "user@example.com", + template_key: "k", + subject: "s", + html: "secret-html" + ) + + rendered = inspect(error) <> Exception.message(error) + refute rendered =~ "phs_secret_key" + refute rendered =~ "secret-html" + end + end + + describe "send/2 idempotency and retries" do + test "every attempt sends the same generated UUID v4 key", %{name: name} do + expect_reply(refused(503, %{"code" => "service_unavailable"})) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:request, :post, _, first} + assert_received {:request, :post, _, second} + key = header(first, "idempotency-key") + assert key == header(second, "idempotency-key") + assert key =~ ~r/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + end + + test "makes 3 attempts by default and waits about 1 and then 2 seconds", %{name: name} do + expect_reply(3, refused(503, nil)) + + assert {:error, %Error{code: "api_error", status_code: 503, name: :service_unavailable}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:slept, first} + assert_received {:slept, second} + refute_received {:slept, _} + assert first in 800..1200 + assert second in 1600..2400 + end + + test "a per-send max_retries of 0 makes one attempt", %{name: name} do + expect_reply(refused(503, %{"code" => "service_unavailable"})) + + assert {:error, %Error{code: "service_unavailable"}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 0 + ) + + refute_received {:slept, _} + end + + @tag config: [email_request_max_retries: 0] + test "the client's email_request_max_retries sets the default", %{name: name} do + expect_reply(refused(503, nil)) + + assert {:error, %Error{status_code: 503}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + end + + test "retries network errors, throttling and an in-flight key", %{name: name} do + expect_reply({:error, %Req.TransportError{reason: :econnrefused}}) + expect_reply(refused(429, %{"code" => "throttled"})) + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 3 + ) + + expect_reply(refused(409, %{"code" => "idempotency_key_in_flight"})) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + end + + test "a network error on the last attempt is service_unavailable", %{name: name} do + expect_reply({:error, %Req.TransportError{reason: :timeout}}) + + assert {:error, %Error{code: "network_error", name: :service_unavailable, status_code: nil}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 0 + ) + end + + test "does not retry a monthly quota, a conflict or a suppressed recipient", %{name: name} do + for {status, code, error_name} <- [ + {429, "quota_limited", :monthly_quota_exceeded}, + {409, "template_outdated", :validation_error}, + {422, "recipient_suppressed", :validation_error}, + {422, "idempotency_key_reused", :invalid_idempotent_request}, + {400, "invalid_recipient", :validation_error}, + {401, "authentication_failed", :missing_api_key}, + {403, "permission_denied", :invalid_permission}, + {404, "not_found", :not_found}, + {500, "server_error", :application_error} + ] do + expect_reply(refused(status, %{"code" => code})) + + assert {:error, %Error{code: ^code, name: ^error_name, status_code: ^status}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + end + + refute_received {:slept, _} + end + + test "an in-flight key that stays in flight is concurrent_idempotent_requests", %{name: name} do + expect_reply(refused(409, %{"code" => "idempotency_key_in_flight"})) + + assert {:error, %Error{name: :concurrent_idempotent_requests}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 0 + ) + end + + test "waits for a Retry-After longer than the backoff", %{name: name} do + expect_reply(refused(429, %{"code" => "throttled"}, %{"retry-after" => ["7"]})) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, to: "user@example.com", template_id: @template_id) + assert_received {:slept, 7_000} + end + + test "reads a Retry-After date", %{name: name} do + later = + DateTime.utc_now() + |> DateTime.add(20) + |> Calendar.strftime("%a, %d %b %Y %H:%M:%S GMT") + + expect_reply(refused(503, %{"code" => "service_unavailable"}, [{"retry-after", later}])) + + assert {:error, %Error{retry_after_ms: wait}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 0 + ) + + assert wait in 18_000..20_000 + end + + test "fails at once on a Retry-After over 30 seconds", %{name: name} do + expect_reply(refused(429, %{"code" => "throttled"}, %{"retry-after" => ["60"]})) + + assert {:error, + %Error{code: "throttled", retry_after_ms: 60_000, name: :rate_limit_exceeded}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + refute_received {:slept, _} + end + + test "an acceptance without a message ID is api_error and not retried", %{name: name} do + expect_reply(refused(202, %{})) + + assert {:error, %Error{code: "api_error", status_code: 202}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + refute_received {:slept, _} + end + + @tag capture_log: true + test "a client module that raises is unexpected_error", %{name: name} do + expect(API.Mock, :request, fn _client, _method, _url, _opts -> raise "boom" end) + + assert {:error, %Error{code: "unexpected_error", name: :application_error}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + end + end + + describe "send/2 replies" do + test "keeps every field of a refusal", %{name: name} do + expect_reply( + refused(422, %{ + "type" => "validation_error", + "code" => "recipient_suppressed", + "detail" => "Nope.", + "attr" => "to", + "why" => "The address bounced.", + "fix" => "Send to another address.", + "docs_url" => "https://posthog.com/docs" + }) + ) + + assert {:error, error} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert %Error{ + name: :validation_error, + code: "recipient_suppressed", + message: "Nope.", + status_code: 422, + field: "to", + why: "The address bounced.", + fix: "Send to another address.", + docs_url: "https://posthog.com/docs" + } = error + end + + test "reads a JSON body the client module did not decode", %{name: name} do + expect_reply(refused(404, ~s({"code": "not_found", "detail": "No template."}))) + + assert {:error, %Error{code: "not_found", message: "No template."}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + end + + test "returns a replay and logs each warning", %{name: name} do + expect_reply( + accepted(%{ + "message_id" => @message_id, + "replayed" => true, + "warnings" => [ + %{"code" => "no_text", "detail" => "No text part.", "fix" => "Add text."} + ] + }) + ) + + log = + capture_log(fn -> + assert {:ok, %Emails.SendResult{replayed: true, warnings: [warning]}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert warning == %Emails.Warning{ + code: "no_text", + message: "No text part.", + fix: "Add text." + } + end) + + assert log =~ "PostHog email warning no_text: No text part." + end + end + + describe "get/2" do + test "reads the status of a sent email", %{name: name} do + expect_reply( + {:ok, + %{ + status: 200, + headers: %{}, + body: %{ + "id" => @message_id, + "status" => "bounced", + "created_at" => "2026-10-28T16:27:44.364000Z", + "status_at" => "2026-10-28T16:30:02.100000Z", + "reason" => "hard" + } + }} + ) + + assert {:ok, %Emails.Status{} = status} = Emails.get(name, String.upcase(@message_id)) + + assert status.id == @message_id + assert status.status == "bounced" + assert status.reason == "hard" + assert status.created_at == ~U[2026-10-28 16:27:44.364000Z] + assert status.status_at == ~U[2026-10-28 16:30:02.100000Z] + + assert_received {:request, :get, + "https://us.posthog.com/api/projects/@current/messaging/emails/01a18356-c56c-0001-e494-7666bc915188/", + opts} + + assert header(opts, "authorization") == "Bearer phs_secret_key" + assert header(opts, "idempotency-key") == nil + refute Keyword.has_key?(opts, :body) + end + + test "passes an unknown status through", %{name: name} do + expect_reply( + {:ok, + %{ + status: 200, + body: %{ + "id" => @message_id, + "status" => "archived", + "created_at" => "2026-10-28T16:27:44Z", + "status_at" => "2026-10-28T16:27:44Z" + } + }} + ) + + assert {:ok, %Emails.Status{status: "archived", reason: nil}} = + Emails.get(name, @message_id) + end + + test "a record without its times is api_error", %{name: name} do + expect_reply({:ok, %{status: 200, body: %{"id" => @message_id, "status" => "sent"}}}) + + assert {:error, %Error{code: "api_error", status_code: 200}} = Emails.get(name, @message_id) + end + + test "does not retry an unknown ID", %{name: name} do + expect_reply(refused(404, %{"code" => "not_found"})) + + assert {:error, %Error{code: "not_found", name: :not_found}} = Emails.get(name, @message_id) + refute_received {:slept, _} + end + + test "refuses an ID that is not a UUID", %{name: name} do + assert {:error, %Error{code: "invalid_message_id", name: :validation_error}} = + Emails.get(name, "msg_1") + + refute_requested() + end + end + + describe "the default instance" do + test "send/1 and get/1 use the PostHog instance" do + # The test config doesn't start the PostHog instance. + assert {:error, %Error{code: "client_disabled"}} = + Emails.send(to: "user@example.com", template_id: @template_id) + + assert {:error, %Error{code: "invalid_message_id"}} = Emails.get("nope") + end + end + + describe "config" do + test "validates migrated_from as a header token" do + assert {:error, %NimbleOptions.ValidationError{}} = + PostHog.Config.validate(api_key: "phc", migrated_from: "re send") + end + end +end From dc04ec9d19fdb3ccee6537263a934397d6e7e3d8 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:00:39 +0000 Subject: [PATCH 02/38] fix(emails): bound each request, parse replies in the SDK, require an absolute app_host - Turn off Req's body decoding, so a 2xx with malformed JSON is an api_error with its status instead of a retried network_error. - Run each request in a task stopped at email_request_timeout_ms, so the timeout bounds the whole request, not only each socket read. - Validate app_host as an absolute http(s) URL, so a relative value can't resolve against the ingestion host and send it the secret key. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/config_schema.ex | 16 +++++- lib/posthog/emails.ex | 32 +++++++++++- test/posthog/emails_test.exs | 94 +++++++++++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/lib/posthog/config_schema.ex b/lib/posthog/config_schema.ex index 0889281d..dcd08143 100644 --- a/lib/posthog/config_schema.ex +++ b/lib/posthog/config_schema.ex @@ -51,7 +51,7 @@ defmodule PostHog.Config.Schema do """ ], app_host: [ - type: {:or, [:string, nil]}, + type: {:or, [{:custom, __MODULE__, :http_url, []}, nil]}, default: nil, doc: """ The PostHog app host that `PostHog.Emails` sends to, such as `https://posthog.example.com` @@ -185,6 +185,20 @@ defmodule PostHog.Config.Schema do ] ++ shared() end + @doc false + def http_url(value) when is_binary(value) do + case URI.parse(value) do + %URI{scheme: scheme, host: host} + when scheme in ["http", "https"] and host not in [nil, ""] -> + {:ok, value} + + _other -> + {:error, "expected an absolute http or https URL, got: #{inspect(value)}"} + end + end + + def http_url(value), do: {:error, "expected a string, got: #{inspect(value)}"} + @doc false def header_token(value) when is_binary(value) do if value =~ ~r/\A[\x21-\x7e]+\z/ do diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index f0b6d3bf..dd4fe1d7 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -79,7 +79,8 @@ defmodule PostHog.Emails do already retries the send, pass `max_retries: 0` and an `:idempotency_key` that names the email, such as `"welcome:user_123"`. A generated key doesn't survive a job retry. - Sends and status reads run in the calling process. The SDK keeps no email queue. + Sends and status reads block the calling process until they finish. The SDK keeps no email + queue. ## Errors @@ -616,18 +617,22 @@ defmodule PostHog.Emails do # Requests defp request(connection, method, path, headers, body) do + # The SDK parses the body itself, so a malformed acceptance stays an api_error with its status. options = [ headers: headers, retry: false, redirect: false, compress_body: false, + decode_body: false, receive_timeout: connection.timeout_ms ] ++ if(body, do: [body: body], else: []) %{module: module, client: client} = connection.api_client - case module.request(client, method, connection.app_host <> path, options) do + case within_deadline(connection.timeout_ms, fn -> + module.request(client, method, connection.app_host <> path, options) + end) do {:ok, %{status: status} = response} when is_integer(status) -> {:ok, status, fields(Map.get(response, :body)), retry_after_ms(Map.get(response, :headers))} @@ -637,6 +642,29 @@ defmodule PostHog.Emails do end end + # The client's own timeouts bound each phase, not the whole request, so the request runs in a + # task that is stopped at the deadline. Failures inside it come back to the caller. + defp within_deadline(timeout_ms, fun) do + task = + Task.async(fn -> + try do + {:returned, fun.()} + rescue + exception -> {:raised, exception, __STACKTRACE__} + catch + kind, reason -> {:caught, kind, reason, __STACKTRACE__} + end + end) + + case Task.yield(task, timeout_ms) || Task.shutdown(task, :brutal_kill) do + {:ok, {:returned, result}} -> result + {:ok, {:raised, exception, stacktrace}} -> reraise exception, stacktrace + {:ok, {:caught, kind, reason, stacktrace}} -> :erlang.raise(kind, reason, stacktrace) + # Stopped at the deadline, or ended without a reply. + _timeout_or_exit -> {:error, :timeout} + end + end + defp fields(body) when is_map(body), do: body defp fields(body) when is_binary(body) do diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 6abfd6f1..96806725 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -1,3 +1,20 @@ +defmodule PostHog.EmailsTest.ReqTestClient do + @moduledoc false + # The default client, with Req.Test in place of the network. + @behaviour PostHog.API.Client + + alias PostHog.API.Client + + @impl true + def client(api_key, api_host) do + default = Client.client(api_key, api_host) + %{default | client: Req.merge(default.client, plug: {Req.Test, __MODULE__})} + end + + @impl true + defdelegate request(client, method, url, opts), to: Client +end + defmodule PostHog.EmailsTest do use ExUnit.Case, async: true @@ -333,7 +350,7 @@ defmodule PostHog.EmailsTest do assert header(opts, "content-type") == "application/json" assert header(opts, "accept") == "application/json" assert header(opts, "idempotency-key") == "welcome:user_123" - assert header(opts, "user-agent") == PostHog.API.Client.user_agent() + assert header(opts, "user-agent") == API.Client.user_agent() assert header(opts, "x-posthog-migrated-from") == nil assert opts[:retry] == false assert opts[:redirect] == false @@ -742,5 +759,80 @@ defmodule PostHog.EmailsTest do assert {:error, %NimbleOptions.ValidationError{}} = PostHog.Config.validate(api_key: "phc", migrated_from: "re send") end + + test "refuses an app_host that is not an absolute HTTP(S) URL" do + for app_host <- ["posthog.example.com", "/", "ftp://posthog.example.com", "https://"] do + assert {:error, %NimbleOptions.ValidationError{key: :app_host}} = + PostHog.Config.validate(api_key: "phc", app_host: app_host) + end + + assert {:ok, %{app_host: "http://localhost:8010"}} = + PostHog.Config.validate(api_key: "phc", app_host: " http://localhost:8010 ") + end + end + + describe "through the default Req client" do + @describetag config: [api_client_module: PostHog.EmailsTest.ReqTestClient] + + test "an acceptance with malformed JSON is api_error with its status, not retried", %{ + name: name + } do + Req.Test.expect(__MODULE__.ReqTestClient, fn conn -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(202, "{not json") + end) + + assert {:error, %Error{code: "api_error", status_code: 202}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + refute_received {:slept, _} + end + + test "sends the body as is, without the project API key", %{name: name} do + test_pid = self() + + Req.Test.expect(__MODULE__.ReqTestClient, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test_pid, {:raw, conn.method, conn.host, conn.request_path, conn.req_headers, body}) + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(202, ~s({"message_id": "#{@message_id}"})) + end) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:raw, "POST", "us.posthog.com", + "/api/projects/@current/messaging/send/email/", headers, body} + + refute Map.has_key?(Jason.decode!(body), "api_key") + assert {"content-encoding", "gzip"} not in headers + + assert Enum.count(headers, fn {name, _value} -> name == "user-agent" end) == 1 + end + + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + email_request_timeout_ms: 50 + ] + test "a request that outlasts the timeout is a network error at the deadline", %{name: name} do + Req.Test.expect(__MODULE__.ReqTestClient, fn conn -> + Process.sleep(2_000) + Plug.Conn.send_resp(conn, 202, "{}") + end) + + started = System.monotonic_time(:millisecond) + + assert {:error, %Error{code: "network_error", name: :service_unavailable}} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + max_retries: 0 + ) + + assert System.monotonic_time(:millisecond) - started < 1_000 + end end end From 9a35f9c2e41122ebc077eba2ca740ecc9e5c397a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:04:18 +0000 Subject: [PATCH 03/38] fix(emails): run each request in a monitored worker, not a linked task A linked Task left {:EXIT, pid, :normal} in the mailbox of a caller that traps exits, which crashes a GenServer without a matching handle_info/2. The worker is now spawned with a monitor only, and the deadline path waits for its DOWN message and drops a late reply, so the caller's mailbox stays clean after a reply and after a timeout. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 61 ++++++++++++++++++++++++++---------- test/posthog/emails_test.exs | 28 +++++++++++++++++ 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index dd4fe1d7..efcdf0c6 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -643,28 +643,57 @@ defmodule PostHog.Emails do end # The client's own timeouts bound each phase, not the whole request, so the request runs in a - # task that is stopped at the deadline. Failures inside it come back to the caller. + # worker that is killed at the deadline. The worker is monitored, not linked, so a caller that + # traps exits gets no exit message. Failures inside it come back to the caller. defp within_deadline(timeout_ms, fun) do - task = - Task.async(fn -> - try do - {:returned, fun.()} - rescue - exception -> {:raised, exception, __STACKTRACE__} - catch - kind, reason -> {:caught, kind, reason, __STACKTRACE__} - end + caller = self() + reply = make_ref() + # Keeps test allowances, such as Mox's, for the worker. + callers = [caller | Process.get(:"$callers", [])] + + {worker, monitor} = + spawn_monitor(fn -> + Process.put(:"$callers", callers) + Kernel.send(caller, {reply, run_capturing(fun)}) end) - case Task.yield(task, timeout_ms) || Task.shutdown(task, :brutal_kill) do - {:ok, {:returned, result}} -> result - {:ok, {:raised, exception, stacktrace}} -> reraise exception, stacktrace - {:ok, {:caught, kind, reason, stacktrace}} -> :erlang.raise(kind, reason, stacktrace) - # Stopped at the deadline, or ended without a reply. - _timeout_or_exit -> {:error, :timeout} + receive do + {^reply, result} -> + Process.demonitor(monitor, [:flush]) + unwrap(result) + + {:DOWN, ^monitor, :process, ^worker, _reason} -> + {:error, :worker_down} + after + timeout_ms -> + Process.exit(worker, :kill) + # The DOWN message comes after any reply the worker sent, so both leave the mailbox here. + receive do + {:DOWN, ^monitor, :process, ^worker, _reason} -> :ok + end + + receive do + {^reply, _late} -> :ok + after + 0 -> :ok + end + + {:error, :timeout} end end + defp run_capturing(fun) do + {:returned, fun.()} + rescue + exception -> {:raised, exception, __STACKTRACE__} + catch + kind, reason -> {:caught, kind, reason, __STACKTRACE__} + end + + defp unwrap({:returned, result}), do: result + defp unwrap({:raised, exception, stacktrace}), do: reraise(exception, stacktrace) + defp unwrap({:caught, kind, reason, stacktrace}), do: :erlang.raise(kind, reason, stacktrace) + defp fields(body) when is_map(body), do: body defp fields(body) when is_binary(body) do diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 96806725..ee29153a 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -608,6 +608,34 @@ defmodule PostHog.EmailsTest do end end + describe "send/2 in a caller that traps exits" do + @tag config: [email_request_timeout_ms: 50] + test "leaves the caller's mailbox clean after a reply and after a timeout", %{name: name} do + expect_reply(accepted()) + + expect(API.Mock, :request, fn _client, _method, _url, _opts -> + Process.sleep(1_000) + accepted() + end) + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + sent = Emails.send(name, to: "user@example.com", template_id: @template_id) + + timed_out = + Emails.send(name, to: "user@example.com", template_id: @template_id, max_retries: 0) + + # Give any late exit or reply message time to arrive. + Process.sleep(100) + {sent, timed_out, Process.info(self(), :messages)} + end) + + assert {{:ok, _}, {:error, %Error{code: "network_error"}}, {:messages, []}} = + Task.await(task) + end + end + describe "send/2 replies" do test "keeps every field of a refusal", %{name: name} do expect_reply( From a04a561538f8381a3c6e4db346e89c396e203ef0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:08:34 +0000 Subject: [PATCH 04/38] fix(emails): stop the request worker when its caller dies The unlinked worker outlived a caller killed mid-request, such as by a job timeout, so the request could still go out after cancellation. The worker is now spawned linked and monitored in one step. It unlinks before it replies, and the caller unlinks before it kills the worker at the deadline, so a caller that traps exits still gets no exit message. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 19 +++++++++++++------ test/posthog/emails_test.exs | 27 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index efcdf0c6..c4fe3225 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -643,8 +643,9 @@ defmodule PostHog.Emails do end # The client's own timeouts bound each phase, not the whole request, so the request runs in a - # worker that is killed at the deadline. The worker is monitored, not linked, so a caller that - # traps exits gets no exit message. Failures inside it come back to the caller. + # worker that is killed at the deadline. The worker is linked, so it dies with a caller that is + # killed or shut down mid-request, and monitored. Each side unlinks before it ends the worker's + # life, so a caller that traps exits gets no exit message. Failures come back to the caller. defp within_deadline(timeout_ms, fun) do caller = self() reply = make_ref() @@ -652,10 +653,15 @@ defmodule PostHog.Emails do callers = [caller | Process.get(:"$callers", [])] {worker, monitor} = - spawn_monitor(fn -> - Process.put(:"$callers", callers) - Kernel.send(caller, {reply, run_capturing(fun)}) - end) + :erlang.spawn_opt( + fn -> + Process.put(:"$callers", callers) + result = run_capturing(fun) + Process.unlink(caller) + Kernel.send(caller, {reply, result}) + end, + [:link, :monitor] + ) receive do {^reply, result} -> @@ -666,6 +672,7 @@ defmodule PostHog.Emails do {:error, :worker_down} after timeout_ms -> + Process.unlink(worker) Process.exit(worker, :kill) # The DOWN message comes after any reply the worker sent, so both leave the mailbox here. receive do diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index ee29153a..2c7b03f4 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -636,6 +636,33 @@ defmodule PostHog.EmailsTest do end end + describe "send/2 when its caller dies" do + @tag config: [email_request_timeout_ms: 5_000] + test "the request stops with the caller", %{name: name} do + test_pid = self() + + expect(API.Mock, :request, fn _client, _method, _url, _opts -> + send(test_pid, {:requesting, self()}) + Process.sleep(500) + send(test_pid, :request_finished) + accepted() + end) + + caller = + spawn(fn -> + Process.put(:"$callers", [test_pid]) + Emails.send(name, to: "user@example.com", template_id: @template_id) + end) + + assert_receive {:requesting, worker} + worker_ref = Process.monitor(worker) + Process.exit(caller, :kill) + + assert_receive {:DOWN, ^worker_ref, :process, ^worker, _reason} + refute_receive :request_finished, 700 + end + end + describe "send/2 replies" do test "keeps every field of a refusal", %{name: name} do expect_reply( From 63923636cfdb50f13ad1436ac8992a0f14272545 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:13:30 +0000 Subject: [PATCH 05/38] fix(emails): keep request worker failures out of the caller A linked worker killed its caller when it crashed. The worker is now only monitored, so its death returns network_error, and a watchdog kills it when the caller dies. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 33 ++++++++++++++++++++------------- test/posthog/emails_test.exs | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index c4fe3225..336668f0 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -643,9 +643,9 @@ defmodule PostHog.Emails do end # The client's own timeouts bound each phase, not the whole request, so the request runs in a - # worker that is killed at the deadline. The worker is linked, so it dies with a caller that is - # killed or shut down mid-request, and monitored. Each side unlinks before it ends the worker's - # life, so a caller that traps exits gets no exit message. Failures come back to the caller. + # monitored worker that is killed at the deadline. The worker is not linked, so its failure comes + # back as an error rather than killing the caller, and a caller that traps exits gets no exit + # message. A watchdog kills the worker when the caller dies, so the request ends with its caller. defp within_deadline(timeout_ms, fun) do caller = self() reply = make_ref() @@ -653,15 +653,12 @@ defmodule PostHog.Emails do callers = [caller | Process.get(:"$callers", [])] {worker, monitor} = - :erlang.spawn_opt( - fn -> - Process.put(:"$callers", callers) - result = run_capturing(fun) - Process.unlink(caller) - Kernel.send(caller, {reply, result}) - end, - [:link, :monitor] - ) + spawn_monitor(fn -> + worker = self() + spawn(fn -> watch(caller, worker) end) + Process.put(:"$callers", callers) + Kernel.send(caller, {reply, run_capturing(fun)}) + end) receive do {^reply, result} -> @@ -672,7 +669,6 @@ defmodule PostHog.Emails do {:error, :worker_down} after timeout_ms -> - Process.unlink(worker) Process.exit(worker, :kill) # The DOWN message comes after any reply the worker sent, so both leave the mailbox here. receive do @@ -689,6 +685,17 @@ defmodule PostHog.Emails do end end + # Monitoring a caller that is already gone delivers DOWN at once, so no window is missed. + defp watch(caller, worker) do + caller_monitor = Process.monitor(caller) + worker_monitor = Process.monitor(worker) + + receive do + {:DOWN, ^caller_monitor, :process, ^caller, _reason} -> Process.exit(worker, :kill) + {:DOWN, ^worker_monitor, :process, ^worker, _reason} -> :ok + end + end + defp run_capturing(fun) do {:returned, fun.()} rescue diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 2c7b03f4..62de30d3 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -663,6 +663,25 @@ defmodule PostHog.EmailsTest do end end + describe "send/2 when its request worker dies" do + test "returns network_error and leaves the caller running", %{name: name} do + expect(API.Mock, :request, fn _client, _method, _url, _opts -> + Process.exit(self(), :kill) + end) + + task = + Task.async(fn -> + result = + Emails.send(name, to: "user@example.com", template_id: @template_id, max_retries: 0) + + Process.sleep(50) + {result, Process.info(self(), :messages)} + end) + + assert {{:error, %Error{code: "network_error"}}, {:messages, []}} = Task.await(task) + end + end + describe "send/2 replies" do test "keeps every field of a refusal", %{name: name} do expect_reply( From 2244ae8d271ea34510d92e3a5a2980dfa82ffc44 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:37:38 +0000 Subject: [PATCH 06/38] test(emails): run the default-instance test in the PostHog group Other async modules start the default PostHog instance, so asserting it was not running flaked on CI. The test now starts the instance in the PostHog group and checks that send/1 and get/1 reach it. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_default_instance_test.exs | 53 +++++++++++++++++++ test/posthog/emails_test.exs | 10 ---- 2 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 test/posthog/emails_default_instance_test.exs diff --git a/test/posthog/emails_default_instance_test.exs b/test/posthog/emails_default_instance_test.exs new file mode 100644 index 00000000..efc8777b --- /dev/null +++ b/test/posthog/emails_default_instance_test.exs @@ -0,0 +1,53 @@ +defmodule PostHog.EmailsDefaultInstanceTest do + # Other modules start the default PostHog instance too, so this one runs in their group. + use PostHog.Case, + async: Version.match?(System.version(), ">= 1.18.0"), + group: PostHog + + @moduletag config: [supervisor_name: PostHog, secret_key: "phs_secret_key"] + + import Mox + + alias PostHog.Emails + + setup :setup_supervisor + setup :verify_on_exit! + + @message_id "01a18356-c56c-0001-e494-7666bc915188" + + test "send/1 and get/1 use the PostHog instance" do + expect(PostHog.API.Mock, :request, fn _client, :post, url, opts -> + assert url == "https://us.posthog.com/api/projects/@current/messaging/send/email/" + assert {"authorization", "Bearer phs_secret_key"} in opts[:headers] + {:ok, %{status: 202, headers: %{}, body: %{"message_id" => @message_id}}} + end) + + expect(PostHog.API.Mock, :request, fn _client, :get, url, _opts -> + assert url == + "https://us.posthog.com/api/projects/@current/messaging/emails/#{@message_id}/" + + {:ok, + %{ + status: 200, + headers: %{}, + body: %{ + "id" => @message_id, + "status" => "queued", + "created_at" => "2026-11-02T10:00:00Z", + "status_at" => "2026-11-02T10:00:00Z", + "reason" => nil + } + }} + end) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send( + to: "user@example.com", + template_key: "receipt", + subject: "Hi", + text: "Hi" + ) + + assert {:ok, %Emails.Status{id: @message_id, status: "queued"}} = Emails.get(@message_id) + end +end diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 62de30d3..46578cc2 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -818,16 +818,6 @@ defmodule PostHog.EmailsTest do end end - describe "the default instance" do - test "send/1 and get/1 use the PostHog instance" do - # The test config doesn't start the PostHog instance. - assert {:error, %Error{code: "client_disabled"}} = - Emails.send(to: "user@example.com", template_id: @template_id) - - assert {:error, %Error{code: "invalid_message_id"}} = Emails.get("nope") - end - end - describe "config" do test "validates migrated_from as a header token" do assert {:error, %NimbleOptions.ValidationError{}} = From f94951540bf901315b527d0993fdd7150eb17007 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:40:01 +0000 Subject: [PATCH 07/38] test(emails): keep the flag loader out of the default-instance test A secret key starts the flag definition loader, whose requests could take the mock's expectations. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_default_instance_test.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/posthog/emails_default_instance_test.exs b/test/posthog/emails_default_instance_test.exs index efc8777b..8678a61d 100644 --- a/test/posthog/emails_default_instance_test.exs +++ b/test/posthog/emails_default_instance_test.exs @@ -4,7 +4,13 @@ defmodule PostHog.EmailsDefaultInstanceTest do async: Version.match?(System.version(), ">= 1.18.0"), group: PostHog - @moduletag config: [supervisor_name: PostHog, secret_key: "phs_secret_key"] + # A secret key would also start the flag definition loader, whose requests take the mock's + # expectations. + @moduletag config: [ + supervisor_name: PostHog, + secret_key: "phs_secret_key", + enable_local_evaluation: false + ] import Mox From cb165cc51f8c9da30ee1d522b6d5f80d1b3f5c51 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:16:46 +0000 Subject: [PATCH 08/38] fix(emails): retry pool checkout timeouts and stop retries after shutdown Finch raises when no pooled connection frees up in time, which skipped the retry loop as an unexpected error; it is now a retryable network error. A call waiting to retry now checks the PostHog instance first and returns client_closed, without a request, when the instance has stopped. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 38 ++++++++++++++++++++++++++++++++++-- lib/posthog/emails/error.ex | 2 +- test/posthog/emails_test.exs | 29 ++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 336668f0..95e8c1e6 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -95,6 +95,8 @@ defmodule PostHog.Emails do project. - `client_disabled`: the PostHog instance is not running or has no `:api_key`, so it sends nothing. + - `client_closed`: the PostHog instance stopped while a call waited to retry, so the retry + was not sent. An option the SDK refuses returns an error with an `invalid_*` code, such as `invalid_template_id`, and the name `:validation_error`, before any request. @@ -541,6 +543,8 @@ defmodule PostHog.Emails do {:ok, secret_key} <- secret_key(name, operation) do {:ok, %{ + name: name, + nothing_done: nothing_done, api_client: config.api_client, app_host: app_host, secret_key: secret_key, @@ -562,6 +566,20 @@ defmodule PostHog.Emails do )} end + defp check_running(%{name: name, nothing_done: nothing_done}) do + case PostHog.Registry.config(name) do + %{enabled: true} -> + :ok + + _stopped -> + {:error, + Error.new( + "client_closed", + "The PostHog instance #{inspect(name)} stopped before the retry, so #{nothing_done}." + )} + end + end + # A custom host is often an ingestion proxy, which must not receive the secret key. defp app_host(config) do case Map.get(config, :app_host) do @@ -631,7 +649,7 @@ defmodule PostHog.Emails do %{module: module, client: client} = connection.api_client case within_deadline(connection.timeout_ms, fn -> - module.request(client, method, connection.app_host <> path, options) + call_client(module, client, method, connection.app_host <> path, options) end) do {:ok, %{status: status} = response} when is_integer(status) -> {:ok, status, fields(Map.get(response, :body)), @@ -642,6 +660,16 @@ defmodule PostHog.Emails do end end + # Finch raises, rather than returns, when no pooled connection frees up in time. + defp call_client(module, client, method, url, options) do + module.request(client, method, url, options) + rescue + exception in RuntimeError -> + if String.starts_with?(exception.message, "Finch was unable to provide a connection"), + do: {:error, exception}, + else: reraise(exception, __STACKTRACE__) + end + # The client's own timeouts bound each phase, not the whole request, so the request runs in a # monitored worker that is killed at the deadline. The worker is not linked, so its failure comes # back as an error rather than killing the caller, and a caller that traps exits gets no exit @@ -731,11 +759,17 @@ defmodule PostHog.Emails do delay_ms -> connection.sleep.(delay_ms) - with_retries(connection, max_retries, attempt_once, attempt + 1) + retry(connection, max_retries, attempt_once, attempt + 1) end end end + defp retry(connection, max_retries, attempt_once, attempt) do + with :ok <- check_running(connection) do + with_retries(connection, max_retries, attempt_once, attempt) + end + end + # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" diff --git a/lib/posthog/emails/error.ex b/lib/posthog/emails/error.ex index 7ecc6002..2aaed8d1 100644 --- a/lib/posthog/emails/error.ex +++ b/lib/posthog/emails/error.ex @@ -10,7 +10,7 @@ defmodule PostHog.Emails.Error do - `:name` - the category, one of `t:name/0`. - `:code` - PostHog's error code, such as `"quota_limited"` or `"recipient_suppressed"`, or an SDK code: `"missing_secret_key"`, `"missing_app_host"`, `"client_disabled"`, - `"network_error"`, `"api_error"` (an answer without a code), `"unexpected_error"`, or an + `"client_closed"`, `"network_error"`, `"api_error"` (an answer without a code), `"unexpected_error"`, or an `"invalid_*"` code for an option the SDK refused before any request. - `:message` - human-readable message. - `:status_code` - the HTTP status, when PostHog answered. diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 46578cc2..e7eff710 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -46,7 +46,10 @@ defmodule PostHog.EmailsTest do ] |> Keyword.merge(context[:config] || []) |> PostHog.Config.validate!() - |> Map.put(:email_sleep, fn ms -> send(test_pid, {:slept, ms}) end) + |> Map.put(:email_sleep, fn ms -> + send(test_pid, {:slept, ms}) + if context[:stop_instance_on_sleep], do: stop_supervised!(context.test) + end) start_link_supervised!({PostHog.Supervisor, config}) %{name: context.test} @@ -599,6 +602,30 @@ defmodule PostHog.EmailsTest do refute_received {:slept, _} end + test "retries a connection pool checkout timeout as a network error", %{name: name} do + expect(API.Mock, :request, 2, fn _client, _method, _url, _opts -> + raise "Finch was unable to provide a connection within the timeout due to excess queuing" + end) + + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:slept, _} + assert_received {:slept, _} + end + + @tag stop_instance_on_sleep: true + test "sends no retry after the instance stops", %{name: name} do + expect_reply(refused(503, %{"code" => "service_unavailable"})) + + assert {:error, %Error{code: "client_closed"}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:slept, _} + end + @tag capture_log: true test "a client module that raises is unexpected_error", %{name: name} do expect(API.Mock, :request, fn _client, _method, _url, _opts -> raise "boom" end) From da5e56c13998f6797bdc1852fdc2175cb10a7e3f Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:19:23 +0000 Subject: [PATCH 09/38] docs(emails): say that secret_key also turns on flag polling to api_host Email requests never send the secret key to an ingestion proxy, but local flag evaluation fetches definitions from api_host with the same key. Tell users behind such a proxy to turn local evaluation off. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/config_schema.ex | 6 ++++-- lib/posthog/emails.ex | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/posthog/config_schema.ex b/lib/posthog/config_schema.ex index dcd08143..2dd02dd6 100644 --- a/lib/posthog/config_schema.ex +++ b/lib/posthog/config_schema.ex @@ -56,8 +56,10 @@ defmodule PostHog.Config.Schema do doc: """ The PostHog app host that `PostHog.Emails` sends to, such as `https://posthog.example.com` for a self-hosted instance. Without it, a PostHog Cloud `api_host` maps to its region's app - host, and any other `api_host` makes email calls fail with `missing_app_host`, because a - reverse proxy for ingestion must not receive the secret key. + host, and any other `api_host` makes email calls fail with `missing_app_host`, so email + requests never send the secret key to a reverse proxy for ingestion. Local flag + evaluation still fetches definitions from `api_host` with `secret_key`; set + `enable_local_evaluation: false` when the proxy must not see the key. """ ], email_request_timeout_ms: [ diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 95e8c1e6..e844010a 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -63,8 +63,12 @@ defmodule PostHog.Emails do A client on PostHog Cloud sends to its region's app host, `https://us.posthog.com` or `https://eu.posthog.com`. For any other `:api_host`, such as a reverse proxy, set `:app_host` - in the config, because the proxy must not receive the secret key. Without it, a send returns - the error `missing_app_host`. + in the config, because email requests must not send the secret key to the proxy. Without it, + a send returns the error `missing_app_host`. + + `:secret_key` also turns on local feature flag evaluation, which fetches flag definitions + from `:api_host` with that key. When `:api_host` is a proxy that must not see the key, set + `enable_local_evaluation: false` as well. ## Timeouts, retries and idempotency From 919a31375b0dac2f7f797a1eb3344261b45fe2fc Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:48:14 +0000 Subject: [PATCH 10/38] feat(emails): send cc, bcc and attachments A send with template_key takes cc and bcc, one address or a list, and attachments, maps of filename, content bytes, content_type and content_id. The SDK encodes the bytes as Base64 once, before the retry loop, and refuses a path or unknown key with invalid_attachment and a field that names it. Empty address and attachment lists stay out of the request, so an older server still sends the email. Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-attachments.md | 5 + lib/posthog/emails.ex | 141 ++++++++++++++++-- test/posthog/emails_test.exs | 189 +++++++++++++++++++++++- 3 files changed, 322 insertions(+), 13 deletions(-) create mode 100644 .sampo/changesets/emails-attachments.md diff --git a/.sampo/changesets/emails-attachments.md b/.sampo/changesets/emails-attachments.md new file mode 100644 index 00000000..592d1e99 --- /dev/null +++ b/.sampo/changesets/emails-attachments.md @@ -0,0 +1,5 @@ +--- +hex/posthog: minor +--- + +`PostHog.Emails.send/2` takes `cc`, `bcc` and `attachments` on a send with `template_key`. `cc` and `bcc` are one address or a list. Each attachment is a map with `filename`, `content` (the file's bytes, which the SDK encodes as Base64) and optionally `content_type` and `content_id`. An empty `reply_to`, `cc`, `bcc` or `attachments` list is now left out of the request. diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index e844010a..a93ede80 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -47,6 +47,20 @@ defmodule PostHog.Emails do idempotency_key: "password-reset:\#{reset.id}" ) + Copy finance and attach the receipt: + + PostHog.Emails.send( + to: "user@example.com", + cc: "finance@example.com", + template_key: "receipt", + subject: "Your receipt", + html: rendered_html, + attachments: [ + %{filename: "receipt.pdf", content: File.read!(pdf_path), content_type: "application/pdf"} + ], + distinct_id: "user_123" + ) + Read what happened to it: {:ok, %PostHog.Emails.Status{status: status}} = PostHog.Emails.get(message_id) @@ -139,7 +153,20 @@ defmodule PostHog.Emails do @shared_fields [:to, :distinct_id, :idempotency_key, :max_retries] @template_fields @shared_fields ++ [:template_id, :template_data] @code_fields @shared_fields ++ - [:template_key, :subject, :html, :text, :from, :reply_to, :capture_engagement] + [ + :template_key, + :subject, + :html, + :text, + :from, + :reply_to, + :cc, + :bcc, + :attachments, + :capture_engagement + ] + @address_fields [:reply_to, :cc, :bcc] + @attachment_fields [:filename, :content, :content_type, :content_id] @typedoc """ Options for `send/2`. Pass `:template_id` or `:template_key`, never both. @@ -171,11 +198,22 @@ defmodule PostHog.Emails do - `:from` - a verified sender, such as `"Acme "`. Without it, the project's default sender. - `:reply_to` - one address or a list of addresses. + - `:cc` and `:bcc` - one address or a list of at most 10 bare addresses, such as + `"finance@example.com"`, that get a visible or a hidden copy. + - `:attachments` - a list of files, at most 10 and 512 KB in total. Each is a map with + `:filename`, `:content` (the file's bytes, such as `File.read!/1` returns), and optionally + `:content_type`, such as `"application/pdf"`, and `:content_id`, which shows the file + inline: reference it in `:html` as `` with `content_id: "logo"`. The + SDK encodes the bytes as Base64. PostHog can't fetch a file from a path or URL. A file the + SDK can't send returns the error code `invalid_attachment`. - `:capture_engagement` - whether PostHog captures the email's opens and clicks as events of the recipient. Defaults to `true`. A project whose admin turned engagement capture off captures nothing, whatever the send asks. - An option set to `nil` counts as left out. + An option set to `nil` counts as left out, and so does an empty `:reply_to`, `:cc`, `:bcc` or + `:attachments` list. PostHog holds the limits. A PostHog version that doesn't take a field + yet, such as `:attachments`, refuses the send with an error whose `:field` names it, and never + drops the field to send the rest. """ @type send_option() :: {:to, String.t()} @@ -190,8 +228,21 @@ defmodule PostHog.Emails do | {:text, String.t() | nil} | {:from, String.t() | nil} | {:reply_to, String.t() | [String.t()] | nil} + | {:cc, String.t() | [String.t()] | nil} + | {:bcc, String.t() | [String.t()] | nil} + | {:attachments, [attachment()] | nil} | {:capture_engagement, boolean() | nil} + @typedoc """ + A file sent with the email. `:content` is the file's bytes; the SDK encodes them as Base64. + """ + @type attachment() :: %{ + required(:filename) => String.t(), + required(:content) => binary(), + optional(:content_type) => String.t() | nil, + optional(:content_id) => String.t() | nil + } + @doc """ Queues one email, from a template made in PostHog or from content your code rendered. @@ -345,12 +396,14 @@ defmodule PostHog.Emails do defp code_content(%{template_key: template_key} = options) do with :ok <- check_template_key(template_key), :ok <- refuse_fields_outside(options, :code), - :ok <- check_code_fields(options) do + :ok <- check_code_fields(options), + {:ok, attachments} <- attachments(Map.get(options, :attachments, [])) do {:ok, options |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) - |> put_reply_to(options)} + |> put_addresses(options) + |> put_unless_empty(:attachments, attachments)} end end @@ -393,11 +446,12 @@ defmodule PostHog.Emails do not_string?(options, :from) -> invalid("invalid_form", "from must be a string.", "from") - Map.has_key?(options, :reply_to) and not addresses?(options.reply_to) -> + field = + Enum.find(@address_fields, &(Map.has_key?(options, &1) and not addresses?(options[&1]))) -> invalid( "invalid_form", - "reply_to must be one address or a list of addresses.", - "reply_to" + "#{field} must be one address or a list of addresses.", + to_string(field) ) not is_boolean(Map.get(options, :capture_engagement, true)) -> @@ -411,16 +465,81 @@ defmodule PostHog.Emails do defp not_string?(options, field), do: Map.has_key?(options, field) and not is_binary(Map.fetch!(options, field)) - defp put_reply_to(content, %{reply_to: reply_to}) when is_binary(reply_to), - do: Map.put(content, :reply_to, [reply_to]) + defp put_addresses(content, options) do + Enum.reduce(@address_fields, content, fn field, content -> + put_unless_empty(content, field, List.wrap(Map.get(options, field))) + end) + end - defp put_reply_to(content, %{reply_to: reply_to}), do: Map.put(content, :reply_to, reply_to) - defp put_reply_to(content, _options), do: content + # A server that doesn't take a field yet refuses it even when empty, so an empty list stays out. + defp put_unless_empty(content, _field, []), do: content + defp put_unless_empty(content, field, list), do: Map.put(content, field, list) defp addresses?(address) when is_binary(address), do: nonblank?(address) defp addresses?(addresses) when is_list(addresses), do: Enum.all?(addresses, &nonblank?/1) defp addresses?(_addresses), do: false + defp attachments(attachments) when is_list(attachments) do + attachments + |> Enum.with_index() + |> Enum.reduce_while({:ok, []}, fn {attachment, index}, {:ok, encoded} -> + case attachment(attachment, "attachments__#{index}") do + {:ok, attachment} -> {:cont, {:ok, [attachment | encoded]}} + error -> {:halt, error} + end + end) + |> case do + {:ok, encoded} -> {:ok, Enum.reverse(encoded)} + error -> error + end + end + + defp attachments(_attachments), + do: invalid_attachment("attachments must be a list of files.", "attachments") + + defp attachment(attachment, field) when is_map(attachment) and not is_struct(attachment) do + attachment = attachment |> Enum.reject(fn {_key, value} -> is_nil(value) end) |> Map.new() + unknown = attachment |> Map.keys() |> Enum.reject(&(&1 in @attachment_fields)) |> Enum.sort() + + cond do + Map.has_key?(attachment, :path) -> + invalid_attachment( + "PostHog can't fetch an attachment from a path or URL. Read the file, for example " <> + "with File.read!/1, and pass its bytes as content.", + field <> "__path" + ) + + unknown != [] -> + key = to_string(hd(unknown)) + + invalid_attachment( + "An attachment takes filename, content, content_type and content_id, not #{key}.", + field <> "__" <> key + ) + + not is_binary(attachment[:filename]) -> + invalid_attachment("An attachment needs its filename as a string.", field <> "__filename") + + not is_binary(attachment[:content]) -> + invalid_attachment( + "An attachment needs content: the file's bytes as a binary.", + field <> "__content" + ) + + key = Enum.find([:content_type, :content_id], ¬_string?(attachment, &1)) -> + invalid_attachment("An attachment's #{key} must be a string.", "#{field}__#{key}") + + true -> + {:ok, Map.update!(attachment, :content, &Base.encode64/1)} + end + end + + defp attachment(_attachment, field), + do: invalid_attachment("An attachment is a map with filename and content.", field) + + # The message never holds the file's content, so the error is safe to log. + defp invalid_attachment(message, field), do: invalid("invalid_attachment", message, field) + defp refuse_fields_outside(options, form) do {fields, other_fields} = case form do diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index e7eff710..d26c461a 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -281,6 +281,186 @@ defmodule PostHog.EmailsTest do end end + describe "send/2 with cc, bcc and attachments" do + @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] + @pdf <<37, 80, 68, 70, 45, 0, 255, 10>> + + test "posts copies and Base64 attachments", %{name: name} do + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send( + name, + @code_send ++ + [ + cc: "finance@example.com", + bcc: ["audit@example.com", "archive@example.com"], + attachments: [ + %{filename: "invoice.pdf", content: @pdf, content_type: "application/pdf"}, + %{filename: "logo.png", content: "png bytes", content_id: "logo"} + ] + ] + ) + + body = sent_body() + assert body["cc"] == ["finance@example.com"] + assert body["bcc"] == ["audit@example.com", "archive@example.com"] + + assert body["attachments"] == [ + %{ + "filename" => "invoice.pdf", + "content" => Base.encode64(@pdf), + "content_type" => "application/pdf" + }, + %{ + "filename" => "logo.png", + "content" => Base.encode64("png bytes"), + "content_id" => "logo" + } + ] + end + + test "leaves out empty lists and nil attachment fields", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send( + name, + @code_send ++ + [ + reply_to: [], + cc: [], + bcc: nil, + attachments: [], + distinct_id: "user_123" + ] + ) + + body = sent_body() + refute Map.has_key?(body, "reply_to") + refute Map.has_key?(body, "cc") + refute Map.has_key?(body, "bcc") + refute Map.has_key?(body, "attachments") + + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send( + name, + @code_send ++ + [attachments: [%{filename: "a.txt", content: "a", content_type: nil}]] + ) + + assert sent_body()["attachments"] == [%{"filename" => "a.txt", "content" => "YQ=="}] + end + + test "a template send refuses them before any request", %{name: name} do + for {field, value} <- [ + cc: "finance@example.com", + bcc: "audit@example.com", + attachments: [%{filename: "a.txt", content: "a"}] + ] do + assert {:error, %Error{code: "invalid_form", field: field_name} = error} = + Emails.send(name, [ + {field, value}, + to: "user@example.com", + template_id: @template_id + ]) + + assert field_name == to_string(field) + assert error.fix =~ "template_key" + end + + refute_requested() + end + + test "refuses cc and bcc that are not addresses", %{name: name} do + for {field, value} <- [cc: [:nope], bcc: " ", cc: %{a: 1}] do + assert {:error, %Error{code: "invalid_form", field: field_name}} = + Emails.send(name, @code_send ++ [{field, value}]) + + assert field_name == to_string(field) + end + + refute_requested() + end + + test "refuses an attachment it can't send, naming the field", %{name: name} do + for {attachments, field} <- [ + {%{filename: "a.txt", content: "a"}, "attachments"}, + {"invoice.pdf", "attachments"}, + {[[filename: "a.txt", content: "a"]], "attachments__0"}, + {[%{filename: "a.txt", content: "a"}, "b.txt"], "attachments__1"}, + {[%{filename: "a.txt", path: "/tmp/a.txt"}], "attachments__0__path"}, + {[%{filename: "a.txt", content: "a", cid: "logo"}], "attachments__0__cid"}, + {[%{"filename" => "a.txt", "content" => "a"}], "attachments__0__content"}, + {[%{filename: "a.txt"}], "attachments__0__content"}, + {[%{filename: "a.txt", content: [97]}], "attachments__0__content"}, + {[%{filename: "a.txt", content: <<1::3>>}], "attachments__0__content"}, + {[%{content: "a"}], "attachments__0__filename"}, + {[%{filename: :a, content: "a"}], "attachments__0__filename"}, + {[%{filename: "a.txt", content: "a", content_type: :pdf}], + "attachments__0__content_type"}, + {[%{filename: "a.txt", content: "a", content_id: 1}], "attachments__0__content_id"} + ] do + assert {:error, + %Error{code: "invalid_attachment", name: :validation_error, field: ^field}} = + Emails.send(name, @code_send ++ [attachments: attachments]) + end + + refute_requested() + end + + test "a path points to reading the file", %{name: name} do + assert {:error, %Error{message: message}} = + Emails.send( + name, + @code_send ++ [attachments: [%{filename: "a.pdf", path: "/srv/a.pdf"}]] + ) + + assert message =~ "File.read" + end + + test "never repeats the file's content in an error", %{name: name} do + assert {:error, %Error{} = error} = + Emails.send( + name, + @code_send ++ + [attachments: [%{filename: 1, content: "SECRET-FILE-BYTES"}]] + ) + + refute inspect(error) =~ "SECRET-FILE-BYTES" + end + + test "returns a server's refusal of a field it doesn't take", %{name: name} do + expect_reply( + refused(400, %{ + "type" => "validation_error", + "code" => "invalid_request", + "detail" => "PostHog doesn't take this field yet, so it didn't send the email.", + "attr" => "attachments", + "why" => "This PostHog version doesn't take the field yet.", + "fix" => "Send the email without this field." + }) + ) + + assert {:error, + %Error{ + code: "invalid_request", + field: "attachments", + name: :validation_error, + status_code: 400 + }} = + Emails.send( + name, + @code_send ++ [attachments: [%{filename: "a.txt", content: "a"}]] + ) + + assert_received {:request, :post, @send_url, _opts} + refute_received {:request, _, _, _} + end + end + describe "send/2 argument checks" do test "needs exactly one form", %{name: name} do assert {:error, %Error{code: "invalid_form"}} = Emails.send(name, to: "user@example.com") @@ -296,11 +476,16 @@ defmodule PostHog.EmailsTest do end test "refuses an option neither form takes", %{name: name} do - assert {:error, %Error{code: "invalid_form", field: "cc", fix: "Remove cc and send again."}} = + assert {:error, + %Error{ + code: "invalid_form", + field: "headers", + fix: "Remove headers and send again." + }} = Emails.send(name, to: "user@example.com", template_id: @template_id, - cc: "x@example.com" + headers: %{"X-Campaign" => "welcome"} ) refute_requested() From 0b2555402f162cba1edc1586eed49a58e5f284e4 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:56:39 +0000 Subject: [PATCH 11/38] fix(emails): return a client exit as an error, not a crash A client that exits inside the request worker, as Finch does when its pool restarts, came back to the caller as an exit and crashed it. It is now a retryable network_error, and a throw is unexpected_error. Also reuse UUIDv7's encode and decode, read the email timeout and retries straight from the validated config, and name the config helper's argument for what it takes. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/config.ex | 8 ++++---- lib/posthog/emails.ex | 24 ++++++++++-------------- test/posthog/emails_test.exs | 26 ++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/lib/posthog/config.ex b/lib/posthog/config.ex index 22f99710..bd651e84 100644 --- a/lib/posthog/config.ex +++ b/lib/posthog/config.ex @@ -205,14 +205,14 @@ defmodule PostHog.Config do defp normalize_api_key(nil), do: "" defp normalize_api_key(api_key), do: api_key - defp normalize_optional_string(secret_key) when is_binary(secret_key) do - case String.trim(secret_key) do + defp normalize_optional_string(value) when is_binary(value) do + case String.trim(value) do "" -> nil - value -> value + trimmed -> trimmed end end - defp normalize_optional_string(secret_key), do: secret_key + defp normalize_optional_string(value), do: value defp normalize_api_host(api_host) when is_binary(api_host) do api_host diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index e844010a..1cb0a9ca 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -132,8 +132,6 @@ defmodule PostHog.Emails do "https://eu.posthog.com" => @eu_app_host } @max_retry_delay_ms 30_000 - @default_timeout_ms 15_000 - @default_max_retries 2 @log_metadata [posthog_skip_capture: true] @shared_fields [:to, :distinct_id, :idempotency_key, :max_retries] @@ -514,10 +512,7 @@ defmodule PostHog.Emails do end end - defp uuid?(value), - do: - is_binary(value) and - value =~ ~r/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i + defp uuid?(value), do: UUIDv7.decode(value) != :error defp nonempty?(value), do: is_binary(value) and value != "" defp nonblank?(value), do: is_binary(value) and String.trim(value) != "" @@ -527,11 +522,7 @@ defmodule PostHog.Emails do defp uuid4 do <> = :crypto.strong_rand_bytes(16) - <> - |> Base.encode16(case: :lower) - |> then(fn <> -> - Enum.join([p1, p2, p3, p4, p5], "-") - end) + UUIDv7.encode(<>) end # Connection @@ -553,8 +544,8 @@ defmodule PostHog.Emails do app_host: app_host, secret_key: secret_key, migrated_from: Map.get(config, :migrated_from), - timeout_ms: Map.get(config, :email_request_timeout_ms, @default_timeout_ms), - max_retries: Map.get(config, :email_request_max_retries, @default_max_retries), + timeout_ms: config.email_request_timeout_ms, + max_retries: config.email_request_max_retries, sleep: Map.get(config, :email_sleep, &Process.sleep/1) }} end @@ -738,7 +729,12 @@ defmodule PostHog.Emails do defp unwrap({:returned, result}), do: result defp unwrap({:raised, exception, stacktrace}), do: reraise(exception, stacktrace) - defp unwrap({:caught, kind, reason, stacktrace}), do: :erlang.raise(kind, reason, stacktrace) + # Finch exits when its pool goes away mid-request, such as during a restart, so an exit is + # a network failure. + defp unwrap({:caught, :exit, reason, _stacktrace}), do: {:error, {:exit, reason}} + + defp unwrap({:caught, :throw, value, stacktrace}), + do: :erlang.raise(:error, {:nocatch, value}, stacktrace) defp fields(body) when is_map(body), do: body diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index e7eff710..3adebdc5 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -707,6 +707,32 @@ defmodule PostHog.EmailsTest do assert {{:error, %Error{code: "network_error"}}, {:messages, []}} = Task.await(task) end + + test "retries a client exit, such as a restarting pool, as a network error", %{name: name} do + expect(API.Mock, :request, fn _client, _method, _url, _opts -> + exit({:noproc, {GenServer, :call, [:pool, :checkout, 5_000]}}) + end) + + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + + assert_received {:slept, _ms} + end + + test "returns a client exit on a status read as network_error", %{name: name} do + expect(API.Mock, :request, 3, fn _client, :get, _url, _opts -> exit(:normal) end) + + assert {:error, %Error{code: "network_error"}} = Emails.get(name, @message_id) + end + + test "returns a client throw as unexpected_error without crashing the caller", %{name: name} do + expect(API.Mock, :request, fn _client, _method, _url, _opts -> throw(:boom) end) + + assert {:error, %Error{code: "unexpected_error"}} = + Emails.send(name, to: "user@example.com", template_id: @template_id) + end end describe "send/2 replies" do From 085238041543d9bf199635c83fc4dc7ca196e595 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:57:39 +0000 Subject: [PATCH 12/38] docs(emails): say that attachment content is raw bytes, not Base64 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 21f95bd2..ca259ba3 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -202,7 +202,7 @@ defmodule PostHog.Emails do `:filename`, `:content` (the file's bytes, such as `File.read!/1` returns), and optionally `:content_type`, such as `"application/pdf"`, and `:content_id`, which shows the file inline: reference it in `:html` as `` with `content_id: "logo"`. The - SDK encodes the bytes as Base64. PostHog can't fetch a file from a path or URL. A file the + SDK encodes the bytes as Base64. Pass raw bytes, not Base64, or the content is encoded twice. PostHog can't fetch a file from a path or URL. A file the SDK can't send returns the error code `invalid_attachment`. - `:capture_engagement` - whether PostHog captures the email's opens and clicks as events of the recipient. Defaults to `true`. A project whose admin turned engagement capture off @@ -232,7 +232,8 @@ defmodule PostHog.Emails do | {:capture_engagement, boolean() | nil} @typedoc """ - A file sent with the email. `:content` is the file's bytes; the SDK encodes them as Base64. + A file sent with the email. `:content` is the file's raw bytes, which the SDK encodes as + Base64. Don't pass content that is already Base64: it would be encoded twice. """ @type attachment() :: %{ required(:filename) => String.t(), From 74dd12ee5aadfc00957a795896a21baa6b9d010d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:00:18 +0000 Subject: [PATCH 13/38] refactor(emails): name a client exit instead of commenting it Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 1cb0a9ca..e02be297 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -729,9 +729,7 @@ defmodule PostHog.Emails do defp unwrap({:returned, result}), do: result defp unwrap({:raised, exception, stacktrace}), do: reraise(exception, stacktrace) - # Finch exits when its pool goes away mid-request, such as during a restart, so an exit is - # a network failure. - defp unwrap({:caught, :exit, reason, _stacktrace}), do: {:error, {:exit, reason}} + defp unwrap({:caught, :exit, reason, _stacktrace}), do: {:error, {:client_exit, reason}} defp unwrap({:caught, :throw, value, stacktrace}), do: :erlang.raise(:error, {:nocatch, value}, stacktrace) From 99a76e4fa6fe6cdda740e042cdf4a653dbeca19a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:11:07 +0000 Subject: [PATCH 14/38] fix(emails): name string and odd attachment keys, and why a template refuses copies An attachment with string keys now asks for atom keys, and a key with no string form is refused as invalid_attachment rather than raising. A template send refusing cc, bcc or attachments says the template sets its recipients and content. The two new comments become names: put_if_any/3. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 47 ++++++++++++++++++++++++------------ test/posthog/emails_test.exs | 23 ++++++++++++++++++ 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index b564eaef..ef93b926 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -165,6 +165,8 @@ defmodule PostHog.Emails do ] @address_fields [:reply_to, :cc, :bcc] @attachment_fields [:filename, :content, :content_type, :content_id] + @attachment_field_names Enum.map(@attachment_fields, &Atom.to_string/1) + @template_owned_fields [:cc, :bcc, :attachments] @typedoc """ Options for `send/2`. Pass `:template_id` or `:template_key`, never both. @@ -402,7 +404,7 @@ defmodule PostHog.Emails do |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) |> put_addresses(options) - |> put_unless_empty(:attachments, attachments)} + |> put_if_any(:attachments, attachments)} end end @@ -466,13 +468,12 @@ defmodule PostHog.Emails do defp put_addresses(content, options) do Enum.reduce(@address_fields, content, fn field, content -> - put_unless_empty(content, field, List.wrap(Map.get(options, field))) + put_if_any(content, field, List.wrap(Map.get(options, field))) end) end - # A server that doesn't take a field yet refuses it even when empty, so an empty list stays out. - defp put_unless_empty(content, _field, []), do: content - defp put_unless_empty(content, field, list), do: Map.put(content, field, list) + defp put_if_any(content, _field, []), do: content + defp put_if_any(content, field, list), do: Map.put(content, field, list) defp addresses?(address) when is_binary(address), do: nonblank?(address) defp addresses?(addresses) when is_list(addresses), do: Enum.all?(addresses, &nonblank?/1) @@ -497,7 +498,7 @@ defmodule PostHog.Emails do do: invalid_attachment("attachments must be a list of files.", "attachments") defp attachment(attachment, field) when is_map(attachment) and not is_struct(attachment) do - attachment = attachment |> Enum.reject(fn {_key, value} -> is_nil(value) end) |> Map.new() + attachment = Map.reject(attachment, fn {_key, value} -> is_nil(value) end) unknown = attachment |> Map.keys() |> Enum.reject(&(&1 in @attachment_fields)) |> Enum.sort() cond do @@ -509,12 +510,7 @@ defmodule PostHog.Emails do ) unknown != [] -> - key = to_string(hd(unknown)) - - invalid_attachment( - "An attachment takes filename, content, content_type and content_id, not #{key}.", - field <> "__" <> key - ) + unknown_attachment_key(hd(unknown), field) not is_binary(attachment[:filename]) -> invalid_attachment("An attachment needs its filename as a string.", field <> "__filename") @@ -536,7 +532,22 @@ defmodule PostHog.Emails do defp attachment(_attachment, field), do: invalid_attachment("An attachment is a map with filename and content.", field) - # The message never holds the file's content, so the error is safe to log. + defp unknown_attachment_key(key, field) when key in @attachment_field_names, + do: + invalid_attachment( + ~s(Attachment keys are atoms: use :#{key}, not "#{key}".), + "#{field}__#{key}" + ) + + defp unknown_attachment_key(key, field) when is_atom(key) or is_binary(key), + do: invalid_attachment(attachment_keys_message(key), "#{field}__#{key}") + + defp unknown_attachment_key(key, field), + do: invalid_attachment(attachment_keys_message(inspect(key)), field) + + defp attachment_keys_message(key), + do: "An attachment takes filename, content, content_type and content_id, not #{key}." + defp invalid_attachment(message, field), do: invalid("invalid_attachment", message, field) defp refuse_fields_outside(options, form) do @@ -555,6 +566,13 @@ defmodule PostHog.Emails do end end + defp template_owns(field) when field in @template_owned_fields, + do: "A send with template_id takes its recipients and content from the template." + + defp template_owns(_field), + do: + "A template made in PostHog sets its own sender, subject and content, and follows the project's engagement setting." + defp field_outside_error(form, field, other_form_takes_it?) do form_field = if form == :template, do: "template_id", else: "template_key" message = "A send with #{form_field} doesn't take #{field}, so no email was sent." @@ -563,8 +581,7 @@ defmodule PostHog.Emails do {why, how} = case form do :template -> - {"A template made in PostHog sets its own sender, subject and content, and follows the project's engagement setting.", - "send content from your code with template_key"} + {template_owns(field), "send content from your code with template_key"} :code -> {"A send with template_key takes its content from your code.", diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index fac5a64d..eb44ec35 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -368,12 +368,35 @@ defmodule PostHog.EmailsTest do ]) assert field_name == to_string(field) + assert error.why =~ "recipients and content from the template" assert error.fix =~ "template_key" end refute_requested() end + test "asks for atom keys when an attachment has string keys", %{name: name} do + assert {:error, + %Error{code: "invalid_attachment", field: "attachments__0__content"} = error} = + Emails.send( + name, + @code_send ++ [attachments: [%{"filename" => "a.txt", "content" => "a"}]] + ) + + assert error.message =~ ~s(:content, not "content") + refute_requested() + end + + test "refuses an attachment key that has no string form", %{name: name} do + attachment = %{{:extra, :field} => true, filename: "a.txt", content: "a"} + + assert {:error, %Error{code: "invalid_attachment", field: "attachments__0"} = error} = + Emails.send(name, @code_send ++ [attachments: [attachment]]) + + assert error.message =~ "{:extra, :field}" + refute_requested() + end + test "refuses cc and bcc that are not addresses", %{name: name} do for {field, value} <- [cc: [:nope], bcc: " ", cc: %{a: 1}] do assert {:error, %Error{code: "invalid_form", field: field_name}} = From 3d041643599e6eed40b0c94c1aef7507704768ab Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:13:28 +0000 Subject: [PATCH 15/38] fix(emails): ask for atom keys when a string key names an attachment field A string-keyed attachment with an extra key that sorts first, such as "cid", reported that key instead of the string keys, so a user fixed one thing and then hit the real cause. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 5 ++++- test/posthog/emails_test.exs | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index ef93b926..d6afc876 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -510,7 +510,7 @@ defmodule PostHog.Emails do ) unknown != [] -> - unknown_attachment_key(hd(unknown), field) + unknown_attachment_key(key_to_report(unknown), field) not is_binary(attachment[:filename]) -> invalid_attachment("An attachment needs its filename as a string.", field <> "__filename") @@ -532,6 +532,9 @@ defmodule PostHog.Emails do defp attachment(_attachment, field), do: invalid_attachment("An attachment is a map with filename and content.", field) + defp key_to_report(unknown_keys), + do: Enum.find(unknown_keys, hd(unknown_keys), &(&1 in @attachment_field_names)) + defp unknown_attachment_key(key, field) when key in @attachment_field_names, do: invalid_attachment( diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index eb44ec35..6a367b96 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -384,6 +384,14 @@ defmodule PostHog.EmailsTest do ) assert error.message =~ ~s(:content, not "content") + + assert {:error, %Error{field: "attachments__0__content"}} = + Emails.send( + name, + @code_send ++ + [attachments: [%{"filename" => "a.txt", "content" => "a", "cid" => "logo"}]] + ) + refute_requested() end From 3de2ed2c6e4859a175f6ed6e66fead801792eccb Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:12:55 +0000 Subject: [PATCH 16/38] feat(emails): upload attachments by hash, then send references A send with attachments now hashes each file, asks PostHog which files it doesn't have yet, PUTs those straight to PostHog's file storage with the returned headers and without the secret key, and then sends the email with references to the hashes. A failed upload sends nothing and returns the storage's answer. The SDK checks 10 files and 40 MB per email after Base64 encoding, about 29 MB of files, before any request, with PostHog's own estimate. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-attachments.md | 2 +- lib/posthog/emails.ex | 268 +++++++------ lib/posthog/emails/attachments.ex | 293 +++++++++++++++ lib/posthog/emails/error.ex | 11 +- test/posthog/emails_test.exs | 477 +++++++++++++++++++----- 5 files changed, 850 insertions(+), 201 deletions(-) create mode 100644 lib/posthog/emails/attachments.ex diff --git a/.sampo/changesets/emails-attachments.md b/.sampo/changesets/emails-attachments.md index 592d1e99..c8b8888f 100644 --- a/.sampo/changesets/emails-attachments.md +++ b/.sampo/changesets/emails-attachments.md @@ -2,4 +2,4 @@ hex/posthog: minor --- -`PostHog.Emails.send/2` takes `cc`, `bcc` and `attachments` on a send with `template_key`. `cc` and `bcc` are one address or a list. Each attachment is a map with `filename`, `content` (the file's bytes, which the SDK encodes as Base64) and optionally `content_type` and `content_id`. An empty `reply_to`, `cc`, `bcc` or `attachments` list is now left out of the request. +`PostHog.Emails.send/2` takes `cc`, `bcc` and `attachments` on a send with `template_key`. `cc` and `bcc` are one address or a list. Each attachment is a map with `filename`, `content` (the file's raw bytes) and optionally `content_type` and `content_id`. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The SDK hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, without the secret key, and then sends the email with references to them. If an upload fails, it sends nothing and returns `attachment_upload_failed`. An empty `reply_to`, `cc`, `bcc` or `attachments` list is now left out of the request. diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 98f000a6..dabb7917 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -85,13 +85,31 @@ defmodule PostHog.Emails do from `:api_host` with that key. When `:api_host` is a proxy that must not see the key, set `enable_local_evaluation: false` as well. + ## Attachments + + An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The + SDK checks both before any request, counting each file and body as Base64, as PostHog does. + + A send with files takes three steps: + + 1. The SDK hashes each file with SHA-256 and asks PostHog which files it doesn't have yet. + 2. It uploads each of those straight to PostHog's file storage, with the headers PostHog + gave and without your secret key. A file PostHog already has, such as the same terms PDF + sent to every customer, uploads nothing again. + 3. Once every upload succeeded, it sends the email, which names each file by its hash. + + Your app must be able to reach the storage host that PostHog's answer names. If an upload + fails, the SDK sends nothing and returns `attachment_upload_failed` with what the storage + answered. An upload gets the request timeout plus one second per MB of the file. + ## Timeouts, retries and idempotency Each request times out after `:email_request_timeout_ms` (15 seconds by default). A send retries network errors, rate limits, an idempotency key still in flight and an unavailable service `:email_request_max_retries` times (2 by default), waiting about 1 and then 2 seconds, or the server's `Retry-After` when it is longer. A `Retry-After` over 30 seconds fails at once - with `:retry_after_ms` set. A monthly quota is never retried. + with `:retry_after_ms` set. A monthly quota and the day's upload quota are never retried. + The upload steps of a send with files retry the same way. Every attempt sends the same idempotency key, so a retry never sends twice. Without `:idempotency_key`, the SDK makes a UUID for the call. When a job runner such as Oban @@ -117,6 +135,17 @@ defmodule PostHog.Emails do - `client_closed`: the PostHog instance stopped while a call waited to retry, so the retry was not sent. + A send with files can also return: + + - `too_many_attachments` or `message_too_large`: over 10 files, or over 40 MB after Base64 + encoding. The SDK checks both before any request, and PostHog checks them again. + - `attachment_upload_failed`: an upload to PostHog's file storage failed, so no email was + sent. The message has the storage's answer. + - `attachment_not_uploaded`: PostHog has no uploaded file with a hash the send names. + - `upload_quota_exceeded` (429): the project uploaded as many file bytes as it can today. + Files PostHog already has don't count. + - `field_not_accepted`: the project can't send files yet. + An option the SDK refuses returns an error with an `invalid_*` code, such as `invalid_template_id`, and the name `:validation_error`, before any request. @@ -130,12 +159,14 @@ defmodule PostHog.Emails do require Logger + alias PostHog.Emails.Attachments alias PostHog.Emails.Error alias PostHog.Emails.SendResult alias PostHog.Emails.Status alias PostHog.Emails.Warning @send_path "/api/projects/@current/messaging/send/email/" + @uploads_path "/api/projects/@current/messaging/attachments/uploads/" @status_path "/api/projects/@current/messaging/emails/" @us_app_host "https://us.posthog.com" @eu_app_host "https://eu.posthog.com" @@ -165,8 +196,6 @@ defmodule PostHog.Emails do :capture_engagement ] @address_fields [:reply_to, :cc, :bcc] - @attachment_fields [:filename, :content, :content_type, :content_id] - @attachment_field_names Enum.map(@attachment_fields, &Atom.to_string/1) @template_owned_fields [:cc, :bcc, :attachments] @typedoc """ @@ -201,20 +230,21 @@ defmodule PostHog.Emails do - `:reply_to` - one address or a list of addresses. - `:cc` and `:bcc` - one address or a list of at most 10 bare addresses, such as `"finance@example.com"`, that get a visible or a hidden copy. - - `:attachments` - a list of files, at most 10 and 512 KB in total. Each is a map with - `:filename`, `:content` (the file's bytes, such as `File.read!/1` returns), and optionally - `:content_type`, such as `"application/pdf"`, and `:content_id`, which shows the file - inline: reference it in `:html` as `` with `content_id: "logo"`. The - SDK encodes the bytes as Base64. Pass raw bytes, not Base64, or the content is encoded twice. PostHog can't fetch a file from a path or URL. A file the - SDK can't send returns the error code `invalid_attachment`. + - `:attachments` - a list of files: at most 10 files and 40 MB per email after Base64 + encoding, about 29 MB of files. Each is a map with `:filename`, `:content` (the file's + bytes, such as `File.read!/1` returns), and optionally `:content_type`, such as + `"application/pdf"`, and `:content_id`, which shows the file inline: reference it in + `:html` as `` with `content_id: "logo"`. Pass raw bytes, not Base64. + PostHog can't fetch a file from a path or URL. A file the SDK can't send returns the error + code `invalid_attachment`. See "Attachments" in `PostHog.Emails` for how files travel. - `:capture_engagement` - whether PostHog captures the email's opens and clicks as events of the recipient. Defaults to `true`. A project whose admin turned engagement capture off captures nothing, whatever the send asks. An option set to `nil` counts as left out, and so does an empty `:reply_to`, `:cc`, `:bcc` or - `:attachments` list. PostHog holds the limits. A PostHog version that doesn't take a field - yet, such as `:attachments`, refuses the send with an error whose `:field` names it, and never - drops the field to send the rest. + `:attachments` list. A PostHog version that doesn't take a field yet, such as `:attachments`, + refuses the send with an error whose `:field` names it, and never drops the field to send the + rest. """ @type send_option() :: {:to, String.t()} @@ -235,8 +265,8 @@ defmodule PostHog.Emails do | {:capture_engagement, boolean() | nil} @typedoc """ - A file sent with the email. `:content` is the file's raw bytes, which the SDK encodes as - Base64. Don't pass content that is already Base64: it would be encoded twice. + A file sent with the email. `:content` is the file's raw bytes, such as `File.read!/1` + returns. Don't pass content that is already Base64: the recipient would get the Base64 text. """ @type attachment() :: %{ required(:filename) => String.t(), @@ -273,6 +303,7 @@ defmodule PostHog.Emails do def send(name \\ PostHog, options) do with {:ok, prepared} <- prepare_send(options), {:ok, connection} <- connect(name, :send), + :ok <- upload(connection, prepared), {:ok, result} <- deliver(connection, prepared) do log_warnings(result) {:ok, result} @@ -324,6 +355,69 @@ defmodule PostHog.Emails do end) end + # Files are uploaded straight to PostHog's file storage, which never gets the secret key. + defp upload(_connection, %{files: []}), do: :ok + + defp upload(connection, prepared) do + max_retries = prepared.max_retries || connection.max_retries + + with {:ok, uploads} <- ask_where_to_upload(connection, prepared.files, max_retries) do + put_files(connection, uploads, max_retries) + end + end + + defp put_files(_connection, [], _max_retries), do: :ok + + defp put_files(connection, [{file, upload} | uploads], max_retries) do + with :ok <- put_file(connection, file, upload, max_retries) do + put_files(connection, uploads, max_retries) + end + end + + defp ask_where_to_upload(connection, files, max_retries) do + headers = [{"content-type", "application/json"} | headers(connection)] + body = Jason.encode!(Attachments.upload_request(files)) + + with_retries(connection, max_retries, fn -> + connection + |> request(:post, @uploads_path, headers, body) + |> parse_uploads_reply(files) + end) + end + + defp put_file(connection, file, upload, max_retries) do + # A large file takes longer to send than a request without one. + timeout_ms = connection.timeout_ms + div(file.size, 1_000_000) * 1_000 + + connection + |> with_retries(max_retries, fn -> + case send_request(connection, :put, upload.url, upload.headers, file.content, timeout_ms) do + {:ok, status, _body, _retry_after_ms} when status in 200..299 -> + {:ok, :uploaded} + + {:ok, status, body, retry_after_ms} -> + {:error, + %{ + Attachments.upload_failed(file, upload.url, status, body) + | retry_after_ms: retry_after_ms + }} + + {:error, error} -> + {:error, error} + end + end) + |> case do + {:ok, :uploaded} -> + :ok + + {:error, %Error{code: "network_error"}} -> + {:error, Attachments.upload_failed(file, upload.url, nil, nil)} + + {:error, error} -> + {:error, error} + end + end + # Options defp prepare_send(options) when is_list(options) or is_map(options) do @@ -341,11 +435,12 @@ defmodule PostHog.Emails do defp prepare(options) do with :ok <- check_recipient(options), - {:ok, content} <- content(options), + {:ok, {content, files}} <- content(options), {:ok, idempotency_key} <- idempotency_key(options), {:ok, max_retries} <- max_retries(options), {:ok, body} <- encode(content, options) do - {:ok, %{body: body, idempotency_key: idempotency_key, max_retries: max_retries}} + {:ok, + %{body: body, files: files, idempotency_key: idempotency_key, max_retries: max_retries}} end end @@ -388,9 +483,12 @@ defmodule PostHog.Emails do true -> with :ok <- refuse_fields_outside(options, :template) do {:ok, - options - |> Map.take([:to, :distinct_id]) - |> Map.merge(%{template_id: String.downcase(template_id), message_data: template_data})} + {options + |> Map.take([:to, :distinct_id]) + |> Map.merge(%{ + template_id: String.downcase(template_id), + message_data: template_data + }), []}} end end end @@ -399,13 +497,17 @@ defmodule PostHog.Emails do with :ok <- check_template_key(template_key), :ok <- refuse_fields_outside(options, :code), :ok <- check_code_fields(options), - {:ok, attachments} <- attachments(Map.get(options, :attachments, [])) do + {:ok, files} <- + Attachments.prepare( + Map.get(options, :attachments, []), + Enum.flat_map([:html, :text], &List.wrap(options[&1])) + ) do {:ok, - options - |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) - |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) - |> put_addresses(options) - |> put_if_any(:attachments, attachments)} + {options + |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) + |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) + |> put_addresses(options) + |> put_if_any(:attachments, Enum.map(files, & &1.reference)), files}} end end @@ -480,80 +582,6 @@ defmodule PostHog.Emails do defp addresses?(addresses) when is_list(addresses), do: Enum.all?(addresses, &nonblank?/1) defp addresses?(_addresses), do: false - defp attachments(attachments) when is_list(attachments) do - attachments - |> Enum.with_index() - |> Enum.reduce_while({:ok, []}, fn {attachment, index}, {:ok, encoded} -> - case attachment(attachment, "attachments__#{index}") do - {:ok, attachment} -> {:cont, {:ok, [attachment | encoded]}} - error -> {:halt, error} - end - end) - |> case do - {:ok, encoded} -> {:ok, Enum.reverse(encoded)} - error -> error - end - end - - defp attachments(_attachments), - do: invalid_attachment("attachments must be a list of files.", "attachments") - - defp attachment(attachment, field) when is_map(attachment) and not is_struct(attachment) do - attachment = Map.reject(attachment, fn {_key, value} -> is_nil(value) end) - unknown = attachment |> Map.keys() |> Enum.reject(&(&1 in @attachment_fields)) |> Enum.sort() - - cond do - Map.has_key?(attachment, :path) -> - invalid_attachment( - "PostHog can't fetch an attachment from a path or URL. Read the file, for example " <> - "with File.read!/1, and pass its bytes as content.", - field <> "__path" - ) - - unknown != [] -> - unknown_attachment_key(key_to_report(unknown), field) - - not is_binary(attachment[:filename]) -> - invalid_attachment("An attachment needs its filename as a string.", field <> "__filename") - - not is_binary(attachment[:content]) -> - invalid_attachment( - "An attachment needs content: the file's bytes as a binary.", - field <> "__content" - ) - - key = Enum.find([:content_type, :content_id], ¬_string?(attachment, &1)) -> - invalid_attachment("An attachment's #{key} must be a string.", "#{field}__#{key}") - - true -> - {:ok, Map.update!(attachment, :content, &Base.encode64/1)} - end - end - - defp attachment(_attachment, field), - do: invalid_attachment("An attachment is a map with filename and content.", field) - - defp key_to_report(unknown_keys), - do: Enum.find(unknown_keys, hd(unknown_keys), &(&1 in @attachment_field_names)) - - defp unknown_attachment_key(key, field) when key in @attachment_field_names, - do: - invalid_attachment( - ~s(Attachment keys are atoms: use :#{key}, not "#{key}".), - "#{field}__#{key}" - ) - - defp unknown_attachment_key(key, field) when is_atom(key) or is_binary(key), - do: invalid_attachment(attachment_keys_message(key), "#{field}__#{key}") - - defp unknown_attachment_key(key, field), - do: invalid_attachment(attachment_keys_message(inspect(key)), field) - - defp attachment_keys_message(key), - do: "An attachment takes filename, content, content_type and content_id, not #{key}." - - defp invalid_attachment(message, field), do: invalid("invalid_attachment", message, field) - defp refuse_fields_outside(options, form) do {fields, other_fields} = case form do @@ -771,6 +799,15 @@ defmodule PostHog.Emails do # Requests defp request(connection, method, path, headers, body) do + url = connection.app_host <> path + + case send_request(connection, method, url, headers, body, connection.timeout_ms) do + {:ok, status, body, retry_after_ms} -> {:ok, status, fields(body), retry_after_ms} + {:error, error} -> {:error, error} + end + end + + defp send_request(connection, method, url, headers, body, timeout_ms) do # The SDK parses the body itself, so a malformed acceptance stays an api_error with its status. options = [ @@ -779,17 +816,14 @@ defmodule PostHog.Emails do redirect: false, compress_body: false, decode_body: false, - receive_timeout: connection.timeout_ms + receive_timeout: timeout_ms ] ++ if(body, do: [body: body], else: []) %{module: module, client: client} = connection.api_client - case within_deadline(connection.timeout_ms, fn -> - call_client(module, client, method, connection.app_host <> path, options) - end) do + case within_deadline(timeout_ms, fn -> call_client(module, client, method, url, options) end) do {:ok, %{status: status} = response} when is_integer(status) -> - {:ok, status, fields(Map.get(response, :body)), - retry_after_ms(Map.get(response, :headers))} + {:ok, status, Map.get(response, :body), retry_after_ms(Map.get(response, :headers))} {:error, _reason} -> {:error, Error.new("network_error", "The request to PostHog failed.")} @@ -912,8 +946,10 @@ defmodule PostHog.Emails do # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" - # Waiting doesn't fix a monthly quota. - defp retryable?(%Error{status_code: 429, code: code}), do: code != "quota_limited" + # Waiting doesn't fix a monthly quota, or the day's upload quota. + defp retryable?(%Error{status_code: 429, code: code}), + do: code not in ["quota_limited", "upload_quota_exceeded"] + defp retryable?(%Error{status_code: status}), do: status == 503 defp retry_delay_ms(error, attempt, max_retries) do @@ -953,6 +989,24 @@ defmodule PostHog.Emails do defp parse_send_reply(reply), do: reply_error(reply) + defp parse_uploads_reply({:ok, status, fields, _retry_after}, files) when status in 200..299 do + case Attachments.uploads(fields, files) do + {:ok, uploads} -> + {:ok, uploads} + + :error -> + {:error, + Error.new("api_error", "PostHog answered the upload request in a way the SDK can't use.", + status_code: status + )} + end + end + + defp parse_uploads_reply(reply, files) do + {:error, error} = reply_error(reply) + {:error, %{error | field: Attachments.field(error.field, files)}} + end + defp warnings(warnings) when is_list(warnings) do for %{} = warning <- warnings do %Warning{ diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex new file mode 100644 index 00000000..4ebdef07 --- /dev/null +++ b/lib/posthog/emails/attachments.ex @@ -0,0 +1,293 @@ +defmodule PostHog.Emails.Attachments do + @moduledoc false + # The files of a send: their checks before any request, the upload request, PostHog's answer to + # it, and the errors of an upload. `PostHog.Emails` makes the requests. + + alias PostHog.Emails.Error + + @max_files 10 + @max_message_bytes 40_000_000 + @headers_and_mime_boundaries_bytes 64 * 1024 + @limit "10 files and 40 MB per email after Base64 encoding, about 29 MB of files" + @fields [:filename, :content, :content_type, :content_id] + @field_names Enum.map(@fields, &Atom.to_string/1) + + @type file() :: %{ + field: String.t(), + filename: String.t(), + content: binary(), + sha256: String.t(), + size: pos_integer(), + reference: map() + } + + @type upload() :: %{url: String.t(), headers: [{String.t(), String.t()}]} + + @doc """ + Checks the files and the size of the email they make with `bodies`, and hashes each file. + """ + @spec prepare(term(), [String.t()]) :: {:ok, [file()]} | {:error, Error.t()} + def prepare(attachments, bodies) when is_list(attachments) do + with :ok <- check_count(attachments), + {:ok, files} <- check_each(attachments) do + check_size(files, bodies) + end + end + + def prepare(_attachments, _bodies), + do: invalid("attachments must be a list of files.", "attachments") + + @doc "The body of the upload request: each distinct file once." + @spec upload_request([file()]) :: map() + def upload_request(files), + do: %{files: for(file <- distinct(files), do: %{sha256: file.sha256, size: file.size})} + + @doc """ + The files PostHog asks the SDK to upload, from its answer to the upload request, or `:error` + when the answer leaves a file out or names an upload the SDK can't make. + """ + @spec uploads(map(), [file()]) :: {:ok, [{file(), upload()}]} | :error + def uploads(%{"files" => answers}, files) when is_list(answers), + do: files |> distinct() |> uploads(answers, []) + + def uploads(_answer, _files), do: :error + + defp uploads([], _answers, uploads), do: {:ok, Enum.reverse(uploads)} + + defp uploads([file | files], answers, uploads) do + answer = Enum.find(answers, &match?(%{"sha256" => sha256} when sha256 == file.sha256, &1)) + + case upload(answer) do + :present -> uploads(files, answers, uploads) + {:ok, upload} -> uploads(files, answers, [{file, upload} | uploads]) + :error -> :error + end + end + + @doc """ + Names the option a refusal of the upload request is about. PostHog names the upload request's + `files`, and the caller passed `attachments`. + """ + @spec field(String.t() | nil, [file()]) :: String.t() | nil + def field("files", _files), do: "attachments" + + def field("files__" <> rest, files) do + with [index | _] <- String.split(rest, "__"), + {index, ""} <- Integer.parse(index), + %{field: field} <- Enum.at(distinct(files), index) do + field <> "__content" + else + _other -> "attachments" + end + end + + def field(attr, _files), do: attr + + @doc "The error of an upload that failed, with what the storage answered, if anything." + @spec upload_failed(file(), String.t(), pos_integer() | nil, term()) :: Error.t() + def upload_failed(file, url, status, body) do + host = URI.parse(url).host + + answer = + if status, + do: "the storage answered #{storage_answer(status, body)}", + else: "the request failed" + + Error.new( + "attachment_upload_failed", + "Uploading #{file.filename} to #{host} failed: #{answer}.", + status_code: status, + field: file.field, + why: + "The SDK sends an email only once each of its files is uploaded, so no email was sent.", + fix: + "Send again. If it keeps failing, check that your app can reach #{host}, and that no proxy changes the upload." + ) + end + + defp check_count(attachments) when length(attachments) > @max_files do + {:error, + Error.new( + "too_many_attachments", + "An email carries at most #{@max_files} files, so no email was sent.", + field: "attachments", + why: "PostHog sends at most #{@limit}.", + fix: "Attach at most #{@max_files} files, and link to any others from the email body." + )} + end + + defp check_count(_attachments), do: :ok + + defp check_each(attachments) do + attachments + |> Enum.with_index() + |> Enum.reduce_while({:ok, []}, fn {attachment, index}, {:ok, files} -> + case file(attachment, "attachments__#{index}") do + {:ok, file} -> {:cont, {:ok, [file | files]}} + error -> {:halt, error} + end + end) + |> case do + {:ok, files} -> {:ok, Enum.reverse(files)} + error -> error + end + end + + defp check_size(files, bodies) do + sizes = Enum.map(files, & &1.size) ++ Enum.map(bodies, &byte_size/1) + + if message_bytes(sizes) > @max_message_bytes do + {:error, + Error.new( + "message_too_large", + "These files make the email too large to send, so no email was sent.", + field: "attachments", + why: + "PostHog sends emails up to 40 MB, and Base64 encoding makes each file and body about a third larger.", + fix: + "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." + )} + else + {:ok, files} + end + end + + # PostHog's own estimate: each file and body as Base64 in 76-character lines, plus headers. + defp message_bytes(sizes) do + Enum.reduce(sizes, @headers_and_mime_boundaries_bytes, fn size, total -> + total + ceil_div(ceil_div(size, 3) * 4 * 78, 76) + end) + end + + defp ceil_div(dividend, divisor), do: div(dividend + divisor - 1, divisor) + + defp file(attachment, field) when is_map(attachment) and not is_struct(attachment) do + attachment = Map.reject(attachment, fn {_key, value} -> is_nil(value) end) + + with :ok <- check_keys(attachment, field), + :ok <- check_values(attachment, field) do + sha256 = :crypto.hash(:sha256, attachment.content) |> Base.encode16(case: :lower) + + {:ok, + %{ + field: field, + filename: attachment.filename, + content: attachment.content, + sha256: sha256, + size: byte_size(attachment.content), + reference: + attachment + |> Map.take([:filename, :content_type, :content_id]) + |> Map.put(:sha256, sha256) + }} + end + end + + defp file(_attachment, field), + do: invalid("An attachment is a map with filename and content.", field) + + defp check_keys(attachment, field) do + unknown = attachment |> Map.keys() |> Enum.reject(&(&1 in @fields)) |> Enum.sort() + + cond do + Map.has_key?(attachment, :path) -> + invalid( + "PostHog can't fetch an attachment from a path or URL. Read the file, for example " <> + "with File.read!/1, and pass its bytes as content.", + field <> "__path" + ) + + unknown != [] -> + unknown_key(key_to_report(unknown), field) + + true -> + :ok + end + end + + defp check_values(attachment, field) do + cond do + not is_binary(attachment[:filename]) -> + invalid("An attachment needs its filename as a string.", field <> "__filename") + + not is_binary(attachment[:content]) -> + invalid( + "An attachment needs content: the file's bytes as a binary.", + field <> "__content" + ) + + attachment.content == "" -> + invalid( + "An attachment's content is empty, so the recipient would get nothing to open.", + field <> "__content" + ) + + key = Enum.find([:content_type, :content_id], ¬_string?(attachment, &1)) -> + invalid("An attachment's #{key} must be a string.", "#{field}__#{key}") + + true -> + :ok + end + end + + defp not_string?(attachment, key), + do: Map.has_key?(attachment, key) and not is_binary(attachment[key]) + + defp key_to_report(unknown_keys), + do: Enum.find(unknown_keys, hd(unknown_keys), &(&1 in @field_names)) + + defp unknown_key(key, field) when key in @field_names, + do: invalid(~s(Attachment keys are atoms: use :#{key}, not "#{key}".), "#{field}__#{key}") + + defp unknown_key(key, field) when is_atom(key) or is_binary(key), + do: invalid(keys_message(key), "#{field}__#{key}") + + defp unknown_key(key, field), do: invalid(keys_message(inspect(key)), field) + + defp keys_message(key), + do: "An attachment takes filename, content, content_type and content_id, not #{key}." + + defp invalid(message, field), + do: {:error, Error.new("invalid_attachment", message, field: field)} + + defp distinct(files), do: Enum.uniq_by(files, & &1.sha256) + + defp upload(%{"status" => "present"}), do: :present + + defp upload(%{"status" => "upload", "url" => url, "method" => "PUT", "headers" => headers}) + when is_binary(url) and is_map(headers) do + with %URI{scheme: scheme, host: host} + when scheme in ["https", "http"] and is_binary(host) and host != "" <- + URI.parse(url), + true <- Enum.all?(headers, fn {name, value} -> is_binary(name) and is_binary(value) end) do + {:ok, %{url: url, headers: Map.to_list(headers)}} + else + _unusable -> :error + end + end + + defp upload(_answer), do: :error + + # S3 answers an error as XML with a Code and a Message. + defp storage_answer(status, body) when is_binary(body) do + case {xml_text(body, "Code"), xml_text(body, "Message")} do + {nil, _message} -> + "HTTP #{status}" + + {code, nil} -> + "HTTP #{status} #{code}" + + {code, message} -> + "HTTP #{status} #{code}: #{message |> String.slice(0, 300) |> String.trim_trailing(".")}" + end + end + + defp storage_answer(status, _body), do: "HTTP #{status}" + + defp xml_text(body, tag) do + case Regex.run(~r{<#{tag}>([^<]*)}, body, capture: :all_but_first) do + [text] when text != "" -> text + _none -> nil + end + end +end diff --git a/lib/posthog/emails/error.ex b/lib/posthog/emails/error.ex index 2aaed8d1..3c46a11c 100644 --- a/lib/posthog/emails/error.ex +++ b/lib/posthog/emails/error.ex @@ -10,8 +10,10 @@ defmodule PostHog.Emails.Error do - `:name` - the category, one of `t:name/0`. - `:code` - PostHog's error code, such as `"quota_limited"` or `"recipient_suppressed"`, or an SDK code: `"missing_secret_key"`, `"missing_app_host"`, `"client_disabled"`, - `"client_closed"`, `"network_error"`, `"api_error"` (an answer without a code), `"unexpected_error"`, or an - `"invalid_*"` code for an option the SDK refused before any request. + `"client_closed"`, `"network_error"`, `"api_error"` (an answer without a code), + `"unexpected_error"`, `"attachment_upload_failed"`, `"too_many_attachments"` or + `"message_too_large"` for files the SDK refused before any request, or an `"invalid_*"` + code for an option the SDK refused before any request. - `:message` - human-readable message. - `:status_code` - the HTTP status, when PostHog answered. - `:why` and `:fix` - why PostHog refused the email and how to fix it, when it said. @@ -69,6 +71,11 @@ defmodule PostHog.Emails.Error do defp name(401, _code), do: :missing_api_key defp name(nil, "network_error"), do: :service_unavailable defp name(nil, "invalid_" <> _), do: :validation_error + + defp name(nil, code) when code in ["too_many_attachments", "message_too_large"], + do: :validation_error + + defp name(_status, "attachment_upload_failed"), do: :application_error defp name(nil, _code), do: :application_error defp name(400, _code), do: :validation_error defp name(403, _code), do: :invalid_permission diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index d4183a0e..f9e49894 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -28,6 +28,7 @@ defmodule PostHog.EmailsTest do setup :verify_on_exit! @send_url "https://us.posthog.com/api/projects/@current/messaging/send/email/" + @uploads_url "https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/" @template_id "0190F6D2-7C1E-7B3A-9F5E-1C2D3E4F5A6B" @message_id "01a18356-c56c-0001-e494-7666bc915188" @@ -281,59 +282,31 @@ defmodule PostHog.EmailsTest do end end - describe "send/2 with cc, bcc and attachments" do + describe "send/2 with cc and bcc" do @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] - @pdf <<37, 80, 68, 70, 45, 0, 255, 10>> - test "posts copies and Base64 attachments", %{name: name} do + test "posts copies, a string as a one-item list", %{name: name} do expect_reply(accepted()) assert {:ok, %Emails.SendResult{id: @message_id}} = Emails.send( name, @code_send ++ - [ - cc: "finance@example.com", - bcc: ["audit@example.com", "archive@example.com"], - attachments: [ - %{filename: "invoice.pdf", content: @pdf, content_type: "application/pdf"}, - %{filename: "logo.png", content: "png bytes", content_id: "logo"} - ] - ] + [cc: "finance@example.com", bcc: ["audit@example.com", "archive@example.com"]] ) body = sent_body() assert body["cc"] == ["finance@example.com"] assert body["bcc"] == ["audit@example.com", "archive@example.com"] - - assert body["attachments"] == [ - %{ - "filename" => "invoice.pdf", - "content" => Base.encode64(@pdf), - "content_type" => "application/pdf" - }, - %{ - "filename" => "logo.png", - "content" => Base.encode64("png bytes"), - "content_id" => "logo" - } - ] end - test "leaves out empty lists and nil attachment fields", %{name: name} do + test "leaves out empty lists", %{name: name} do expect_reply(accepted()) assert {:ok, _} = Emails.send( name, - @code_send ++ - [ - reply_to: [], - cc: [], - bcc: nil, - attachments: [], - distinct_id: "user_123" - ] + @code_send ++ [reply_to: [], cc: [], bcc: nil, attachments: []] ) body = sent_body() @@ -341,20 +314,10 @@ defmodule PostHog.EmailsTest do refute Map.has_key?(body, "cc") refute Map.has_key?(body, "bcc") refute Map.has_key?(body, "attachments") - - expect_reply(accepted()) - - assert {:ok, _} = - Emails.send( - name, - @code_send ++ - [attachments: [%{filename: "a.txt", content: "a", content_type: nil}]] - ) - - assert sent_body()["attachments"] == [%{"filename" => "a.txt", "content" => "YQ=="}] + refute_requested() end - test "a template send refuses them before any request", %{name: name} do + test "a template send refuses them and attachments before any request", %{name: name} do for {field, value} <- [ cc: "finance@example.com", bcc: "audit@example.com", @@ -375,42 +338,360 @@ defmodule PostHog.EmailsTest do refute_requested() end - test "asks for atom keys when an attachment has string keys", %{name: name} do - assert {:error, - %Error{code: "invalid_attachment", field: "attachments__0__content"} = error} = + test "refuses cc and bcc that are not addresses", %{name: name} do + for {field, value} <- [cc: [:nope], bcc: " ", cc: %{a: 1}] do + assert {:error, %Error{code: "invalid_form", field: field_name}} = + Emails.send(name, @code_send ++ [{field, value}]) + + assert field_name == to_string(field) + end + + refute_requested() + end + end + + describe "send/2 with attachments" do + @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] + @pdf <<37, 80, 68, 70, 45, 0, 255, 10>> + @logo "png bytes" + @put_url "https://files.example.com/team/1/file?X-Amz-Signature=signed-secret" + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + + defp present(bytes), + do: %{ + "sha256" => sha256(bytes), + "status" => "present", + "expires_at" => "2026-12-16T09:00:00Z" + } + + defp to_upload(bytes) do + %{ + "sha256" => sha256(bytes), + "status" => "upload", + "url" => @put_url, + "method" => "PUT", + "headers" => %{ + "Content-Length" => to_string(byte_size(bytes)), + "x-amz-checksum-sha256" => :crypto.hash(:sha256, bytes) |> Base.encode64() + }, + "url_expires_at" => "2026-11-09T10:15:00Z" + } + end + + defp slots(answers), do: {:ok, %{status: 200, headers: %{}, body: %{"files" => answers}}} + defp stored, do: {:ok, %{status: 200, headers: %{}, body: ""}} + + defp bucket_refused(status, code, message), + do: + {:ok, + %{ + status: status, + headers: %{"content-type" => "application/xml"}, + body: + ~s(\n#{code}#{message}/team/1/file) + }} + + defp requests do + receive do + {:request, method, url, opts} -> [{method, url, opts} | requests()] + after + 0 -> [] + end + end + + defp with_files(attachments), do: @code_send ++ [attachments: attachments] + + test "uploads the files PostHog asks for, then sends their references", %{name: name} do + expect_reply(slots([to_upload(@pdf), present(@logo)])) + expect_reply(stored()) + expect_reply(accepted()) + + assert {:ok, %Emails.SendResult{id: @message_id}} = Emails.send( name, - @code_send ++ [attachments: [%{"filename" => "a.txt", "content" => "a"}]] + with_files([ + %{filename: "invoice.pdf", content: @pdf, content_type: "application/pdf"}, + %{filename: "logo.png", content: @logo, content_id: "logo"} + ]) ) - assert error.message =~ ~s(:content, not "content") + assert [ + {:post, @uploads_url, ask}, + {:put, @put_url, put}, + {:post, @send_url, send} + ] = requests() + + assert Jason.decode!(ask[:body]) == %{ + "files" => [ + %{"sha256" => sha256(@pdf), "size" => byte_size(@pdf)}, + %{"sha256" => sha256(@logo), "size" => byte_size(@logo)} + ] + } - assert {:error, %Error{field: "attachments__0__content"}} = + assert header(ask, "authorization") == "Bearer phs_secret_key" + + assert put[:body] == @pdf + assert header(put, "Content-Length") == "8" + + assert header(put, "x-amz-checksum-sha256") == + :crypto.hash(:sha256, @pdf) |> Base.encode64() + + assert Jason.decode!(send[:body])["attachments"] == [ + %{ + "sha256" => sha256(@pdf), + "filename" => "invoice.pdf", + "content_type" => "application/pdf" + }, + %{"sha256" => sha256(@logo), "filename" => "logo.png", "content_id" => "logo"} + ] + end + + test "uploads without the secret key or any PostHog header", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_reply(stored()) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [_ask, {:put, @put_url, put}, _send] = requests() + header_names = Enum.map(put[:headers], fn {key, _value} -> String.downcase(key) end) + assert Enum.sort(header_names) == ["content-length", "x-amz-checksum-sha256"] + end + + test "uploads nothing when PostHog has every file", %{name: name} do + expect_reply(slots([present(@pdf)])) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, @uploads_url, _}, {:post, @send_url, _}] = requests() + end + + test "asks once for a file attached twice, and sends both references", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_reply(stored()) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send( name, - @code_send ++ - [attachments: [%{"filename" => "a.txt", "content" => "a", "cid" => "logo"}]] + with_files([ + %{filename: "a.pdf", content: @pdf}, + %{filename: "copy.pdf", content: @pdf} + ]) ) - refute_requested() + assert [{:post, _, ask}, {:put, _, _}, {:post, _, send}] = requests() + assert [%{"sha256" => sha}] = Jason.decode!(ask[:body])["files"] + + assert [%{"sha256" => ^sha}, %{"sha256" => ^sha}] = + Jason.decode!(send[:body])["attachments"] end - test "refuses an attachment key that has no string form", %{name: name} do - attachment = %{{:extra, :field} => true, filename: "a.txt", content: "a"} + test "sends nothing after a failed upload, and returns the storage's answer", %{name: name} do + expect_reply(slots([present(@logo), to_upload(@pdf)])) - assert {:error, %Error{code: "invalid_attachment", field: "attachments__0"} = error} = - Emails.send(name, @code_send ++ [attachments: [attachment]]) + expect_reply( + bucket_refused( + 400, + "BadDigest", + "The SHA256 you specified did not match the calculated checksum." + ) + ) - assert error.message =~ "{:extra, :field}" + assert {:error, %Error{} = error} = + Emails.send( + name, + with_files([ + %{filename: "logo.png", content: @logo}, + %{filename: "invoice.pdf", content: @pdf} + ]) + ) + + assert %Error{ + code: "attachment_upload_failed", + name: :application_error, + status_code: 400, + field: "attachments__1" + } = error + + assert error.message =~ "invoice.pdf" + assert error.message =~ "files.example.com" + assert error.message =~ "BadDigest" + assert error.message =~ "did not match the calculated checksum." + refute error.message =~ ".." + assert error.why =~ "no email was sent" + refute inspect(error) =~ "signed-secret" + refute inspect(error) =~ @pdf + + assert [{:post, @uploads_url, _}, {:put, @put_url, _}] = requests() + end + + test "retries an upload the storage couldn't take, then sends", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_reply(bucket_refused(503, "SlowDown", "Please reduce your request rate.")) + expect_reply({:error, %Req.TransportError{reason: :closed}}) + expect_reply(stored()) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, _, _}, {:put, _, _}, {:put, _, _}, {:put, _, _}, {:post, @send_url, _}] = + requests() + end + + test "an upload that keeps failing at the network sends nothing", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_reply({:error, %Req.TransportError{reason: :econnrefused}}) + + assert {:error, %Error{code: "attachment_upload_failed", status_code: nil} = error} = + Emails.send( + name, + with_files([%{filename: "a.pdf", content: @pdf}]) ++ [max_retries: 0] + ) + + assert error.message =~ "files.example.com" + assert error.field == "attachments__0" + refute inspect(error) =~ "signed-secret" + assert [{:post, _, _}, {:put, _, _}] = requests() + end + + test "a send retry doesn't upload again", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_reply(stored()) + expect_reply(refused(503, %{"code" => "service_unavailable"})) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [ + {:post, @uploads_url, _}, + {:put, _, _}, + {:post, @send_url, first}, + {:post, @send_url, second} + ] = + requests() + + assert header(first, "idempotency-key") == header(second, "idempotency-key") + end + + test "returns PostHog's refusal of the uploads, naming the attachment", %{name: name} do + expect_reply( + refused(400, %{ + "type" => "invalid_request", + "code" => "message_too_large", + "detail" => + "These files are too large for one email, so PostHog didn't prepare the uploads.", + "attr" => "files__1__size", + "why" => "An email can be at most 40 MB after Base64 encoding, about 29 MB of files.", + "fix" => "Send fewer or smaller files." + }) + ) + + assert {:error, + %Error{ + code: "message_too_large", + status_code: 400, + field: "attachments__2__content", + why: "An email can be at most 40 MB" <> _ + }} = + Emails.send( + name, + with_files([ + %{filename: "a.png", content: @logo}, + %{filename: "b.png", content: @logo}, + %{filename: "c.pdf", content: @pdf} + ]) + ) + + expect_reply( + refused(400, %{"code" => "field_not_accepted", "attr" => "files", "detail" => "Not yet."}) + ) + + assert {:error, %Error{code: "field_not_accepted", field: "attachments"}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, @uploads_url, _}, {:post, @uploads_url, _}] = requests() + end + + test "doesn't retry the daily upload quota, and retries unavailable storage", %{name: name} do + expect_reply( + refused(429, %{"code" => "upload_quota_exceeded", "detail" => "Enough today."}) + ) + + assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + refute_received {:slept, _} + + expect_reply(refused(503, %{"code" => "delivery_unavailable"})) + expect_reply(slots([present(@pdf)])) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + assert_received {:slept, _} + end + + test "an answer the SDK can't use sends nothing", %{name: name} do + for answer <- [ + [], + [%{"sha256" => sha256(@logo), "status" => "present"}], + [%{"sha256" => sha256(@pdf), "status" => "copied"}], + [Map.delete(to_upload(@pdf), "url")], + [Map.put(to_upload(@pdf), "url", "ftp://files.example.com/a")], + [Map.put(to_upload(@pdf), "method", "POST")], + [Map.put(to_upload(@pdf), "headers", %{"Content-Length" => 8})] + ] do + expect_reply(slots(answer)) + + assert {:error, %Error{code: "api_error", status_code: 200}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, @uploads_url, _}] = requests() + end + end + + test "refuses more than 10 files before any request", %{name: name} do + files = for i <- 1..11, do: %{filename: "#{i}.txt", content: "file #{i}"} + + assert {:error, + %Error{ + code: "too_many_attachments", + name: :validation_error, + field: "attachments" + } = error} = Emails.send(name, with_files(files)) + + assert error.fix =~ "10" refute_requested() end - test "refuses cc and bcc that are not addresses", %{name: name} do - for {field, value} <- [cc: [:nope], bcc: " ", cc: %{a: 1}] do - assert {:error, %Error{code: "invalid_form", field: field_name}} = - Emails.send(name, @code_send ++ [{field, value}]) + test "refuses an email over 40 MB after Base64 encoding before any request", %{name: name} do + largest_that_fits = 29_182_875 + file = :binary.copy(<<0>>, largest_that_fits) - assert field_name == to_string(field) + expect_reply(slots([present(file)])) + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send( + name, + @code_send ++ [html: "", attachments: [%{filename: "a", content: file}]] + ) + + assert [_ask, _send] = requests() + + for options <- [ + [html: "", attachments: [%{filename: "a", content: file <> <<0>>}]], + [html: "

", attachments: [%{filename: "a", content: file}]], + [html: "", text: "x", attachments: [%{filename: "a", content: file}]] + ] do + assert {:error, + %Error{code: "message_too_large", name: :validation_error, field: "attachments"} = + error} = Emails.send(name, @code_send ++ options) + + assert error.why =~ "40 MB" + assert error.fix =~ "29 MB" end refute_requested() @@ -426,6 +707,7 @@ defmodule PostHog.EmailsTest do {[%{filename: "a.txt", content: "a", cid: "logo"}], "attachments__0__cid"}, {[%{"filename" => "a.txt", "content" => "a"}], "attachments__0__content"}, {[%{filename: "a.txt"}], "attachments__0__content"}, + {[%{filename: "a.txt", content: ""}], "attachments__0__content"}, {[%{filename: "a.txt", content: [97]}], "attachments__0__content"}, {[%{filename: "a.txt", content: <<1::3>>}], "attachments__0__content"}, {[%{content: "a"}], "attachments__0__filename"}, @@ -436,59 +718,72 @@ defmodule PostHog.EmailsTest do ] do assert {:error, %Error{code: "invalid_attachment", name: :validation_error, field: ^field}} = - Emails.send(name, @code_send ++ [attachments: attachments]) + Emails.send(name, with_files(attachments)) end refute_requested() end - test "a path points to reading the file", %{name: name} do - assert {:error, %Error{message: message}} = + test "asks for atom keys when an attachment has string keys", %{name: name} do + assert {:error, + %Error{code: "invalid_attachment", field: "attachments__0__content"} = error} = + Emails.send(name, with_files([%{"filename" => "a.txt", "content" => "a"}])) + + assert error.message =~ ~s(:content, not "content") + + assert {:error, %Error{field: "attachments__0__content"}} = Emails.send( name, - @code_send ++ [attachments: [%{filename: "a.pdf", path: "/srv/a.pdf"}]] + with_files([%{"filename" => "a.txt", "content" => "a", "cid" => "logo"}]) ) + refute_requested() + end + + test "refuses an attachment key that has no string form", %{name: name} do + attachment = %{{:extra, :field} => true, filename: "a.txt", content: "a"} + + assert {:error, %Error{code: "invalid_attachment", field: "attachments__0"} = error} = + Emails.send(name, with_files([attachment])) + + assert error.message =~ "{:extra, :field}" + refute_requested() + end + + test "a path points to reading the file", %{name: name} do + assert {:error, %Error{message: message}} = + Emails.send(name, with_files([%{filename: "a.pdf", path: "/srv/a.pdf"}])) + assert message =~ "File.read" end test "never repeats the file's content in an error", %{name: name} do assert {:error, %Error{} = error} = - Emails.send( - name, - @code_send ++ - [attachments: [%{filename: 1, content: "SECRET-FILE-BYTES"}]] - ) + Emails.send(name, with_files([%{filename: 1, content: "SECRET-FILE-BYTES"}])) refute inspect(error) =~ "SECRET-FILE-BYTES" end - test "returns a server's refusal of a field it doesn't take", %{name: name} do + test "returns a send's refusal of a file it can't find", %{name: name} do + expect_reply(slots([present(@pdf)])) + expect_reply( refused(400, %{ - "type" => "validation_error", - "code" => "invalid_request", - "detail" => "PostHog doesn't take this field yet, so it didn't send the email.", - "attr" => "attachments", - "why" => "This PostHog version doesn't take the field yet.", - "fix" => "Send the email without this field." + "type" => "invalid_request", + "code" => "attachment_not_uploaded", + "detail" => "A file this email names isn't uploaded, so PostHog didn't send the email.", + "attr" => "attachments__0__sha256", + "why" => "This project has no uploaded file with this SHA-256.", + "fix" => "Upload the file and wait for its upload to succeed, then send again." }) ) assert {:error, %Error{ - code: "invalid_request", - field: "attachments", - name: :validation_error, + code: "attachment_not_uploaded", + field: "attachments__0__sha256", status_code: 400 - }} = - Emails.send( - name, - @code_send ++ [attachments: [%{filename: "a.txt", content: "a"}]] - ) - - assert_received {:request, :post, @send_url, _opts} - refute_received {:request, _, _, _} + }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end end From 82e72be6eb4ad7f1f2f3421a20f9af88d16c158a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:14:16 +0000 Subject: [PATCH 17/38] fix(emails): name attachments when PostHog refuses the uploads without a field Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails/attachments.ex | 6 +++--- test/posthog/emails_test.exs | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 4ebdef07..62eeb31f 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -65,11 +65,11 @@ defmodule PostHog.Emails.Attachments do end @doc """ - Names the option a refusal of the upload request is about. PostHog names the upload request's - `files`, and the caller passed `attachments`. + Names the option a refusal of the upload request is about: `attachments`, or one of them. + PostHog names the upload request's `files`, and the caller passed `attachments`. """ @spec field(String.t() | nil, [file()]) :: String.t() | nil - def field("files", _files), do: "attachments" + def field(attr, _files) when attr in [nil, "files"], do: "attachments" def field("files__" <> rest, files) do with [index | _] <- String.split(rest, "__"), diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index f9e49894..06cd8e79 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -615,6 +615,21 @@ defmodule PostHog.EmailsTest do assert [{:post, @uploads_url, _}, {:post, @uploads_url, _}] = requests() end + test "a PostHog without the upload route sends nothing", %{name: name} do + expect_reply( + refused(404, %{ + "type" => "invalid_request", + "code" => "not_found", + "detail" => "Not found." + }) + ) + + assert {:error, %Error{code: "not_found", status_code: 404, field: "attachments"}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, @uploads_url, _}] = requests() + end + test "doesn't retry the daily upload quota, and retries unavailable storage", %{name: name} do expect_reply( refused(429, %{"code" => "upload_quota_exceeded", "detail" => "Enough today."}) From 57bbab85c1a169fddac8fd60767cc606ab4238eb Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:21:09 +0000 Subject: [PATCH 18/38] refactor(emails): prepare attachments beside the content, one template reason The content builders return only the request content again, and send/2 prepares the files next to them. A template send refuses every code-form field with one reason. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 58 ++++++++++++++----------------- lib/posthog/emails/attachments.ex | 14 +++----- test/posthog/emails_test.exs | 2 +- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index dabb7917..528e382d 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -196,7 +196,6 @@ defmodule PostHog.Emails do :capture_engagement ] @address_fields [:reply_to, :cc, :bcc] - @template_owned_fields [:cc, :bcc, :attachments] @typedoc """ Options for `send/2`. Pass `:template_id` or `:template_key`, never both. @@ -355,7 +354,6 @@ defmodule PostHog.Emails do end) end - # Files are uploaded straight to PostHog's file storage, which never gets the secret key. defp upload(_connection, %{files: []}), do: :ok defp upload(connection, prepared) do @@ -385,9 +383,11 @@ defmodule PostHog.Emails do end) end + defp upload_timeout_ms(connection, file), + do: connection.timeout_ms + div(file.size, 1_000_000) * 1_000 + defp put_file(connection, file, upload, max_retries) do - # A large file takes longer to send than a request without one. - timeout_ms = connection.timeout_ms + div(file.size, 1_000_000) * 1_000 + timeout_ms = upload_timeout_ms(connection, file) connection |> with_retries(max_retries, fn -> @@ -435,10 +435,12 @@ defmodule PostHog.Emails do defp prepare(options) do with :ok <- check_recipient(options), - {:ok, {content, files}} <- content(options), + {:ok, content} <- content(options), + {:ok, files} <- attachments(options), {:ok, idempotency_key} <- idempotency_key(options), {:ok, max_retries} <- max_retries(options), - {:ok, body} <- encode(content, options) do + {:ok, body} <- + encode(put_if_any(content, :attachments, Enum.map(files, & &1.reference)), options) do {:ok, %{body: body, files: files, idempotency_key: idempotency_key, max_retries: max_retries}} end @@ -483,12 +485,9 @@ defmodule PostHog.Emails do true -> with :ok <- refuse_fields_outside(options, :template) do {:ok, - {options - |> Map.take([:to, :distinct_id]) - |> Map.merge(%{ - template_id: String.downcase(template_id), - message_data: template_data - }), []}} + options + |> Map.take([:to, :distinct_id]) + |> Map.merge(%{template_id: String.downcase(template_id), message_data: template_data})} end end end @@ -496,21 +495,22 @@ defmodule PostHog.Emails do defp code_content(%{template_key: template_key} = options) do with :ok <- check_template_key(template_key), :ok <- refuse_fields_outside(options, :code), - :ok <- check_code_fields(options), - {:ok, files} <- - Attachments.prepare( - Map.get(options, :attachments, []), - Enum.flat_map([:html, :text], &List.wrap(options[&1])) - ) do + :ok <- check_code_fields(options) do {:ok, - {options - |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) - |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) - |> put_addresses(options) - |> put_if_any(:attachments, Enum.map(files, & &1.reference)), files}} + options + |> Map.take([:to, :distinct_id, :template_key, :subject, :html, :text, :from]) + |> Map.put(:capture_engagement, Map.get(options, :capture_engagement, true)) + |> put_addresses(options)} end end + defp attachments(options) do + Attachments.prepare( + Map.get(options, :attachments, []), + Enum.flat_map([:html, :text], &List.wrap(options[&1])) + ) + end + defp check_template_key(template_key) do if is_binary(template_key) and template_key =~ ~r/\A[a-zA-Z0-9_-]{1,400}\z/ do :ok @@ -598,13 +598,6 @@ defmodule PostHog.Emails do end end - defp template_owns(field) when field in @template_owned_fields, - do: "A send with template_id takes its recipients and content from the template." - - defp template_owns(_field), - do: - "A template made in PostHog sets its own sender, subject and content, and follows the project's engagement setting." - defp field_outside_error(form, field, other_form_takes_it?) do form_field = if form == :template, do: "template_id", else: "template_key" message = "A send with #{form_field} doesn't take #{field}, so no email was sent." @@ -613,7 +606,8 @@ defmodule PostHog.Emails do {why, how} = case form do :template -> - {template_owns(field), "send content from your code with template_key"} + {"A template made in PostHog sets its own sender, copies, subject, content and attachments, and follows the project's engagement setting.", + "send content from your code with template_key"} :code -> {"A send with template_key takes its content from your code.", @@ -946,7 +940,7 @@ defmodule PostHog.Emails do # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" - # Waiting doesn't fix a monthly quota, or the day's upload quota. + # Waiting doesn't fix a monthly quota. defp retryable?(%Error{status_code: 429, code: code}), do: code not in ["quota_limited", "upload_quota_exceeded"] diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 62eeb31f..de2bddc2 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -1,7 +1,5 @@ defmodule PostHog.Emails.Attachments do @moduledoc false - # The files of a send: their checks before any request, the upload request, PostHog's answer to - # it, and the errors of an upload. `PostHog.Emails` makes the requests. alias PostHog.Emails.Error @@ -90,7 +88,7 @@ defmodule PostHog.Emails.Attachments do answer = if status, - do: "the storage answered #{storage_answer(status, body)}", + do: "the storage answered #{s3_error(status, body)}", else: "the request failed" Error.new( @@ -136,7 +134,7 @@ defmodule PostHog.Emails.Attachments do defp check_size(files, bodies) do sizes = Enum.map(files, & &1.size) ++ Enum.map(bodies, &byte_size/1) - if message_bytes(sizes) > @max_message_bytes do + if base64_message_bytes(sizes) > @max_message_bytes do {:error, Error.new( "message_too_large", @@ -152,8 +150,7 @@ defmodule PostHog.Emails.Attachments do end end - # PostHog's own estimate: each file and body as Base64 in 76-character lines, plus headers. - defp message_bytes(sizes) do + defp base64_message_bytes(sizes) do Enum.reduce(sizes, @headers_and_mime_boundaries_bytes, fn size, total -> total + ceil_div(ceil_div(size, 3) * 4 * 78, 76) end) @@ -268,8 +265,7 @@ defmodule PostHog.Emails.Attachments do defp upload(_answer), do: :error - # S3 answers an error as XML with a Code and a Message. - defp storage_answer(status, body) when is_binary(body) do + defp s3_error(status, body) when is_binary(body) do case {xml_text(body, "Code"), xml_text(body, "Message")} do {nil, _message} -> "HTTP #{status}" @@ -282,7 +278,7 @@ defmodule PostHog.Emails.Attachments do end end - defp storage_answer(status, _body), do: "HTTP #{status}" + defp s3_error(status, _body), do: "HTTP #{status}" defp xml_text(body, tag) do case Regex.run(~r{<#{tag}>([^<]*)}, body, capture: :all_but_first) do diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 06cd8e79..e2262a94 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -331,7 +331,7 @@ defmodule PostHog.EmailsTest do ]) assert field_name == to_string(field) - assert error.why =~ "recipients and content from the template" + assert error.why =~ "sets its own sender, copies, subject, content and attachments" assert error.fix =~ "template_key" end From 30991585b4b69dd69e87e49089e308675ba7df16 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:39:25 +0000 Subject: [PATCH 19/38] fix(emails): upload with a plain request, and stop a send whose instance stops The storage PUT no longer goes through the api_client, whose default headers, such as one a custom client adds, would reach the storage host. Each request checks the instance still runs, so a send stopped between its steps sends nothing more. Storage 500, 502 and 504 are retried, a storage 401 no longer names the PostHog key, a refusal about the key or an outage names no option, and a template send takes empty lists as left out, as the docs say. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 85 +++++++++++----- lib/posthog/emails/attachments.ex | 17 ++-- lib/posthog/emails/error.ex | 16 +-- test/posthog/emails_test.exs | 159 ++++++++++++++++++++++++------ 4 files changed, 207 insertions(+), 70 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 528e382d..98eee2bd 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -93,14 +93,17 @@ defmodule PostHog.Emails do A send with files takes three steps: 1. The SDK hashes each file with SHA-256 and asks PostHog which files it doesn't have yet. - 2. It uploads each of those straight to PostHog's file storage, with the headers PostHog - gave and without your secret key. A file PostHog already has, such as the same terms PDF - sent to every customer, uploads nothing again. + 2. It uploads each of those straight to PostHog's file storage with a plain `Req` request, + carrying only the headers PostHog gave. The upload doesn't go through + `:api_client_module`, so no secret key or header of your client reaches the storage. A + file PostHog already has, such as the same terms PDF sent to every customer, uploads + nothing again. 3. Once every upload succeeded, it sends the email, which names each file by its hash. Your app must be able to reach the storage host that PostHog's answer names. If an upload fails, the SDK sends nothing and returns `attachment_upload_failed` with what the storage - answered. An upload gets the request timeout plus one second per MB of the file. + answered. An upload gets the request timeout plus one second per MB of the file, and retries + as a send does, as well as when the storage answers HTTP 500, 502 or 504. ## Timeouts, retries and idempotency @@ -132,8 +135,8 @@ defmodule PostHog.Emails do project. - `client_disabled`: the PostHog instance is not running or has no `:api_key`, so it sends nothing. - - `client_closed`: the PostHog instance stopped while a call waited to retry, so the retry - was not sent. + - `client_closed`: the PostHog instance stopped during a call, so the call made no further + request, and a send sent no email. A send with files can also return: @@ -151,8 +154,9 @@ defmodule PostHog.Emails do ## Testing - Requests go through the configured `:api_client_module`, so a `Mox` mock of - `PostHog.API.Client` sees every email request without network calls. + Requests to PostHog go through the configured `:api_client_module`, so a `Mox` mock of + `PostHog.API.Client` sees them without network calls. Attachment uploads go to the storage + host instead, so test a send with files against an answer that marks every file `present`. """ import Kernel, except: [send: 2] @@ -391,7 +395,7 @@ defmodule PostHog.Emails do connection |> with_retries(max_retries, fn -> - case send_request(connection, :put, upload.url, upload.headers, file.content, timeout_ms) do + case send_upload(connection, upload, file.content, timeout_ms) do {:ok, status, _body, _retry_after_ms} when status in 200..299 -> {:ok, :uploaded} @@ -422,7 +426,7 @@ defmodule PostHog.Emails do defp prepare_send(options) when is_list(options) or is_map(options) do if is_map(options) or Keyword.keyword?(options) do - options |> Enum.reject(fn {_key, value} -> is_nil(value) end) |> Map.new() |> prepare() + options |> Enum.reject(&left_out?/1) |> Map.new() |> prepare() else invalid_options() end @@ -430,6 +434,10 @@ defmodule PostHog.Emails do defp prepare_send(_options), do: invalid_options() + # Options are often built generically, so an empty list counts as left out on both forms. + defp left_out?({key, []}) when key in [:attachments | @address_fields], do: true + defp left_out?({_key, value}), do: is_nil(value) + defp invalid_options, do: {:error, Error.new("invalid_form", "send takes a keyword list or a map of options.")} @@ -709,7 +717,8 @@ defmodule PostHog.Emails do migrated_from: Map.get(config, :migrated_from), timeout_ms: config.email_request_timeout_ms, max_retries: config.email_request_max_retries, - sleep: Map.get(config, :email_sleep, &Process.sleep/1) + sleep: Map.get(config, :email_sleep, &Process.sleep/1), + upload_options: Map.get(config, :email_upload_options, []) }} end end @@ -733,7 +742,7 @@ defmodule PostHog.Emails do {:error, Error.new( "client_closed", - "The PostHog instance #{inspect(name)} stopped before the retry, so #{nothing_done}." + "The PostHog instance #{inspect(name)} stopped during the call, so #{nothing_done}." )} end end @@ -814,19 +823,43 @@ defmodule PostHog.Emails do ] ++ if(body, do: [body: body], else: []) %{module: module, client: client} = connection.api_client + run_request(connection, timeout_ms, fn -> module.request(client, method, url, options) end) + end + + # A plain request, not one through the api_client, whose defaults, such as headers a custom client + # adds, are meant for PostHog. The storage gets only the headers PostHog signed the upload with. + defp send_upload(connection, upload, content, timeout_ms) do + options = + [ + method: :put, + url: upload.url, + headers: upload.headers, + body: content, + retry: false, + redirect: false, + decode_body: false, + receive_timeout: timeout_ms + ] ++ connection.upload_options - case within_deadline(timeout_ms, fn -> call_client(module, client, method, url, options) end) do - {:ok, %{status: status} = response} when is_integer(status) -> - {:ok, status, Map.get(response, :body), retry_after_ms(Map.get(response, :headers))} + run_request(connection, timeout_ms, fn -> Req.request(options) end) + end - {:error, _reason} -> - {:error, Error.new("network_error", "The request to PostHog failed.")} + # Each request checks that the instance still runs, so a send stopped between its steps sends + # nothing more. + defp run_request(connection, timeout_ms, request) do + with :ok <- check_running(connection), + {:ok, %{status: status} = response} when is_integer(status) <- + within_deadline(timeout_ms, fn -> call_client(request) end) do + {:ok, status, Map.get(response, :body), retry_after_ms(Map.get(response, :headers))} + else + {:error, %Error{} = error} -> {:error, error} + {:error, _reason} -> {:error, Error.new("network_error", "The request to PostHog failed.")} end end # Finch raises, rather than returns, when no pooled connection frees up in time. - defp call_client(module, client, method, url, options) do - module.request(client, method, url, options) + defp call_client(request) do + request.() rescue exception in RuntimeError -> if String.starts_with?(exception.message, "Finch was unable to provide a connection"), @@ -926,18 +959,16 @@ defmodule PostHog.Emails do delay_ms -> connection.sleep.(delay_ms) - retry(connection, max_retries, attempt_once, attempt + 1) + with_retries(connection, max_retries, attempt_once, attempt + 1) end end end - defp retry(connection, max_retries, attempt_once, attempt) do - with :ok <- check_running(connection) do - with_retries(connection, max_retries, attempt_once, attempt) - end - end - # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. + # An upload writes the same bytes to the same key, and S3 asks clients to retry its 500s. + defp retryable?(%Error{code: "attachment_upload_failed", status_code: status}), + do: status in [500, 502, 503, 504] + defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" # Waiting doesn't fix a monthly quota. @@ -998,7 +1029,7 @@ defmodule PostHog.Emails do defp parse_uploads_reply(reply, files) do {:error, error} = reply_error(reply) - {:error, %{error | field: Attachments.field(error.field, files)}} + {:error, %{error | field: Attachments.field(error, files)}} end defp warnings(warnings) when is_list(warnings) do diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index de2bddc2..616cf875 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -64,12 +64,19 @@ defmodule PostHog.Emails.Attachments do @doc """ Names the option a refusal of the upload request is about: `attachments`, or one of them. - PostHog names the upload request's `files`, and the caller passed `attachments`. + PostHog names the upload request's `files`, and the caller passed `attachments`. A refusal of + the files without a field, such as a PostHog without the upload route, is about + `attachments`; one about the key, the scope, a rate limit or an outage names no option. """ - @spec field(String.t() | nil, [file()]) :: String.t() | nil - def field(attr, _files) when attr in [nil, "files"], do: "attachments" + @spec field(Error.t(), [file()]) :: String.t() | nil + def field(%Error{field: nil, status_code: status}, _files) when status in [400, 404, 413, 422], + do: "attachments" - def field("files__" <> rest, files) do + def field(%Error{field: "files"}, _files), do: "attachments" + def field(%Error{field: "files__" <> rest}, files), do: file_field(rest, files) + def field(%Error{field: field}, _files), do: field + + defp file_field(rest, files) do with [index | _] <- String.split(rest, "__"), {index, ""} <- Integer.parse(index), %{field: field} <- Enum.at(distinct(files), index) do @@ -79,8 +86,6 @@ defmodule PostHog.Emails.Attachments do end end - def field(attr, _files), do: attr - @doc "The error of an upload that failed, with what the storage answered, if anything." @spec upload_failed(file(), String.t(), pos_integer() | nil, term()) :: Error.t() def upload_failed(file, url, status, body) do diff --git a/lib/posthog/emails/error.ex b/lib/posthog/emails/error.ex index 3c46a11c..7ba1e00f 100644 --- a/lib/posthog/emails/error.ex +++ b/lib/posthog/emails/error.ex @@ -11,15 +11,18 @@ defmodule PostHog.Emails.Error do - `:code` - PostHog's error code, such as `"quota_limited"` or `"recipient_suppressed"`, or an SDK code: `"missing_secret_key"`, `"missing_app_host"`, `"client_disabled"`, `"client_closed"`, `"network_error"`, `"api_error"` (an answer without a code), - `"unexpected_error"`, `"attachment_upload_failed"`, `"too_many_attachments"` or - `"message_too_large"` for files the SDK refused before any request, or an `"invalid_*"` - code for an option the SDK refused before any request. + `"unexpected_error"`, `"too_many_attachments"` or `"message_too_large"` for files that + make the email too large, which the SDK checks before any request and PostHog checks again, + `"attachment_upload_failed"` when the storage refused an upload, or an `"invalid_*"` code + for an option the SDK refused before any request. - `:message` - human-readable message. - - `:status_code` - the HTTP status, when PostHog answered. + - `:status_code` - the HTTP status, when PostHog answered, or the storage's for + `"attachment_upload_failed"`. - `:why` and `:fix` - why PostHog refused the email and how to fix it, when it said. - `:docs_url` - a link to the docs for the error, when PostHog sent one. - `:field` - the option the error is about, such as `"to"`. - - `:retry_after_ms` - how long PostHog asked the client to wait before it sends again. + - `:retry_after_ms` - how long PostHog, or the storage, asked the client to wait before it + sends again. """ @typedoc "A coarse category of the failure." @@ -68,6 +71,8 @@ defmodule PostHog.Emails.Error do end defp name(_status, "missing_secret_key"), do: :missing_api_key + # The storage answered, so its status says nothing about PostHog's key or scope. + defp name(_status, "attachment_upload_failed"), do: :application_error defp name(401, _code), do: :missing_api_key defp name(nil, "network_error"), do: :service_unavailable defp name(nil, "invalid_" <> _), do: :validation_error @@ -75,7 +80,6 @@ defmodule PostHog.Emails.Error do defp name(nil, code) when code in ["too_many_attachments", "message_too_large"], do: :validation_error - defp name(_status, "attachment_upload_failed"), do: :application_error defp name(nil, _code), do: :application_error defp name(400, _code), do: :validation_error defp name(403, _code), do: :invalid_permission diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index e2262a94..4dedd80c 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -51,6 +51,7 @@ defmodule PostHog.EmailsTest do send(test_pid, {:slept, ms}) if context[:stop_instance_on_sleep], do: stop_supervised!(context.test) end) + |> Map.put(:email_upload_options, plug: {Req.Test, PostHog.EmailsTest.Storage}) start_link_supervised!({PostHog.Supervisor, config}) %{name: context.test} @@ -379,18 +380,37 @@ defmodule PostHog.EmailsTest do } end + setup {Req.Test, :verify_on_exit!} + defp slots(answers), do: {:ok, %{status: 200, headers: %{}, body: %{"files" => answers}}} - defp stored, do: {:ok, %{status: 200, headers: %{}, body: ""}} - defp bucket_refused(status, code, message), - do: - {:ok, - %{ - status: status, - headers: %{"content-type" => "application/xml"}, - body: - ~s(\n#{code}#{message}/team/1/file) - }} + # The storage, behind the SDK's real Req request: :stored, {:refused, status, code, message}, + # or a transport error's reason. + defp expect_storage(reply) do + test_pid = self() + + Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + url = "#{conn.scheme}://#{conn.host}#{conn.request_path}?#{conn.query_string}" + send(test_pid, {:request, :put, url, [headers: conn.req_headers, body: body]}) + + case reply do + :stored -> + Plug.Conn.send_resp(conn, 200, "") + + {:refused, status, code, message} -> + conn + |> Plug.Conn.put_resp_content_type("application/xml") + |> Plug.Conn.send_resp( + status, + ~s(\n#{code}#{message}/team/1/file) + ) + + reason -> + Req.Test.transport_error(conn, reason) + end + end) + end defp requests do receive do @@ -404,7 +424,7 @@ defmodule PostHog.EmailsTest do test "uploads the files PostHog asks for, then sends their references", %{name: name} do expect_reply(slots([to_upload(@pdf), present(@logo)])) - expect_reply(stored()) + expect_storage(:stored) expect_reply(accepted()) assert {:ok, %Emails.SendResult{id: @message_id}} = @@ -432,7 +452,7 @@ defmodule PostHog.EmailsTest do assert header(ask, "authorization") == "Bearer phs_secret_key" assert put[:body] == @pdf - assert header(put, "Content-Length") == "8" + assert header(put, "content-length") == "8" assert header(put, "x-amz-checksum-sha256") == :crypto.hash(:sha256, @pdf) |> Base.encode64() @@ -447,15 +467,19 @@ defmodule PostHog.EmailsTest do ] end - test "uploads without the secret key or any PostHog header", %{name: name} do + test "uploads with only the headers PostHog gave, not the client's", %{name: name} do expect_reply(slots([to_upload(@pdf)])) - expect_reply(stored()) + expect_storage(:stored) expect_reply(accepted()) assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) assert [_ask, {:put, @put_url, put}, _send] = requests() - header_names = Enum.map(put[:headers], fn {key, _value} -> String.downcase(key) end) + + # Req's own user-agent and accept-encoding aside, as any HTTP client's. + header_names = + for {key, _value} <- put[:headers], key not in ["user-agent", "accept-encoding"], do: key + assert Enum.sort(header_names) == ["content-length", "x-amz-checksum-sha256"] end @@ -470,7 +494,7 @@ defmodule PostHog.EmailsTest do test "asks once for a file attached twice, and sends both references", %{name: name} do expect_reply(slots([to_upload(@pdf)])) - expect_reply(stored()) + expect_storage(:stored) expect_reply(accepted()) assert {:ok, _} = @@ -492,12 +516,9 @@ defmodule PostHog.EmailsTest do test "sends nothing after a failed upload, and returns the storage's answer", %{name: name} do expect_reply(slots([present(@logo), to_upload(@pdf)])) - expect_reply( - bucket_refused( - 400, - "BadDigest", - "The SHA256 you specified did not match the calculated checksum." - ) + expect_storage( + {:refused, 400, "BadDigest", + "The SHA256 you specified did not match the calculated checksum."} ) assert {:error, %Error{} = error} = @@ -528,22 +549,53 @@ defmodule PostHog.EmailsTest do assert [{:post, @uploads_url, _}, {:put, @put_url, _}] = requests() end + @tag config: [email_request_max_retries: 3] test "retries an upload the storage couldn't take, then sends", %{name: name} do expect_reply(slots([to_upload(@pdf)])) - expect_reply(bucket_refused(503, "SlowDown", "Please reduce your request rate.")) - expect_reply({:error, %Req.TransportError{reason: :closed}}) - expect_reply(stored()) + expect_storage({:refused, 503, "SlowDown", "Please reduce your request rate."}) + expect_storage({:refused, 500, "InternalError", "We encountered an internal error."}) + expect_storage(:closed) + expect_storage(:stored) expect_reply(accepted()) assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [{:post, _, _}, {:put, _, _}, {:put, _, _}, {:put, _, _}, {:post, @send_url, _}] = - requests() + assert [ + {:post, @uploads_url, _}, + {:put, _, _}, + {:put, _, _}, + {:put, _, _}, + {:put, _, _}, + {:post, @send_url, _} + ] = requests() + end + + test "doesn't retry an upload the storage refused", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_storage({:refused, 403, "AccessDenied", "Request has expired."}) + + assert {:error, + %Error{ + code: "attachment_upload_failed", + name: :application_error, + status_code: 403 + }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + refute_received {:slept, _} + assert [{:post, _, _}, {:put, _, _}] = requests() + end + + test "a storage 401 doesn't blame the PostHog key", %{name: name} do + expect_reply(slots([to_upload(@pdf)])) + expect_storage({:refused, 401, "Unauthorized", "Proxy needs a login."}) + + assert {:error, %Error{code: "attachment_upload_failed", name: :application_error}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end test "an upload that keeps failing at the network sends nothing", %{name: name} do expect_reply(slots([to_upload(@pdf)])) - expect_reply({:error, %Req.TransportError{reason: :econnrefused}}) + expect_storage(:econnrefused) assert {:error, %Error{code: "attachment_upload_failed", status_code: nil} = error} = Emails.send( @@ -557,9 +609,48 @@ defmodule PostHog.EmailsTest do assert [{:post, _, _}, {:put, _, _}] = requests() end + test "a send whose instance stops between its steps sends nothing more", %{name: name} do + test_pid = self() + + expect(API.Mock, :request, fn _client, :post, @uploads_url, _opts -> + send(test_pid, {:asked, self()}) + + receive do + :answer -> slots([to_upload(@pdf)]) + end + end) + + sending = + Task.async(fn -> Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end) + + assert_receive {:asked, worker} + stop_supervised!(name) + send(worker, :answer) + + assert {:error, %Error{code: "client_closed"} = error} = Task.await(sending) + assert error.message =~ "no email was sent" + end + + test "a template send takes empty lists as left out", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send(name, + to: "user@example.com", + template_id: @template_id, + reply_to: [], + cc: [], + bcc: [], + attachments: [] + ) + + body = sent_body() + refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(body, &1)) + end + test "a send retry doesn't upload again", %{name: name} do expect_reply(slots([to_upload(@pdf)])) - expect_reply(stored()) + expect_storage(:stored) expect_reply(refused(503, %{"code" => "service_unavailable"})) expect_reply(accepted()) @@ -612,7 +703,13 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "field_not_accepted", field: "attachments"}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [{:post, @uploads_url, _}, {:post, @uploads_url, _}] = requests() + expect_reply(refused(401, %{"code" => "authentication_failed", "detail" => "Bad key."})) + + assert {:error, %Error{code: "authentication_failed", field: nil}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert [{:post, @uploads_url, _}, {:post, @uploads_url, _}, {:post, @uploads_url, _}] = + requests() end test "a PostHog without the upload route sends nothing", %{name: name} do @@ -635,7 +732,7 @@ defmodule PostHog.EmailsTest do refused(429, %{"code" => "upload_quota_exceeded", "detail" => "Enough today."}) ) - assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429}} = + assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429, field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) refute_received {:slept, _} From 881cafa04db98aff035e3972461c1a35e79ff293 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:43:34 +0000 Subject: [PATCH 20/38] fix(emails): keep app-wide Req defaults off uploads, and retry a storage 429 Req.new/1 merges the app's Req.default_options, so an auth or compress_body set for other requests reached the storage, leaking the credential or changing the signed bytes. The upload now keeps only its own options. A storage 429 retries again, as it did before the upload clause. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 14 ++++-- test/posthog/emails_test.exs | 94 +++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 98eee2bd..f7ce12c6 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -103,7 +103,8 @@ defmodule PostHog.Emails do Your app must be able to reach the storage host that PostHog's answer names. If an upload fails, the SDK sends nothing and returns `attachment_upload_failed` with what the storage answered. An upload gets the request timeout plus one second per MB of the file, and retries - as a send does, as well as when the storage answers HTTP 500, 502 or 504. + as a send does, as well as when the storage answers HTTP 500, 502 or 504. It ignores your + app's `Req.default_options`, which are meant for other requests. ## Timeouts, retries and idempotency @@ -827,7 +828,9 @@ defmodule PostHog.Emails do end # A plain request, not one through the api_client, whose defaults, such as headers a custom client - # adds, are meant for PostHog. The storage gets only the headers PostHog signed the upload with. + # adds, are meant for PostHog. Req.new/1 also merges the app's Req.default_options, such as an + # auth or compress_body set for other requests, so the upload keeps only its own options. The + # storage gets the bytes and the headers PostHog signed the upload with, nothing else. defp send_upload(connection, upload, content, timeout_ms) do options = [ @@ -841,7 +844,10 @@ defmodule PostHog.Emails do receive_timeout: timeout_ms ] ++ connection.upload_options - run_request(connection, timeout_ms, fn -> Req.request(options) end) + request = Req.new(options) + request = %{request | options: Map.take(request.options, Keyword.keys(options))} + + run_request(connection, timeout_ms, fn -> Req.request(request) end) end # Each request checks that the instance still runs, so a send stopped between its steps sends @@ -967,7 +973,7 @@ defmodule PostHog.Emails do # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. # An upload writes the same bytes to the same key, and S3 asks clients to retry its 500s. defp retryable?(%Error{code: "attachment_upload_failed", status_code: status}), - do: status in [500, 502, 503, 504] + do: status in [429, 500, 502, 503, 504] defp retryable?(%Error{status_code: nil, code: code}), do: code == "network_error" defp retryable?(%Error{status_code: 409, code: code}), do: code == "idempotency_key_in_flight" diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 4dedd80c..321c0b86 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -549,10 +549,11 @@ defmodule PostHog.EmailsTest do assert [{:post, @uploads_url, _}, {:put, @put_url, _}] = requests() end - @tag config: [email_request_max_retries: 3] + @tag config: [email_request_max_retries: 4] test "retries an upload the storage couldn't take, then sends", %{name: name} do expect_reply(slots([to_upload(@pdf)])) expect_storage({:refused, 503, "SlowDown", "Please reduce your request rate."}) + expect_storage({:refused, 429, "TooManyRequests", "Please slow down."}) expect_storage({:refused, 500, "InternalError", "We encountered an internal error."}) expect_storage(:closed) expect_storage(:stored) @@ -566,6 +567,7 @@ defmodule PostHog.EmailsTest do {:put, _, _}, {:put, _, _}, {:put, _, _}, + {:put, _, _}, {:post, @send_url, _} ] = requests() end @@ -1576,3 +1578,93 @@ defmodule PostHog.EmailsTest do end end end + +defmodule PostHog.EmailsReqDefaultsTest do + # Req.default_options is app-wide, so this runs alone, after the async tests. + use ExUnit.Case, async: false + + import Mox + + alias PostHog.API + alias PostHog.Emails + + setup :verify_on_exit! + setup {Req.Test, :verify_on_exit!} + + setup context do + Mox.stub_with(API.Mock, API.Stub) + previous = Application.fetch_env(:req, :default_options) + Application.put_env(:req, :default_options, auth: {:bearer, "app-wide"}, compress_body: true) + + on_exit(fn -> + case previous do + {:ok, options} -> Application.put_env(:req, :default_options, options) + :error -> Application.delete_env(:req, :default_options) + end + end) + + config = + [ + api_host: "https://us.i.posthog.com", + api_key: "phc_project_key", + secret_key: "phs_secret_key", + api_client_module: API.Mock, + supervisor_name: context.test, + enable_local_evaluation: false + ] + |> PostHog.Config.validate!() + |> Map.put(:email_upload_options, plug: {Req.Test, __MODULE__}) + + start_link_supervised!({PostHog.Supervisor, config}) + %{name: context.test} + end + + test "an upload ignores the app's Req defaults", %{name: name} do + file = "file bytes" + sha256 = :crypto.hash(:sha256, file) |> Base.encode16(case: :lower) + + expect(API.Mock, :request, fn _client, :post, _url, _opts -> + {:ok, + %{ + status: 200, + headers: %{}, + body: %{ + "files" => [ + %{ + "sha256" => sha256, + "status" => "upload", + "url" => "https://files.example.com/a?X-Amz-Signature=signed", + "method" => "PUT", + "headers" => %{"Content-Length" => "10"} + } + ] + } + }} + end) + + expect(API.Mock, :request, fn _client, :post, _url, _opts -> + {:ok, %{status: 202, headers: %{}, body: %{"message_id" => "m"}}} + end) + + test_pid = self() + + Req.Test.expect(__MODULE__, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test_pid, {:stored, conn.req_headers, body}) + Plug.Conn.send_resp(conn, 200, "") + end) + + assert {:ok, _} = + Emails.send(name, + to: "user@example.com", + template_key: "receipt", + subject: "Hi", + html: "

", + attachments: [%{filename: "a.txt", content: file}] + ) + + assert_received {:stored, headers, ^file} + refute List.keymember?(headers, "authorization", 0) + refute List.keymember?(headers, "content-encoding", 0) + end +end From 0cfcb3d554f639c83e9915b9f5ed4c7c9d4756fa Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:46:54 +0000 Subject: [PATCH 21/38] fix(emails): upload with a bare request, and hash files only once they fit Filtering a Req.new/1 request's options left defaults it had already applied, such as an app-wide plug or adapter, and could drop an option one needed. The upload is now a bare Req.Request run on Req.Finch: no Req steps, no Req.default_options, only the bytes and the headers PostHog signed. The size check now runs before hashing, so a file that can't be sent isn't read in full first. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 36 ++++++++++++++++--------------- lib/posthog/emails/attachments.ex | 21 +++++++++--------- test/posthog/emails_test.exs | 23 ++++++++++---------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index f7ce12c6..68982fbb 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -719,7 +719,7 @@ defmodule PostHog.Emails do timeout_ms: config.email_request_timeout_ms, max_retries: config.email_request_max_retries, sleep: Map.get(config, :email_sleep, &Process.sleep/1), - upload_options: Map.get(config, :email_upload_options, []) + upload_adapter: Map.get(config, :email_upload_adapter, {Req.Finch, []}) }} end end @@ -827,27 +827,29 @@ defmodule PostHog.Emails do run_request(connection, timeout_ms, fn -> module.request(client, method, url, options) end) end - # A plain request, not one through the api_client, whose defaults, such as headers a custom client - # adds, are meant for PostHog. Req.new/1 also merges the app's Req.default_options, such as an - # auth or compress_body set for other requests, so the upload keeps only its own options. The - # storage gets the bytes and the headers PostHog signed the upload with, nothing else. + # A bare request: not through the api_client, whose defaults, such as headers a custom client + # adds, are meant for PostHog, and without Req's steps and the app's Req.default_options, such as + # an auth or compress_body set for other requests. The storage gets the bytes and the headers + # PostHog signed the upload with, nothing else. defp send_upload(connection, upload, content, timeout_ms) do - options = - [ + {adapter, adapter_options} = connection.upload_adapter + + request = + Req.Request.new( method: :put, url: upload.url, - headers: upload.headers, + headers: for({name, value} <- upload.headers, do: {String.downcase(name), value}), body: content, - retry: false, - redirect: false, - decode_body: false, - receive_timeout: timeout_ms - ] ++ connection.upload_options - - request = Req.new(options) - request = %{request | options: Map.take(request.options, Keyword.keys(options))} + adapter: adapter, + options: [receive_timeout: timeout_ms] ++ adapter_options + ) - run_request(connection, timeout_ms, fn -> Req.request(request) end) + run_request(connection, timeout_ms, fn -> + case Req.Request.run_request(request) do + {_request, %Req.Response{} = response} -> {:ok, response} + {_request, exception} -> {:error, exception} + end + end) end # Each request checks that the instance still runs, so a send stopped between its steps sends diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 616cf875..9830e822 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -26,9 +26,11 @@ defmodule PostHog.Emails.Attachments do """ @spec prepare(term(), [String.t()]) :: {:ok, [file()]} | {:error, Error.t()} def prepare(attachments, bodies) when is_list(attachments) do + # Hashing reads every byte, so it waits until the files are known to fit. with :ok <- check_count(attachments), - {:ok, files} <- check_each(attachments) do - check_size(files, bodies) + {:ok, files} <- check_each(attachments), + :ok <- check_size(files, bodies) do + {:ok, Enum.map(files, &hash/1)} end end @@ -151,7 +153,7 @@ defmodule PostHog.Emails.Attachments do "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." )} else - {:ok, files} + :ok end end @@ -168,19 +170,13 @@ defmodule PostHog.Emails.Attachments do with :ok <- check_keys(attachment, field), :ok <- check_values(attachment, field) do - sha256 = :crypto.hash(:sha256, attachment.content) |> Base.encode16(case: :lower) - {:ok, %{ field: field, filename: attachment.filename, content: attachment.content, - sha256: sha256, size: byte_size(attachment.content), - reference: - attachment - |> Map.take([:filename, :content_type, :content_id]) - |> Map.put(:sha256, sha256) + reference: Map.take(attachment, [:filename, :content_type, :content_id]) }} end end @@ -188,6 +184,11 @@ defmodule PostHog.Emails.Attachments do defp file(_attachment, field), do: invalid("An attachment is a map with filename and content.", field) + defp hash(file) do + sha256 = :crypto.hash(:sha256, file.content) |> Base.encode16(case: :lower) + %{file | reference: Map.put(file.reference, :sha256, sha256)} |> Map.put(:sha256, sha256) + end + defp check_keys(attachment, field) do unknown = attachment |> Map.keys() |> Enum.reject(&(&1 in @fields)) |> Enum.sort() diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 321c0b86..ea3d952b 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -51,7 +51,7 @@ defmodule PostHog.EmailsTest do send(test_pid, {:slept, ms}) if context[:stop_instance_on_sleep], do: stop_supervised!(context.test) end) - |> Map.put(:email_upload_options, plug: {Req.Test, PostHog.EmailsTest.Storage}) + |> Map.put(:email_upload_adapter, {Req.Plug, plug: {Req.Test, PostHog.EmailsTest.Storage}}) start_link_supervised!({PostHog.Supervisor, config}) %{name: context.test} @@ -392,7 +392,8 @@ defmodule PostHog.EmailsTest do Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> {:ok, body, conn} = Plug.Conn.read_body(conn) url = "#{conn.scheme}://#{conn.host}#{conn.request_path}?#{conn.query_string}" - send(test_pid, {:request, :put, url, [headers: conn.req_headers, body: body]}) + headers = for {name, value} <- conn.req_headers, do: {String.downcase(name), value} + send(test_pid, {:request, :put, url, [headers: headers, body: body]}) case reply do :stored -> @@ -475,11 +476,7 @@ defmodule PostHog.EmailsTest do assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) assert [_ask, {:put, @put_url, put}, _send] = requests() - - # Req's own user-agent and accept-encoding aside, as any HTTP client's. - header_names = - for {key, _value} <- put[:headers], key not in ["user-agent", "accept-encoding"], do: key - + header_names = for {key, _value} <- put[:headers], do: key assert Enum.sort(header_names) == ["content-length", "x-amz-checksum-sha256"] end @@ -1594,7 +1591,12 @@ defmodule PostHog.EmailsReqDefaultsTest do setup context do Mox.stub_with(API.Mock, API.Stub) previous = Application.fetch_env(:req, :default_options) - Application.put_env(:req, :default_options, auth: {:bearer, "app-wide"}, compress_body: true) + + Application.put_env(:req, :default_options, + auth: {:bearer, "app-wide"}, + headers: [{"x-app-wide", "1"}], + plug: {Req.Test, PostHog.EmailsReqDefaultsTest.AppWide} + ) on_exit(fn -> case previous do @@ -1613,7 +1615,7 @@ defmodule PostHog.EmailsReqDefaultsTest do enable_local_evaluation: false ] |> PostHog.Config.validate!() - |> Map.put(:email_upload_options, plug: {Req.Test, __MODULE__}) + |> Map.put(:email_upload_adapter, {Req.Plug, plug: {Req.Test, __MODULE__}}) start_link_supervised!({PostHog.Supervisor, config}) %{name: context.test} @@ -1664,7 +1666,6 @@ defmodule PostHog.EmailsReqDefaultsTest do ) assert_received {:stored, headers, ^file} - refute List.keymember?(headers, "authorization", 0) - refute List.keymember?(headers, "content-encoding", 0) + assert Enum.map(headers, &elem(&1, 0)) == ["content-length"] end end From f059d046fc185eb82c273d115adac818688e628c Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:49:23 +0000 Subject: [PATCH 22/38] fix(emails): run uploads on the installed Req's default adapter Req.Finch as a module adapter came with Req 0.7, and the SDK still supports Req 0.6. The bare upload request now leaves the adapter to Req.Request.new/1, so each Req version uses its own. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 68982fbb..0285f3e3 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -719,7 +719,7 @@ defmodule PostHog.Emails do timeout_ms: config.email_request_timeout_ms, max_retries: config.email_request_max_retries, sleep: Map.get(config, :email_sleep, &Process.sleep/1), - upload_adapter: Map.get(config, :email_upload_adapter, {Req.Finch, []}) + upload_adapter: Map.get(config, :email_upload_adapter) }} end end @@ -832,17 +832,15 @@ defmodule PostHog.Emails do # an auth or compress_body set for other requests. The storage gets the bytes and the headers # PostHog signed the upload with, nothing else. defp send_upload(connection, upload, content, timeout_ms) do - {adapter, adapter_options} = connection.upload_adapter - request = Req.Request.new( method: :put, url: upload.url, headers: for({name, value} <- upload.headers, do: {String.downcase(name), value}), body: content, - adapter: adapter, - options: [receive_timeout: timeout_ms] ++ adapter_options + options: [receive_timeout: timeout_ms] ) + |> with_upload_adapter(connection.upload_adapter) run_request(connection, timeout_ms, fn -> case Req.Request.run_request(request) do @@ -852,6 +850,12 @@ defmodule PostHog.Emails do end) end + # Req's own default adapter, which differs between Req versions, unless a test sets one. + defp with_upload_adapter(request, nil), do: request + + defp with_upload_adapter(request, {adapter, options}), + do: %{request | adapter: adapter, options: Map.merge(request.options, Map.new(options))} + # Each request checks that the instance still runs, so a send stopped between its steps sends # nothing more. defp run_request(connection, timeout_ms, request) do From 1d50bfd2f770db816ef4e751622d945f4550b703 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:52:33 +0000 Subject: [PATCH 23/38] test(emails): upload over a real socket with Req's default adapter Every other attachment test sets an adapter, so the path production takes had no test. This one uploads to a local Bandit server through Emails.send/2 with none set. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 4 ++-- test/posthog/emails_test.exs | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 0285f3e3..56a92959 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -719,7 +719,7 @@ defmodule PostHog.Emails do timeout_ms: config.email_request_timeout_ms, max_retries: config.email_request_max_retries, sleep: Map.get(config, :email_sleep, &Process.sleep/1), - upload_adapter: Map.get(config, :email_upload_adapter) + upload_adapter: Map.get(config, :email_upload_adapter, :none) }} end end @@ -851,7 +851,7 @@ defmodule PostHog.Emails do end # Req's own default adapter, which differs between Req versions, unless a test sets one. - defp with_upload_adapter(request, nil), do: request + defp with_upload_adapter(request, :none), do: request defp with_upload_adapter(request, {adapter, options}), do: %{request | adapter: adapter, options: Map.merge(request.options, Map.new(options))} diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index ea3d952b..d281cb2e 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -15,6 +15,22 @@ defmodule PostHog.EmailsTest.ReqTestClient do defdelegate request(client, method, url, opts), to: Client end +defmodule PostHog.EmailsTest.StorageServer do + @moduledoc false + # A storage host on a real socket, which reports each upload to the test. + @behaviour Plug + + @impl true + def init(test_pid), do: test_pid + + @impl true + def call(conn, test_pid) do + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test_pid, {:stored, conn.method, conn.request_path, conn.req_headers, body}) + Plug.Conn.send_resp(conn, 200, "") + end +end + defmodule PostHog.EmailsTest do use ExUnit.Case, async: true @@ -51,7 +67,10 @@ defmodule PostHog.EmailsTest do send(test_pid, {:slept, ms}) if context[:stop_instance_on_sleep], do: stop_supervised!(context.test) end) - |> Map.put(:email_upload_adapter, {Req.Plug, plug: {Req.Test, PostHog.EmailsTest.Storage}}) + |> Map.put( + :email_upload_adapter, + context[:upload_adapter] || {Req.Plug, plug: {Req.Test, PostHog.EmailsTest.Storage}} + ) start_link_supervised!({PostHog.Supervisor, config}) %{name: context.test} @@ -480,6 +499,27 @@ defmodule PostHog.EmailsTest do assert Enum.sort(header_names) == ["content-length", "x-amz-checksum-sha256"] end + # The installed Req's default adapter, over a real socket, as in production. + @tag upload_adapter: :none + test "uploads over HTTP with Req's default adapter", %{name: name} do + {:ok, server} = + start_supervised( + {Bandit, plug: {PostHog.EmailsTest.StorageServer, self()}, ip: :loopback, port: 0} + ) + + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + url = "http://127.0.0.1:#{port}/team/1/file?X-Amz-Signature=signed" + + expect_reply(slots([Map.put(to_upload(@pdf), "url", url)])) + expect_reply(accepted()) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert_received {:stored, "PUT", "/team/1/file", headers, @pdf} + refute List.keymember?(headers, "authorization", 0) + assert {"x-amz-checksum-sha256", :crypto.hash(:sha256, @pdf) |> Base.encode64()} in headers + end + test "uploads nothing when PostHog has every file", %{name: name} do expect_reply(slots([present(@pdf)])) expect_reply(accepted()) From 0184d8cb7656a43b93e91c14e30b9d36d0e0f7bc Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:28:06 +0000 Subject: [PATCH 24/38] fix(emails): count the subject toward the size limit, and give uploads more time - Count the subject toward the 40 MB estimate as PostHog does, 75 bytes per 42 started, so a long subject next to a large file is refused before any upload. - Refuse a blank attachment filename before any request. - Give an upload ten seconds per MB on top of the request timeout, not one. - Say that uploads don't go through a proxy, in the docs and in the upload error's fix. - Declare Bandit for the real-socket upload test. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 38 ++++++++++--------- lib/posthog/emails/attachments.ex | 31 ++++++++++----- mix.exs | 1 + test/posthog/emails_test.exs | 63 ++++++++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 28 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 56a92959..cbd371e8 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -88,7 +88,8 @@ defmodule PostHog.Emails do ## Attachments An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The - SDK checks both before any request, counting each file and body as Base64, as PostHog does. + SDK checks both before any request, counting each file and body as Base64 and the subject as + encoded, as PostHog does. A send with files takes three steps: @@ -100,11 +101,12 @@ defmodule PostHog.Emails do nothing again. 3. Once every upload succeeded, it sends the email, which names each file by its hash. - Your app must be able to reach the storage host that PostHog's answer names. If an upload - fails, the SDK sends nothing and returns `attachment_upload_failed` with what the storage - answered. An upload gets the request timeout plus one second per MB of the file, and retries - as a send does, as well as when the storage answers HTTP 500, 502 or 504. It ignores your - app's `Req.default_options`, which are meant for other requests. + Your app must be able to reach the storage host that PostHog's answer names directly: an + upload doesn't go through a proxy, and it ignores your app's `Req.default_options`, which are + meant for other requests. If an upload fails, the SDK sends nothing and returns + `attachment_upload_failed` with what the storage answered. An upload gets the request timeout + plus ten seconds per MB of the file, and retries as a send does, as well as when the storage + answers HTTP 500, 502 or 504. ## Timeouts, retries and idempotency @@ -388,8 +390,9 @@ defmodule PostHog.Emails do end) end + # Ten seconds per MB: a file that fits still uploads over a link of about 1 Mbit/s. defp upload_timeout_ms(connection, file), - do: connection.timeout_ms + div(file.size, 1_000_000) * 1_000 + do: connection.timeout_ms + div(file.size, 100) defp put_file(connection, file, upload, max_retries) do timeout_ms = upload_timeout_ms(connection, file) @@ -516,7 +519,8 @@ defmodule PostHog.Emails do defp attachments(options) do Attachments.prepare( Map.get(options, :attachments, []), - Enum.flat_map([:html, :text], &List.wrap(options[&1])) + Enum.flat_map([:html, :text], &List.wrap(options[&1])), + Map.get(options, :subject) || "" ) end @@ -803,15 +807,9 @@ defmodule PostHog.Emails do # Requests defp request(connection, method, path, headers, body) do + %{module: module, client: client} = connection.api_client url = connection.app_host <> path - case send_request(connection, method, url, headers, body, connection.timeout_ms) do - {:ok, status, body, retry_after_ms} -> {:ok, status, fields(body), retry_after_ms} - {:error, error} -> {:error, error} - end - end - - defp send_request(connection, method, url, headers, body, timeout_ms) do # The SDK parses the body itself, so a malformed acceptance stays an api_error with its status. options = [ @@ -820,11 +818,15 @@ defmodule PostHog.Emails do redirect: false, compress_body: false, decode_body: false, - receive_timeout: timeout_ms + receive_timeout: connection.timeout_ms ] ++ if(body, do: [body: body], else: []) - %{module: module, client: client} = connection.api_client - run_request(connection, timeout_ms, fn -> module.request(client, method, url, options) end) + case run_request(connection, connection.timeout_ms, fn -> + module.request(client, method, url, options) + end) do + {:ok, status, body, retry_after_ms} -> {:ok, status, fields(body), retry_after_ms} + {:error, error} -> {:error, error} + end end # A bare request: not through the api_client, whose defaults, such as headers a custom client diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 9830e822..c88ed012 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -6,6 +6,8 @@ defmodule PostHog.Emails.Attachments do @max_files 10 @max_message_bytes 40_000_000 @headers_and_mime_boundaries_bytes 64 * 1024 + @subject_bytes_per_encoded_word 42 + @folded_encoded_word_bytes 75 @limit "10 files and 40 MB per email after Base64 encoding, about 29 MB of files" @fields [:filename, :content, :content_type, :content_id] @field_names Enum.map(@fields, &Atom.to_string/1) @@ -22,19 +24,20 @@ defmodule PostHog.Emails.Attachments do @type upload() :: %{url: String.t(), headers: [{String.t(), String.t()}]} @doc """ - Checks the files and the size of the email they make with `bodies`, and hashes each file. + Checks the files and the size of the email they make with `bodies` and `subject`, and hashes + each file. """ - @spec prepare(term(), [String.t()]) :: {:ok, [file()]} | {:error, Error.t()} - def prepare(attachments, bodies) when is_list(attachments) do + @spec prepare(term(), [String.t()], String.t()) :: {:ok, [file()]} | {:error, Error.t()} + def prepare(attachments, bodies, subject) when is_list(attachments) do # Hashing reads every byte, so it waits until the files are known to fit. with :ok <- check_count(attachments), {:ok, files} <- check_each(attachments), - :ok <- check_size(files, bodies) do + :ok <- check_size(files, bodies, subject) do {:ok, Enum.map(files, &hash/1)} end end - def prepare(_attachments, _bodies), + def prepare(_attachments, _bodies, _subject), do: invalid("attachments must be a list of files.", "attachments") @doc "The body of the upload request: each distinct file once." @@ -106,7 +109,7 @@ defmodule PostHog.Emails.Attachments do why: "The SDK sends an email only once each of its files is uploaded, so no email was sent.", fix: - "Send again. If it keeps failing, check that your app can reach #{host}, and that no proxy changes the upload." + "Send again. If it keeps failing, check that your app can reach #{host} directly: uploads don't go through a proxy or your API client's settings." ) end @@ -138,17 +141,17 @@ defmodule PostHog.Emails.Attachments do end end - defp check_size(files, bodies) do + defp check_size(files, bodies, subject) do sizes = Enum.map(files, & &1.size) ++ Enum.map(bodies, &byte_size/1) - if base64_message_bytes(sizes) > @max_message_bytes do + if base64_message_bytes(sizes) + encoded_subject_bytes(subject) > @max_message_bytes do {:error, Error.new( "message_too_large", "These files make the email too large to send, so no email was sent.", field: "attachments", why: - "PostHog sends emails up to 40 MB, and Base64 encoding makes each file and body about a third larger.", + "PostHog sends emails up to 40 MB, and Base64 encoding makes each file and body about a third larger, as encoding makes the subject.", fix: "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." )} @@ -163,6 +166,10 @@ defmodule PostHog.Emails.Attachments do end) end + # A subject is sent as encoded words of 42 bytes each, folded onto lines of 75. + defp encoded_subject_bytes(subject), + do: ceil_div(byte_size(subject), @subject_bytes_per_encoded_word) * @folded_encoded_word_bytes + defp ceil_div(dividend, divisor), do: div(dividend + divisor - 1, divisor) defp file(attachment, field) when is_map(attachment) and not is_struct(attachment) do @@ -213,6 +220,12 @@ defmodule PostHog.Emails.Attachments do not is_binary(attachment[:filename]) -> invalid("An attachment needs its filename as a string.", field <> "__filename") + String.trim(attachment.filename) == "" -> + invalid( + "An attachment's filename is empty, so the recipient couldn't tell what the file is.", + field <> "__filename" + ) + not is_binary(attachment[:content]) -> invalid( "An attachment needs content: the file's bytes as a binary.", diff --git a/mix.exs b/mix.exs index 12187579..b7a01b47 100644 --- a/mix.exs +++ b/mix.exs @@ -82,6 +82,7 @@ defmodule PostHog.MixProject do # Development tools {:ex_doc, "~> 0.37", only: :dev, runtime: false}, {:logger_handler_kit, "~> 0.4", only: :test}, + {:bandit, "~> 1.7", only: :test}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:mox, "~> 1.1", only: :test} ] diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index d281cb2e..9feb6d95 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -609,6 +609,31 @@ defmodule PostHog.EmailsTest do ] = requests() end + @tag config: [email_request_timeout_ms: 50] + test "gives an upload ten seconds per MB on top of the request timeout", %{name: name} do + file = :binary.copy(<<1>>, 1_000_000) + test_pid = self() + + expect_reply(slots([to_upload(file)])) + + Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> + {:ok, _body, conn} = Plug.Conn.read_body(conn, length: 2_000_000) + send(test_pid, {:request, :put, nil, []}) + Process.sleep(1_500) + Plug.Conn.send_resp(conn, 200, "") + end) + + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send( + name, + with_files([%{filename: "a.bin", content: file}]) ++ [max_retries: 0] + ) + + assert [{:post, @uploads_url, _}, {:put, _, _}, {:post, @send_url, _}] = requests() + end + test "doesn't retry an upload the storage refused", %{name: name} do expect_reply(slots([to_upload(@pdf)])) expect_storage({:refused, 403, "AccessDenied", "Request has expired."}) @@ -643,6 +668,7 @@ defmodule PostHog.EmailsTest do ) assert error.message =~ "files.example.com" + assert error.fix =~ "reach files.example.com directly" assert error.field == "attachments__0" refute inspect(error) =~ "signed-secret" assert [{:post, _, _}, {:put, _, _}] = requests() @@ -818,7 +844,7 @@ defmodule PostHog.EmailsTest do end test "refuses an email over 40 MB after Base64 encoding before any request", %{name: name} do - largest_that_fits = 29_182_875 + largest_that_fits = 29_182_821 file = :binary.copy(<<0>>, largest_that_fits) expect_reply(slots([present(file)])) @@ -848,6 +874,39 @@ defmodule PostHog.EmailsTest do refute_requested() end + # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the + # 40 MB estimate free, and 29_182_876 none: PostHog counts a subject as 75 bytes per 42 + # started. + for {label, subject, fits, refused} <- [ + {"of 42 ASCII bytes counts 75", String.duplicate("a", 42), 29_182_821, 29_182_824}, + {"of 43 bytes counts 150", String.duplicate("a", 43), 29_182_767, 29_182_770}, + {"of 21 two-byte characters counts 75", String.duplicate("é", 21), 29_182_821, + 29_182_824}, + {"that is empty counts nothing", "", 29_182_875, 29_182_876} + ] do + test "a subject #{label} toward the 40 MB estimate", %{name: name} do + send_file = fn size -> + Emails.send( + name, + Keyword.merge(@code_send, + subject: unquote(subject), + html: "", + attachments: [%{filename: "a", content: :binary.copy(<<0>>, size)}] + ) + ) + end + + assert {:error, %Error{code: "message_too_large"}} = send_file.(unquote(refused)) + refute_requested() + + file = :binary.copy(<<0>>, unquote(fits)) + expect_reply(slots([present(file)])) + expect_reply(accepted()) + + assert {:ok, _} = send_file.(unquote(fits)) + end + end + test "refuses an attachment it can't send, naming the field", %{name: name} do for {attachments, field} <- [ {%{filename: "a.txt", content: "a"}, "attachments"}, @@ -862,6 +921,8 @@ defmodule PostHog.EmailsTest do {[%{filename: "a.txt", content: [97]}], "attachments__0__content"}, {[%{filename: "a.txt", content: <<1::3>>}], "attachments__0__content"}, {[%{content: "a"}], "attachments__0__filename"}, + {[%{filename: "", content: "a"}], "attachments__0__filename"}, + {[%{filename: " ", content: "a"}], "attachments__0__filename"}, {[%{filename: :a, content: "a"}], "attachments__0__filename"}, {[%{filename: "a.txt", content: "a", content_type: :pdf}], "attachments__0__content_type"}, From 691d833b2d0a4a4ebddab5fc652af0f3a175d5f0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:34:31 +0000 Subject: [PATCH 25/38] fix(emails): leave the size of an email without files to PostHog - Check the 40 MB estimate only for a send with files: without files nothing uploads first, and the refusal named attachments the caller never passed. - Tighten the upload-timeout test to 100 KB, so a lower allowance per MB fails it. - List every storage status an upload retries, and say how the subject counts. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 8 ++++---- lib/posthog/emails/attachments.ex | 6 ++++-- test/posthog/emails_test.exs | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index cbd371e8..b4985e8b 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -88,8 +88,8 @@ defmodule PostHog.Emails do ## Attachments An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The - SDK checks both before any request, counting each file and body as Base64 and the subject as - encoded, as PostHog does. + SDK checks both before any request of a send with files, counting each file and body as Base64 + and the subject as 75 bytes per 42 started, as PostHog does. A send with files takes three steps: @@ -105,8 +105,8 @@ defmodule PostHog.Emails do upload doesn't go through a proxy, and it ignores your app's `Req.default_options`, which are meant for other requests. If an upload fails, the SDK sends nothing and returns `attachment_upload_failed` with what the storage answered. An upload gets the request timeout - plus ten seconds per MB of the file, and retries as a send does, as well as when the storage - answers HTTP 500, 502 or 504. + plus ten seconds per MB of the file. It retries network errors and storage answers of HTTP 429, + 500, 502, 503 and 504, as a send retries, honouring `Retry-After`. ## Timeouts, retries and idempotency diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index c88ed012..7a9b0455 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -25,7 +25,7 @@ defmodule PostHog.Emails.Attachments do @doc """ Checks the files and the size of the email they make with `bodies` and `subject`, and hashes - each file. + each file. An email without files is left to PostHog to size, as nothing is uploaded first. """ @spec prepare(term(), [String.t()], String.t()) :: {:ok, [file()]} | {:error, Error.t()} def prepare(attachments, bodies, subject) when is_list(attachments) do @@ -141,6 +141,8 @@ defmodule PostHog.Emails.Attachments do end end + defp check_size([], _bodies, _subject), do: :ok + defp check_size(files, bodies, subject) do sizes = Enum.map(files, & &1.size) ++ Enum.map(bodies, &byte_size/1) @@ -151,7 +153,7 @@ defmodule PostHog.Emails.Attachments do "These files make the email too large to send, so no email was sent.", field: "attachments", why: - "PostHog sends emails up to 40 MB, and Base64 encoding makes each file and body about a third larger, as encoding makes the subject.", + "PostHog sends emails up to 40 MB. Base64 makes each file and body about a third larger, and the subject counts 75 bytes per 42 bytes started.", fix: "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." )} diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 9feb6d95..376d15a4 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -611,15 +611,16 @@ defmodule PostHog.EmailsTest do @tag config: [email_request_timeout_ms: 50] test "gives an upload ten seconds per MB on top of the request timeout", %{name: name} do - file = :binary.copy(<<1>>, 1_000_000) + file = :binary.copy(<<1>>, 100_000) test_pid = self() expect_reply(slots([to_upload(file)])) Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> - {:ok, _body, conn} = Plug.Conn.read_body(conn, length: 2_000_000) + {:ok, _body, conn} = Plug.Conn.read_body(conn) send(test_pid, {:request, :put, nil, []}) - Process.sleep(1_500) + # 100 KB at ten seconds per MB is 1 s on top of the 50 ms request timeout. + Process.sleep(950) Plug.Conn.send_resp(conn, 200, "") end) @@ -874,6 +875,15 @@ defmodule PostHog.EmailsTest do refute_requested() end + test "leaves the size of an email without files to PostHog", %{name: name} do + expect_reply(accepted()) + + assert {:ok, _} = + Emails.send(name, Keyword.put(@code_send, :html, :binary.copy("a", 40_000_000))) + + assert [{:post, @send_url, _}] = requests() + end + # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the # 40 MB estimate free, and 29_182_876 none: PostHog counts a subject as 75 bytes per 42 # started. From 0a558e1d232a208c6e278226ba449f55b543fea0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:37:06 +0000 Subject: [PATCH 26/38] test(emails): check the whole body reaches PostHog when a send has no files Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 376d15a4..04ed595f 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -876,12 +876,14 @@ defmodule PostHog.EmailsTest do end test "leaves the size of an email without files to PostHog", %{name: name} do + html = :binary.copy("a", 40_000_000) expect_reply(accepted()) - assert {:ok, _} = - Emails.send(name, Keyword.put(@code_send, :html, :binary.copy("a", 40_000_000))) + assert {:ok, %Emails.SendResult{id: @message_id}} = + Emails.send(name, Keyword.put(@code_send, :html, html)) - assert [{:post, @send_url, _}] = requests() + assert %{"html" => ^html, "subject" => "Hi"} = sent_body() + refute_requested() end # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the From 75d83f85577a95b147de80433ea201065d884fc9 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:40:06 +0000 Subject: [PATCH 27/38] test(emails): send the email without files to a real socket, through the real client Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 46 ++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 04ed595f..de2d9095 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -31,6 +31,25 @@ defmodule PostHog.EmailsTest.StorageServer do end end +defmodule PostHog.EmailsTest.ApiServer do + @moduledoc false + # PostHog's send route on a real socket: reports what it received and accepts the email. + @behaviour Plug + + @impl true + def init(test_pid), do: test_pid + + @impl true + def call(conn, test_pid) do + {:ok, body, conn} = Plug.Conn.read_body(conn, length: 64_000_000) + send(test_pid, {:received, conn.method, conn.request_path, Jason.decode!(body)}) + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(202, ~s({"message_id": "01a18356-c56c-0001-e494-7666bc915188"})) + end +end + defmodule PostHog.EmailsTest do use ExUnit.Case, async: true @@ -875,15 +894,34 @@ defmodule PostHog.EmailsTest do refute_requested() end - test "leaves the size of an email without files to PostHog", %{name: name} do + test "leaves the size of an email without files to PostHog", %{test: test} do + {:ok, server} = + start_supervised( + {Bandit, plug: {PostHog.EmailsTest.ApiServer, self()}, ip: :loopback, port: 0} + ) + + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + host = "http://127.0.0.1:#{port}" + name = Module.concat(test, RealClient) + + config = + PostHog.Config.validate!( + api_host: host, + app_host: host, + api_key: "phc_project_key", + secret_key: "phs_secret_key", + supervisor_name: name, + enable_local_evaluation: false + ) + + start_link_supervised!(Supervisor.child_spec({PostHog.Supervisor, config}, id: name)) html = :binary.copy("a", 40_000_000) - expect_reply(accepted()) assert {:ok, %Emails.SendResult{id: @message_id}} = Emails.send(name, Keyword.put(@code_send, :html, html)) - assert %{"html" => ^html, "subject" => "Hi"} = sent_body() - refute_requested() + assert_received {:received, "POST", "/api/projects/@current/messaging/send/email/", + %{"html" => ^html, "subject" => "Hi"}} end # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the From b041927e2827f5edb3b8968c0d4e5a134c438048 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:55:44 +0000 Subject: [PATCH 28/38] test(emails): drive cc, bcc and attachments through a behaving fake PostHog The upload route signs an upload for each file PostHog lacks, the storage keeps a file only when its PUT matches what was signed, and the send route accepts only references to stored files. The tests assert the caller's result, the stored files, the accepted email and the headers the storage got, not the mock's arguments. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 877 ++++++++++++++++++++++------------- 1 file changed, 546 insertions(+), 331 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index de2d9095..90713017 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -15,39 +15,316 @@ defmodule PostHog.EmailsTest.ReqTestClient do defdelegate request(client, method, url, opts), to: Client end -defmodule PostHog.EmailsTest.StorageServer do +defmodule PostHog.EmailsTest.FakePostHog do @moduledoc false - # A storage host on a real socket, which reports each upload to the test. - @behaviour Plug + # PostHog's email routes and its file storage as they behave, behind the SDK's real requests. + # The upload route signs an upload for each file PostHog doesn't have, the storage keeps a file + # only when its upload matches what was signed, as S3 checks a presigned PUT, and the send route + # accepts an email only when PostHog has every file it names. A test reads back the stored + # files, the accepted emails and what each route received, and can make a route's next request + # fail, wait or hold. + # + # The routes are :uploads and :send on PostHog, and :storage. Uploads are signed for + # https://files.example.com unless set_storage_url/2 names another host. + + @secret_key "phs_secret_key" + @uploads_path "/api/projects/@current/messaging/attachments/uploads/" + @send_path "/api/projects/@current/messaging/send/email/" + @storage_url "https://files.example.com" + @max_body 64_000_000 + + defmodule Server do + @moduledoc false + # The fake as a plug, to serve PostHog and the storage on a real socket. + @behaviour Plug + + alias PostHog.EmailsTest.FakePostHog + + @impl true + def init(fake), do: fake + + @impl true + def call(conn, fake), do: FakePostHog.serve(conn, fake) + end - @impl true - def init(test_pid), do: test_pid + @doc "Starts a fake for the calling test and routes the test's PostHog and storage requests to it." + def start! do + state = %{ + test: self(), + storage_url: @storage_url, + files: %{}, + signed: %{}, + messages: [], + received: %{uploads: [], storage: [], send: []}, + next: %{uploads: [], storage: [], send: []} + } + + fake = + ExUnit.Callbacks.start_supervised!(%{ + id: __MODULE__, + start: {Agent, :start_link, [fn -> state end]} + }) + + Req.Test.stub(PostHog.EmailsTest.ReqTestClient, &serve(&1, fake)) + Req.Test.stub(PostHog.EmailsTest.Storage, &serve(&1, fake)) + fake + end - @impl true - def call(conn, test_pid) do - {:ok, body, conn} = Plug.Conn.read_body(conn) - send(test_pid, {:stored, conn.method, conn.request_path, conn.req_headers, body}) - Plug.Conn.send_resp(conn, 200, "") + @doc "Stores files, as an earlier email that uploaded them would have." + def put_files(fake, files) do + Agent.update(fake, fn state -> + %{state | files: Map.merge(state.files, Map.new(files, &{sha256(&1), &1}))} + end) end -end -defmodule PostHog.EmailsTest.ApiServer do - @moduledoc false - # PostHog's send route on a real socket: reports what it received and accepts the email. - @behaviour Plug + @doc "Deletes a stored file, as when it expires." + def delete_file(fake, bytes), + do: Agent.update(fake, &%{&1 | files: Map.delete(&1.files, sha256(bytes))}) + + @doc "Signs later uploads for the storage at `url`." + def set_storage_url(fake, url), do: Agent.update(fake, &%{&1 | storage_url: url}) + + @doc """ + Answers the route's next request with `reply` instead: `{status, json}` on PostHog's routes, + and `{status, s3_code, s3_message}` or a transport error's reason, such as `:econnrefused`, on + the storage. + """ + def fail_next(fake, route, reply), do: script(fake, route, {:reply, reply}) + + @doc "Makes the route take `ms` longer to answer its next request." + def delay_next(fake, route, ms), do: script(fake, route, {:delay, ms}) + + @doc """ + Holds the route's next request before it answers: the test gets `{:held, route, pid}`, and the + route answers once the test sends `:release` to `pid`. + """ + def hold_next(fake, route), do: script(fake, route, :hold) + + @doc "The stored files, by their hex SHA-256." + def files(fake), do: Agent.get(fake, & &1.files) + + @doc "The accepted emails as `{message_id, email}`, oldest first." + def messages(fake), do: Agent.get(fake, &Enum.reverse(&1.messages)) + + @doc """ + What the route received, oldest first: the JSON body on PostHog's routes, and the method, + path, headers and body on the storage. + """ + def received(fake, route), do: Agent.get(fake, &Enum.reverse(&1.received[route])) + + @doc "Whether any route received a request." + def received_any?(fake), + do: Agent.get(fake, fn state -> Enum.any?(state.received, &(elem(&1, 1) != [])) end) + + @doc "The answer PostHog gives for a file it doesn't have, signed for the default storage." + def upload_slot(bytes), do: upload_slot(@storage_url, sha256(bytes), byte_size(bytes)) + + defp upload_slot(storage_url, sha256, size) do + %{ + "sha256" => sha256, + "status" => "upload", + "url" => "#{storage_url}/team/1/#{sha256}?X-Amz-Signature=signed-secret", + "method" => "PUT", + "headers" => %{ + "Content-Length" => to_string(size), + "x-amz-checksum-sha256" => checksum(sha256) + }, + "url_expires_at" => "2026-11-09T10:15:00Z" + } + end - @impl true - def init(test_pid), do: test_pid + @doc false + def serve(conn, fake) do + case conn.request_path do + @uploads_path -> posthog(conn, :uploads, fake) + @send_path -> posthog(conn, :send, fake) + "/api/" <> _other -> json(conn, 404, refusal("not_found", "Not found.", nil)) + _storage -> storage(conn, fake) + end + end - @impl true - def call(conn, test_pid) do - {:ok, body, conn} = Plug.Conn.read_body(conn, length: 64_000_000) - send(test_pid, {:received, conn.method, conn.request_path, Jason.decode!(body)}) + defp script(fake, route, step), + do: Agent.update(fake, &update_in(&1.next[route], fn steps -> steps ++ [step] end)) + + # Records what the route received and takes its next scripted step, if any. + defp receive_request(fake, route, request) do + Agent.get_and_update(fake, fn state -> + {step, steps} = + case state.next[route] do + [step | steps] -> {step, steps} + [] -> {:answer, []} + end + + state = update_in(state.received[route], &[request | &1]) + {{step, state.test}, put_in(state.next[route], steps)} + end) + end + + defp run_step({:delay, ms}, _route, _test), do: Process.sleep(ms) + + defp run_step(:hold, route, test) do + send(test, {:held, route, self()}) + + receive do + :release -> :ok + end + end + + defp run_step(:answer, _route, _test), do: :ok + + # PostHog + + defp posthog(conn, route, fake) do + {:ok, body, conn} = Plug.Conn.read_body(conn, length: @max_body) + request = Jason.decode!(body) + + case receive_request(fake, route, request) do + {{:reply, {status, reply}}, _test} -> + json(conn, status, reply) + + {step, test} -> + run_step(step, route, test) + answer(conn, route, request, fake) + end + end + + defp answer(conn, route, request, fake) do + cond do + Plug.Conn.get_req_header(conn, "authorization") != ["Bearer #{@secret_key}"] -> + json(conn, 401, refusal("authentication_failed", "Invalid secret API key.", nil)) + + route == :uploads -> + sign_uploads(conn, request, fake) + + true -> + accept(conn, request, fake) + end + end + defp sign_uploads(conn, %{"files" => files}, fake) when is_list(files) do + answers = + Agent.get_and_update( + fake, + &Enum.map_reduce(files, &1, fn file, state -> sign(file, state) end) + ) + + json(conn, 200, %{"files" => answers}) + end + + defp sign_uploads(conn, _request, _fake), + do: json(conn, 400, refusal("required", "This field is required.", "files")) + + defp sign(%{"sha256" => sha256, "size" => size}, state) do + if Map.has_key?(state.files, sha256) do + {%{"sha256" => sha256, "status" => "present", "expires_at" => "2026-12-16T09:00:00Z"}, + state} + else + slot = upload_slot(state.storage_url, sha256, size) + {slot, put_in(state.signed[URI.parse(slot["url"]).path], %{sha256: sha256, size: size})} + end + end + + defp accept(conn, email, fake) do + files = files(fake) + + missing = + email + |> Map.get("attachments", []) + |> Enum.find_index(&(not Map.has_key?(files, &1["sha256"]))) + + if missing do + json( + conn, + 400, + refusal( + "attachment_not_uploaded", + "A file this email names isn't uploaded, so PostHog didn't send the email.", + "attachments__#{missing}__sha256" + ) + ) + else + json(conn, 202, %{"message_id" => record_message(fake, email)}) + end + end + + defp record_message(fake, email) do + Agent.get_and_update(fake, fn state -> + number = state.messages |> length() |> Kernel.+(1) |> Integer.to_string() + id = "01a18356-c56c-0001-e494-" <> String.pad_leading(number, 12, "0") + {id, %{state | messages: [{id, email} | state.messages]}} + end) + end + + defp refusal(code, detail, attr), + do: %{"type" => "validation_error", "code" => code, "detail" => detail, "attr" => attr} + + defp json(conn, status, body) do conn |> Plug.Conn.put_resp_content_type("application/json") - |> Plug.Conn.send_resp(202, ~s({"message_id": "01a18356-c56c-0001-e494-7666bc915188"})) + |> Plug.Conn.send_resp(status, Jason.encode!(body)) end + + # Storage + + defp storage(conn, fake) do + {:ok, body, conn} = Plug.Conn.read_body(conn, length: @max_body) + headers = for {name, value} <- conn.req_headers, do: {String.downcase(name), value} + request = %{method: conn.method, path: conn.request_path, headers: headers, body: body} + + case receive_request(fake, :storage, request) do + {{:reply, {status, code, message}}, _test} -> + s3_error(conn, status, code, message) + + {{:reply, reason}, _test} -> + Req.Test.transport_error(conn, reason) + + {step, test} -> + run_step(step, :storage, test) + store(conn, request, fake) + end + end + + defp store(conn, request, fake) do + signed = Agent.get(fake, & &1.signed[request.path]) + + cond do + request.method != "PUT" or signed == nil -> + s3_error(conn, 403, "AccessDenied", "Access Denied.") + + List.keymember?(request.headers, "authorization", 0) -> + s3_error(conn, 400, "InvalidArgument", "Only one auth mechanism allowed.") + + header(request, "x-amz-checksum-sha256") != checksum(signed.sha256) or + header(request, "content-length") != to_string(signed.size) -> + s3_error(conn, 403, "SignatureDoesNotMatch", "The request signature does not match.") + + Base.encode64(:crypto.hash(:sha256, request.body)) != checksum(signed.sha256) -> + s3_error(conn, 400, "BadDigest", "The SHA256 you specified did not match.") + + true -> + Agent.update(fake, &put_in(&1.files[signed.sha256], request.body)) + Plug.Conn.send_resp(conn, 200, "") + end + end + + defp header(request, name) do + case List.keyfind(request.headers, name, 0) do + {^name, value} -> value + nil -> nil + end + end + + defp s3_error(conn, status, code, message) do + conn + |> Plug.Conn.put_resp_content_type("application/xml") + |> Plug.Conn.send_resp( + status, + ~s(\n#{code}#{message}/team/1/file) + ) + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + defp checksum(sha256), do: sha256 |> Base.decode16!(case: :lower) |> Base.encode64() end defmodule PostHog.EmailsTest do @@ -59,11 +336,11 @@ defmodule PostHog.EmailsTest do alias PostHog.API alias PostHog.Emails alias PostHog.Emails.Error + alias PostHog.EmailsTest.FakePostHog setup :verify_on_exit! @send_url "https://us.posthog.com/api/projects/@current/messaging/send/email/" - @uploads_url "https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/" @template_id "0190F6D2-7C1E-7B3A-9F5E-1C2D3E4F5A6B" @message_id "01a18356-c56c-0001-e494-7666bc915188" @@ -322,41 +599,42 @@ defmodule PostHog.EmailsTest do end describe "send/2 with cc and bcc" do + @describetag config: [api_client_module: PostHog.EmailsTest.ReqTestClient] @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] - test "posts copies, a string as a one-item list", %{name: name} do - expect_reply(accepted()) + setup do + %{fake: FakePostHog.start!()} + end - assert {:ok, %Emails.SendResult{id: @message_id}} = + test "PostHog gets the copies, a string as a one-item list", %{name: name, fake: fake} do + assert {:ok, %Emails.SendResult{id: id}} = Emails.send( name, @code_send ++ [cc: "finance@example.com", bcc: ["audit@example.com", "archive@example.com"]] ) - body = sent_body() - assert body["cc"] == ["finance@example.com"] - assert body["bcc"] == ["audit@example.com", "archive@example.com"] + assert [{^id, email}] = FakePostHog.messages(fake) + assert email["cc"] == ["finance@example.com"] + assert email["bcc"] == ["audit@example.com", "archive@example.com"] end - test "leaves out empty lists", %{name: name} do - expect_reply(accepted()) - + test "leaves out empty lists, and uploads nothing for no files", %{name: name, fake: fake} do assert {:ok, _} = Emails.send( name, @code_send ++ [reply_to: [], cc: [], bcc: nil, attachments: []] ) - body = sent_body() - refute Map.has_key?(body, "reply_to") - refute Map.has_key?(body, "cc") - refute Map.has_key?(body, "bcc") - refute Map.has_key?(body, "attachments") - refute_requested() + assert [{_id, email}] = FakePostHog.messages(fake) + refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(email, &1)) + assert FakePostHog.received(fake, :uploads) == [] end - test "a template send refuses them and attachments before any request", %{name: name} do + test "a template send refuses them and attachments before any request", %{ + name: name, + fake: fake + } do for {field, value} <- [ cc: "finance@example.com", bcc: "audit@example.com", @@ -374,10 +652,10 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "template_key" end - refute_requested() + refute FakePostHog.received_any?(fake) end - test "refuses cc and bcc that are not addresses", %{name: name} do + test "refuses cc and bcc that are not addresses", %{name: name, fake: fake} do for {field, value} <- [cc: [:nope], bcc: " ", cc: %{a: 1}] do assert {:error, %Error{code: "invalid_form", field: field_name}} = Emails.send(name, @code_send ++ [{field, value}]) @@ -385,88 +663,31 @@ defmodule PostHog.EmailsTest do assert field_name == to_string(field) end - refute_requested() + refute FakePostHog.received_any?(fake) end end describe "send/2 with attachments" do + @describetag config: [api_client_module: PostHog.EmailsTest.ReqTestClient] @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] @pdf <<37, 80, 68, 70, 45, 0, 255, 10>> @logo "png bytes" - @put_url "https://files.example.com/team/1/file?X-Amz-Signature=signed-secret" - - defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) - defp present(bytes), - do: %{ - "sha256" => sha256(bytes), - "status" => "present", - "expires_at" => "2026-12-16T09:00:00Z" - } - - defp to_upload(bytes) do - %{ - "sha256" => sha256(bytes), - "status" => "upload", - "url" => @put_url, - "method" => "PUT", - "headers" => %{ - "Content-Length" => to_string(byte_size(bytes)), - "x-amz-checksum-sha256" => :crypto.hash(:sha256, bytes) |> Base.encode64() - }, - "url_expires_at" => "2026-11-09T10:15:00Z" - } - end - - setup {Req.Test, :verify_on_exit!} - - defp slots(answers), do: {:ok, %{status: 200, headers: %{}, body: %{"files" => answers}}} - - # The storage, behind the SDK's real Req request: :stored, {:refused, status, code, message}, - # or a transport error's reason. - defp expect_storage(reply) do - test_pid = self() - - Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> - {:ok, body, conn} = Plug.Conn.read_body(conn) - url = "#{conn.scheme}://#{conn.host}#{conn.request_path}?#{conn.query_string}" - headers = for {name, value} <- conn.req_headers, do: {String.downcase(name), value} - send(test_pid, {:request, :put, url, [headers: headers, body: body]}) - - case reply do - :stored -> - Plug.Conn.send_resp(conn, 200, "") - - {:refused, status, code, message} -> - conn - |> Plug.Conn.put_resp_content_type("application/xml") - |> Plug.Conn.send_resp( - status, - ~s(\n#{code}#{message}/team/1/file) - ) - - reason -> - Req.Test.transport_error(conn, reason) - end - end) + setup do + %{fake: FakePostHog.start!()} end - defp requests do - receive do - {:request, method, url, opts} -> [{method, url, opts} | requests()] - after - 0 -> [] - end - end + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) defp with_files(attachments), do: @code_send ++ [attachments: attachments] - test "uploads the files PostHog asks for, then sends their references", %{name: name} do - expect_reply(slots([to_upload(@pdf), present(@logo)])) - expect_storage(:stored) - expect_reply(accepted()) + test "uploads the files PostHog asks for, then sends their references", %{ + name: name, + fake: fake + } do + FakePostHog.put_files(fake, [@logo]) - assert {:ok, %Emails.SendResult{id: @message_id}} = + assert {:ok, %Emails.SendResult{id: id}} = Emails.send( name, with_files([ @@ -475,28 +696,10 @@ defmodule PostHog.EmailsTest do ]) ) - assert [ - {:post, @uploads_url, ask}, - {:put, @put_url, put}, - {:post, @send_url, send} - ] = requests() - - assert Jason.decode!(ask[:body]) == %{ - "files" => [ - %{"sha256" => sha256(@pdf), "size" => byte_size(@pdf)}, - %{"sha256" => sha256(@logo), "size" => byte_size(@logo)} - ] - } - - assert header(ask, "authorization") == "Bearer phs_secret_key" + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf, sha256(@logo) => @logo} + assert [{^id, email}] = FakePostHog.messages(fake) - assert put[:body] == @pdf - assert header(put, "content-length") == "8" - - assert header(put, "x-amz-checksum-sha256") == - :crypto.hash(:sha256, @pdf) |> Base.encode64() - - assert Jason.decode!(send[:body])["attachments"] == [ + assert email["attachments"] == [ %{ "sha256" => sha256(@pdf), "filename" => "invoice.pdf", @@ -506,53 +709,52 @@ defmodule PostHog.EmailsTest do ] end - test "uploads with only the headers PostHog gave, not the client's", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage(:stored) - expect_reply(accepted()) - + test "the storage gets only the headers PostHog signed, not the client's", %{ + name: name, + fake: fake + } do assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [_ask, {:put, @put_url, put}, _send] = requests() - header_names = for {key, _value} <- put[:headers], do: key - assert Enum.sort(header_names) == ["content-length", "x-amz-checksum-sha256"] + assert [%{headers: headers}] = FakePostHog.received(fake, :storage) + + assert headers |> Enum.map(&elem(&1, 0)) |> Enum.sort() == [ + "content-length", + "x-amz-checksum-sha256" + ] end # The installed Req's default adapter, over a real socket, as in production. @tag upload_adapter: :none - test "uploads over HTTP with Req's default adapter", %{name: name} do + test "uploads over HTTP with Req's default adapter", %{name: name, fake: fake} do {:ok, server} = - start_supervised( - {Bandit, plug: {PostHog.EmailsTest.StorageServer, self()}, ip: :loopback, port: 0} - ) + start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) {:ok, {_ip, port}} = ThousandIsland.listener_info(server) - url = "http://127.0.0.1:#{port}/team/1/file?X-Amz-Signature=signed" - - expect_reply(slots([Map.put(to_upload(@pdf), "url", url)])) - expect_reply(accepted()) + FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{port}") - assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert_received {:stored, "PUT", "/team/1/file", headers, @pdf} + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} + assert [{^id, _email}] = FakePostHog.messages(fake) + assert [%{headers: headers}] = FakePostHog.received(fake, :storage) refute List.keymember?(headers, "authorization", 0) - assert {"x-amz-checksum-sha256", :crypto.hash(:sha256, @pdf) |> Base.encode64()} in headers end - test "uploads nothing when PostHog has every file", %{name: name} do - expect_reply(slots([present(@pdf)])) - expect_reply(accepted()) + test "uploads nothing when PostHog has every file", %{name: name, fake: fake} do + FakePostHog.put_files(fake, [@pdf]) assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [{:post, @uploads_url, _}, {:post, @send_url, _}] = requests() + assert FakePostHog.received(fake, :storage) == [] + assert [{_id, %{"attachments" => [%{"sha256" => sha}]}}] = FakePostHog.messages(fake) + assert sha == sha256(@pdf) end - test "asks once for a file attached twice, and sends both references", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage(:stored) - expect_reply(accepted()) - + test "uploads a file attached twice once, and sends both references", %{ + name: name, + fake: fake + } do assert {:ok, _} = Emails.send( name, @@ -562,19 +764,29 @@ defmodule PostHog.EmailsTest do ]) ) - assert [{:post, _, ask}, {:put, _, _}, {:post, _, send}] = requests() - assert [%{"sha256" => sha}] = Jason.decode!(ask[:body])["files"] + assert [_one_put] = FakePostHog.received(fake, :storage) + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} + assert [{_id, email}] = FakePostHog.messages(fake) - assert [%{"sha256" => ^sha}, %{"sha256" => ^sha}] = - Jason.decode!(send[:body])["attachments"] + assert [ + %{"sha256" => sha, "filename" => "a.pdf"}, + %{"sha256" => sha, "filename" => "copy.pdf"} + ] = + email["attachments"] + + assert sha == sha256(@pdf) end - test "sends nothing after a failed upload, and returns the storage's answer", %{name: name} do - expect_reply(slots([present(@logo), to_upload(@pdf)])) + test "sends nothing after a failed upload, and returns the storage's answer", %{ + name: name, + fake: fake + } do + FakePostHog.put_files(fake, [@logo]) - expect_storage( - {:refused, 400, "BadDigest", - "The SHA256 you specified did not match the calculated checksum."} + FakePostHog.fail_next( + fake, + :storage, + {400, "BadDigest", "The SHA256 you specified did not match the calculated checksum."} ) assert {:error, %Error{} = error} = @@ -602,48 +814,44 @@ defmodule PostHog.EmailsTest do refute inspect(error) =~ "signed-secret" refute inspect(error) =~ @pdf - assert [{:post, @uploads_url, _}, {:put, @put_url, _}] = requests() + assert FakePostHog.messages(fake) == [] + assert FakePostHog.received(fake, :send) == [] end - @tag config: [email_request_max_retries: 4] - test "retries an upload the storage couldn't take, then sends", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage({:refused, 503, "SlowDown", "Please reduce your request rate."}) - expect_storage({:refused, 429, "TooManyRequests", "Please slow down."}) - expect_storage({:refused, 500, "InternalError", "We encountered an internal error."}) - expect_storage(:closed) - expect_storage(:stored) - expect_reply(accepted()) + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + email_request_max_retries: 4 + ] + test "retries an upload the storage couldn't take, then sends", %{name: name, fake: fake} do + FakePostHog.fail_next(fake, :storage, {503, "SlowDown", "Please reduce your request rate."}) + FakePostHog.fail_next(fake, :storage, {429, "TooManyRequests", "Please slow down."}) + + FakePostHog.fail_next( + fake, + :storage, + {500, "InternalError", "We encountered an internal error."} + ) - assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + FakePostHog.fail_next(fake, :storage, :closed) - assert [ - {:post, @uploads_url, _}, - {:put, _, _}, - {:put, _, _}, - {:put, _, _}, - {:put, _, _}, - {:put, _, _}, - {:post, @send_url, _} - ] = requests() + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} + assert [{^id, _email}] = FakePostHog.messages(fake) end - @tag config: [email_request_timeout_ms: 50] - test "gives an upload ten seconds per MB on top of the request timeout", %{name: name} do + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + email_request_timeout_ms: 50 + ] + test "gives an upload ten seconds per MB on top of the request timeout", %{ + name: name, + fake: fake + } do file = :binary.copy(<<1>>, 100_000) - test_pid = self() - - expect_reply(slots([to_upload(file)])) - - Req.Test.expect(PostHog.EmailsTest.Storage, fn conn -> - {:ok, _body, conn} = Plug.Conn.read_body(conn) - send(test_pid, {:request, :put, nil, []}) - # 100 KB at ten seconds per MB is 1 s on top of the 50 ms request timeout. - Process.sleep(950) - Plug.Conn.send_resp(conn, 200, "") - end) - - expect_reply(accepted()) + # 100 KB at ten seconds per MB is 1 s on top of the 50 ms request timeout. + FakePostHog.delay_next(fake, :storage, 950) assert {:ok, _} = Emails.send( @@ -651,12 +859,11 @@ defmodule PostHog.EmailsTest do with_files([%{filename: "a.bin", content: file}]) ++ [max_retries: 0] ) - assert [{:post, @uploads_url, _}, {:put, _, _}, {:post, @send_url, _}] = requests() + assert FakePostHog.files(fake) == %{sha256(file) => file} end - test "doesn't retry an upload the storage refused", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage({:refused, 403, "AccessDenied", "Request has expired."}) + test "doesn't retry an upload the storage refused", %{name: name, fake: fake} do + FakePostHog.fail_next(fake, :storage, {403, "AccessDenied", "Request has expired."}) assert {:error, %Error{ @@ -666,20 +873,19 @@ defmodule PostHog.EmailsTest do }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) refute_received {:slept, _} - assert [{:post, _, _}, {:put, _, _}] = requests() + assert [_one_put] = FakePostHog.received(fake, :storage) + assert FakePostHog.messages(fake) == [] end - test "a storage 401 doesn't blame the PostHog key", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage({:refused, 401, "Unauthorized", "Proxy needs a login."}) + test "a storage 401 doesn't blame the PostHog key", %{name: name, fake: fake} do + FakePostHog.fail_next(fake, :storage, {401, "Unauthorized", "Proxy needs a login."}) assert {:error, %Error{code: "attachment_upload_failed", name: :application_error}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end - test "an upload that keeps failing at the network sends nothing", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage(:econnrefused) + test "an upload that keeps failing at the network sends nothing", %{name: name, fake: fake} do + FakePostHog.fail_next(fake, :storage, :econnrefused) assert {:error, %Error{code: "attachment_upload_failed", status_code: nil} = error} = Emails.send( @@ -691,34 +897,30 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "reach files.example.com directly" assert error.field == "attachments__0" refute inspect(error) =~ "signed-secret" - assert [{:post, _, _}, {:put, _, _}] = requests() + assert FakePostHog.files(fake) == %{} + assert FakePostHog.received(fake, :send) == [] end - test "a send whose instance stops between its steps sends nothing more", %{name: name} do - test_pid = self() - - expect(API.Mock, :request, fn _client, :post, @uploads_url, _opts -> - send(test_pid, {:asked, self()}) - - receive do - :answer -> slots([to_upload(@pdf)]) - end - end) + test "a send whose instance stops between its steps sends nothing more", %{ + name: name, + fake: fake + } do + FakePostHog.hold_next(fake, :uploads) sending = Task.async(fn -> Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end) - assert_receive {:asked, worker} + assert_receive {:held, :uploads, request} stop_supervised!(name) - send(worker, :answer) + send(request, :release) assert {:error, %Error{code: "client_closed"} = error} = Task.await(sending) assert error.message =~ "no email was sent" + assert FakePostHog.received(fake, :storage) == [] + assert FakePostHog.received(fake, :send) == [] end - test "a template send takes empty lists as left out", %{name: name} do - expect_reply(accepted()) - + test "a template send takes empty lists as left out", %{name: name, fake: fake} do assert {:ok, _} = Emails.send(name, to: "user@example.com", @@ -729,40 +931,39 @@ defmodule PostHog.EmailsTest do attachments: [] ) - body = sent_body() - refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(body, &1)) + assert [{_id, email}] = FakePostHog.messages(fake) + refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(email, &1)) end - test "a send retry doesn't upload again", %{name: name} do - expect_reply(slots([to_upload(@pdf)])) - expect_storage(:stored) - expect_reply(refused(503, %{"code" => "service_unavailable"})) - expect_reply(accepted()) - - assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + test "a send retry doesn't upload again", %{name: name, fake: fake} do + FakePostHog.fail_next(fake, :send, {503, %{"code" => "service_unavailable"}}) - assert [ - {:post, @uploads_url, _}, - {:put, _, _}, - {:post, @send_url, first}, - {:post, @send_url, second} - ] = - requests() + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert header(first, "idempotency-key") == header(second, "idempotency-key") + assert [_first, _retry] = FakePostHog.received(fake, :send) + assert [_one_ask] = FakePostHog.received(fake, :uploads) + assert [_one_put] = FakePostHog.received(fake, :storage) + assert [{^id, _email}] = FakePostHog.messages(fake) end - test "returns PostHog's refusal of the uploads, naming the attachment", %{name: name} do - expect_reply( - refused(400, %{ - "type" => "invalid_request", - "code" => "message_too_large", - "detail" => - "These files are too large for one email, so PostHog didn't prepare the uploads.", - "attr" => "files__1__size", - "why" => "An email can be at most 40 MB after Base64 encoding, about 29 MB of files.", - "fix" => "Send fewer or smaller files." - }) + test "returns PostHog's refusal of the uploads, naming the attachment", %{ + name: name, + fake: fake + } do + FakePostHog.fail_next( + fake, + :uploads, + {400, + %{ + "type" => "invalid_request", + "code" => "message_too_large", + "detail" => + "These files are too large for one email, so PostHog didn't prepare the uploads.", + "attr" => "files__1__size", + "why" => "An email can be at most 40 MB after Base64 encoding, about 29 MB of files.", + "fix" => "Send fewer or smaller files." + }} ) assert {:error, @@ -781,75 +982,89 @@ defmodule PostHog.EmailsTest do ]) ) - expect_reply( - refused(400, %{"code" => "field_not_accepted", "attr" => "files", "detail" => "Not yet."}) + FakePostHog.fail_next( + fake, + :uploads, + {400, %{"code" => "field_not_accepted", "attr" => "files", "detail" => "Not yet."}} ) assert {:error, %Error{code: "field_not_accepted", field: "attachments"}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - expect_reply(refused(401, %{"code" => "authentication_failed", "detail" => "Bad key."})) + FakePostHog.fail_next( + fake, + :uploads, + {401, %{"code" => "authentication_failed", "detail" => "Bad key."}} + ) assert {:error, %Error{code: "authentication_failed", field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [{:post, @uploads_url, _}, {:post, @uploads_url, _}, {:post, @uploads_url, _}] = - requests() + assert FakePostHog.received(fake, :storage) == [] + assert FakePostHog.received(fake, :send) == [] end - test "a PostHog without the upload route sends nothing", %{name: name} do - expect_reply( - refused(404, %{ - "type" => "invalid_request", - "code" => "not_found", - "detail" => "Not found." - }) + test "a PostHog without the upload route sends nothing", %{name: name, fake: fake} do + FakePostHog.fail_next( + fake, + :uploads, + {404, %{"type" => "invalid_request", "code" => "not_found", "detail" => "Not found."}} ) assert {:error, %Error{code: "not_found", status_code: 404, field: "attachments"}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [{:post, @uploads_url, _}] = requests() + assert FakePostHog.received(fake, :send) == [] end - test "doesn't retry the daily upload quota, and retries unavailable storage", %{name: name} do - expect_reply( - refused(429, %{"code" => "upload_quota_exceeded", "detail" => "Enough today."}) + test "doesn't retry the daily upload quota, and retries unavailable storage", %{ + name: name, + fake: fake + } do + FakePostHog.fail_next( + fake, + :uploads, + {429, %{"code" => "upload_quota_exceeded", "detail" => "Enough today."}} ) assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429, field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) refute_received {:slept, _} + assert FakePostHog.messages(fake) == [] - expect_reply(refused(503, %{"code" => "delivery_unavailable"})) - expect_reply(slots([present(@pdf)])) - expect_reply(accepted()) + FakePostHog.fail_next(fake, :uploads, {503, %{"code" => "delivery_unavailable"}}) + + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) assert_received {:slept, _} + assert [{^id, _email}] = FakePostHog.messages(fake) end - test "an answer the SDK can't use sends nothing", %{name: name} do + test "an answer the SDK can't use sends nothing", %{name: name, fake: fake} do + slot = FakePostHog.upload_slot(@pdf) + for answer <- [ [], [%{"sha256" => sha256(@logo), "status" => "present"}], [%{"sha256" => sha256(@pdf), "status" => "copied"}], - [Map.delete(to_upload(@pdf), "url")], - [Map.put(to_upload(@pdf), "url", "ftp://files.example.com/a")], - [Map.put(to_upload(@pdf), "method", "POST")], - [Map.put(to_upload(@pdf), "headers", %{"Content-Length" => 8})] + [Map.delete(slot, "url")], + [Map.put(slot, "url", "ftp://files.example.com/a")], + [Map.put(slot, "method", "POST")], + [Map.put(slot, "headers", %{"Content-Length" => 8})] ] do - expect_reply(slots(answer)) + FakePostHog.fail_next(fake, :uploads, {200, %{"files" => answer}}) assert {:error, %Error{code: "api_error", status_code: 200}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - - assert [{:post, @uploads_url, _}] = requests() end + + assert FakePostHog.received(fake, :storage) == [] + assert FakePostHog.received(fake, :send) == [] end - test "refuses more than 10 files before any request", %{name: name} do + test "refuses more than 10 files before any request", %{name: name, fake: fake} do files = for i <- 1..11, do: %{filename: "#{i}.txt", content: "file #{i}"} assert {:error, @@ -860,24 +1075,16 @@ defmodule PostHog.EmailsTest do } = error} = Emails.send(name, with_files(files)) assert error.fix =~ "10" - refute_requested() + refute FakePostHog.received_any?(fake) end - test "refuses an email over 40 MB after Base64 encoding before any request", %{name: name} do + test "refuses an email over 40 MB after Base64 encoding before any request", %{ + name: name, + fake: fake + } do largest_that_fits = 29_182_821 file = :binary.copy(<<0>>, largest_that_fits) - expect_reply(slots([present(file)])) - expect_reply(accepted()) - - assert {:ok, _} = - Emails.send( - name, - @code_send ++ [html: "", attachments: [%{filename: "a", content: file}]] - ) - - assert [_ask, _send] = requests() - for options <- [ [html: "", attachments: [%{filename: "a", content: file <> <<0>>}]], [html: "

", attachments: [%{filename: "a", content: file}]], @@ -891,14 +1098,22 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "29 MB" end - refute_requested() + refute FakePostHog.received_any?(fake) + + FakePostHog.put_files(fake, [file]) + + assert {:ok, _} = + Emails.send( + name, + @code_send ++ [html: "", attachments: [%{filename: "a", content: file}]] + ) + + assert [{_id, _email}] = FakePostHog.messages(fake) end - test "leaves the size of an email without files to PostHog", %{test: test} do + test "leaves the size of an email without files to PostHog", %{test: test, fake: fake} do {:ok, server} = - start_supervised( - {Bandit, plug: {PostHog.EmailsTest.ApiServer, self()}, ip: :loopback, port: 0} - ) + start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) {:ok, {_ip, port}} = ThousandIsland.listener_info(server) host = "http://127.0.0.1:#{port}" @@ -917,11 +1132,10 @@ defmodule PostHog.EmailsTest do start_link_supervised!(Supervisor.child_spec({PostHog.Supervisor, config}, id: name)) html = :binary.copy("a", 40_000_000) - assert {:ok, %Emails.SendResult{id: @message_id}} = + assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, Keyword.put(@code_send, :html, html)) - assert_received {:received, "POST", "/api/projects/@current/messaging/send/email/", - %{"html" => ^html, "subject" => "Hi"}} + assert [{^id, %{"html" => ^html, "subject" => "Hi"}}] = FakePostHog.messages(fake) end # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the @@ -934,7 +1148,7 @@ defmodule PostHog.EmailsTest do 29_182_824}, {"that is empty counts nothing", "", 29_182_875, 29_182_876} ] do - test "a subject #{label} toward the 40 MB estimate", %{name: name} do + test "a subject #{label} toward the 40 MB estimate", %{name: name, fake: fake} do send_file = fn size -> Emails.send( name, @@ -947,17 +1161,16 @@ defmodule PostHog.EmailsTest do end assert {:error, %Error{code: "message_too_large"}} = send_file.(unquote(refused)) - refute_requested() + refute FakePostHog.received_any?(fake) - file = :binary.copy(<<0>>, unquote(fits)) - expect_reply(slots([present(file)])) - expect_reply(accepted()) + FakePostHog.put_files(fake, [:binary.copy(<<0>>, unquote(fits))]) assert {:ok, _} = send_file.(unquote(fits)) + assert [{_id, _email}] = FakePostHog.messages(fake) end end - test "refuses an attachment it can't send, naming the field", %{name: name} do + test "refuses an attachment it can't send, naming the field", %{name: name, fake: fake} do for {attachments, field} <- [ {%{filename: "a.txt", content: "a"}, "attachments"}, {"invoice.pdf", "attachments"}, @@ -983,10 +1196,10 @@ defmodule PostHog.EmailsTest do Emails.send(name, with_files(attachments)) end - refute_requested() + refute FakePostHog.received_any?(fake) end - test "asks for atom keys when an attachment has string keys", %{name: name} do + test "asks for atom keys when an attachment has string keys", %{name: name, fake: fake} do assert {:error, %Error{code: "invalid_attachment", field: "attachments__0__content"} = error} = Emails.send(name, with_files([%{"filename" => "a.txt", "content" => "a"}])) @@ -999,17 +1212,17 @@ defmodule PostHog.EmailsTest do with_files([%{"filename" => "a.txt", "content" => "a", "cid" => "logo"}]) ) - refute_requested() + refute FakePostHog.received_any?(fake) end - test "refuses an attachment key that has no string form", %{name: name} do + test "refuses an attachment key that has no string form", %{name: name, fake: fake} do attachment = %{{:extra, :field} => true, filename: "a.txt", content: "a"} assert {:error, %Error{code: "invalid_attachment", field: "attachments__0"} = error} = Emails.send(name, with_files([attachment])) assert error.message =~ "{:extra, :field}" - refute_requested() + refute FakePostHog.received_any?(fake) end test "a path points to reading the file", %{name: name} do @@ -1026,26 +1239,28 @@ defmodule PostHog.EmailsTest do refute inspect(error) =~ "SECRET-FILE-BYTES" end - test "returns a send's refusal of a file it can't find", %{name: name} do - expect_reply(slots([present(@pdf)])) + test "returns a send's refusal of a file that expired after its upload request", %{ + name: name, + fake: fake + } do + FakePostHog.put_files(fake, [@pdf]) + FakePostHog.hold_next(fake, :send) - expect_reply( - refused(400, %{ - "type" => "invalid_request", - "code" => "attachment_not_uploaded", - "detail" => "A file this email names isn't uploaded, so PostHog didn't send the email.", - "attr" => "attachments__0__sha256", - "why" => "This project has no uploaded file with this SHA-256.", - "fix" => "Upload the file and wait for its upload to succeed, then send again." - }) - ) + sending = + Task.async(fn -> Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end) + + assert_receive {:held, :send, request} + FakePostHog.delete_file(fake, @pdf) + send(request, :release) assert {:error, %Error{ code: "attachment_not_uploaded", field: "attachments__0__sha256", status_code: 400 - }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + }} = Task.await(sending) + + assert FakePostHog.messages(fake) == [] end end From 8d3b81f5b5c71f459729aaf891ff6db70bc9f6be Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:06:42 +0000 Subject: [PATCH 29/38] test(emails): let the storage fake refuse unsigned headers and a wrong signature The tests no longer read storage headers. The fake storage enforces what S3 checks on a presigned PUT, and a client that adds its own headers proves the upload still lands. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 78 ++++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 90713017..14235d0a 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -15,11 +15,37 @@ defmodule PostHog.EmailsTest.ReqTestClient do defdelegate request(client, method, url, opts), to: Client end +defmodule PostHog.EmailsTest.HeaderedClient do + @moduledoc false + # The default client with Req.Test in place of the network, set up as an app might: its own + # authorization and an extra header on every request it makes. + @behaviour PostHog.API.Client + + alias PostHog.API.Client + + @impl true + def client(api_key, api_host) do + default = Client.client(api_key, api_host) + + client = + Req.merge(default.client, + plug: {Req.Test, PostHog.EmailsTest.ReqTestClient}, + headers: [{"authorization", "Bearer client-token"}, {"x-client", "app"}] + ) + + %{default | client: client} + end + + @impl true + defdelegate request(client, method, url, opts), to: Client +end + defmodule PostHog.EmailsTest.FakePostHog do @moduledoc false # PostHog's email routes and its file storage as they behave, behind the SDK's real requests. # The upload route signs an upload for each file PostHog doesn't have, the storage keeps a file - # only when its upload matches what was signed, as S3 checks a presigned PUT, and the send route + # only when its upload matches what was signed, as S3 checks a presigned PUT: the signed URL, + # the signed headers and no others but the transport's, and the bytes. The send route # accepts an email only when PostHog has every file it names. A test reads back the stored # files, the accepted emails and what each route received, and can make a route's next request # fail, wait or hold. @@ -32,6 +58,9 @@ defmodule PostHog.EmailsTest.FakePostHog do @send_path "/api/projects/@current/messaging/send/email/" @storage_url "https://files.example.com" @max_body 64_000_000 + @signature "X-Amz-Signature=signed-secret" + # What the HTTP client adds on a real socket. Any other header would be outside the signature. + @transport_headers ["host", "user-agent"] defmodule Server do @moduledoc false @@ -108,7 +137,7 @@ defmodule PostHog.EmailsTest.FakePostHog do @doc """ What the route received, oldest first: the JSON body on PostHog's routes, and the method, - path, headers and body on the storage. + path, query, headers and body on the storage. """ def received(fake, route), do: Agent.get(fake, &Enum.reverse(&1.received[route])) @@ -123,7 +152,7 @@ defmodule PostHog.EmailsTest.FakePostHog do %{ "sha256" => sha256, "status" => "upload", - "url" => "#{storage_url}/team/1/#{sha256}?X-Amz-Signature=signed-secret", + "url" => "#{storage_url}/team/1/#{sha256}?#{@signature}", "method" => "PUT", "headers" => %{ "Content-Length" => to_string(size), @@ -220,7 +249,8 @@ defmodule PostHog.EmailsTest.FakePostHog do state} else slot = upload_slot(state.storage_url, sha256, size) - {slot, put_in(state.signed[URI.parse(slot["url"]).path], %{sha256: sha256, size: size})} + %URI{path: path, query: query} = URI.parse(slot["url"]) + {slot, put_in(state.signed[path], %{sha256: sha256, size: size, query: query})} end end @@ -269,7 +299,14 @@ defmodule PostHog.EmailsTest.FakePostHog do defp storage(conn, fake) do {:ok, body, conn} = Plug.Conn.read_body(conn, length: @max_body) headers = for {name, value} <- conn.req_headers, do: {String.downcase(name), value} - request = %{method: conn.method, path: conn.request_path, headers: headers, body: body} + + request = %{ + method: conn.method, + path: conn.request_path, + query: conn.query_string, + headers: headers, + body: body + } case receive_request(fake, :storage, request) do {{:reply, {status, code, message}}, _test} -> @@ -294,8 +331,7 @@ defmodule PostHog.EmailsTest.FakePostHog do List.keymember?(request.headers, "authorization", 0) -> s3_error(conn, 400, "InvalidArgument", "Only one auth mechanism allowed.") - header(request, "x-amz-checksum-sha256") != checksum(signed.sha256) or - header(request, "content-length") != to_string(signed.size) -> + not signature_matches?(request, signed) -> s3_error(conn, 403, "SignatureDoesNotMatch", "The request signature does not match.") Base.encode64(:crypto.hash(:sha256, request.body)) != checksum(signed.sha256) -> @@ -307,6 +343,16 @@ defmodule PostHog.EmailsTest.FakePostHog do end end + # The signed query and headers, and no headers but those and the transport's. + defp signature_matches?(request, signed) do + allowed = ["content-length", "x-amz-checksum-sha256" | @transport_headers] + + request.query == signed.query and + header(request, "x-amz-checksum-sha256") == checksum(signed.sha256) and + header(request, "content-length") == to_string(signed.size) and + Enum.all?(request.headers, fn {name, _value} -> name in allowed end) + end + defp header(request, name) do case List.keyfind(request.headers, name, 0) do {^name, value} -> value @@ -709,18 +755,18 @@ defmodule PostHog.EmailsTest do ] end - test "the storage gets only the headers PostHog signed, not the client's", %{ + # The storage refuses a header outside the signed ones, such as the client's. + @tag config: [api_client_module: PostHog.EmailsTest.HeaderedClient] + test "uploads with only the headers PostHog signed, not the client's", %{ name: name, fake: fake } do - assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - - assert [%{headers: headers}] = FakePostHog.received(fake, :storage) + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert headers |> Enum.map(&elem(&1, 0)) |> Enum.sort() == [ - "content-length", - "x-amz-checksum-sha256" - ] + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} + assert [{^id, %{"attachments" => [%{"sha256" => sha}]}}] = FakePostHog.messages(fake) + assert sha == sha256(@pdf) end # The installed Req's default adapter, over a real socket, as in production. @@ -737,8 +783,6 @@ defmodule PostHog.EmailsTest do assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} assert [{^id, _email}] = FakePostHog.messages(fake) - assert [%{headers: headers}] = FakePostHog.received(fake, :storage) - refute List.keymember?(headers, "authorization", 0) end test "uploads nothing when PostHog has every file", %{name: name, fake: fake} do From 14761d9c07da4c1db0d72b82ca3361c82bcb0b8b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:12:04 +0000 Subject: [PATCH 30/38] test(emails): refuse in the storage fake only what S3 refuses on a presigned PUT S3 ignores ordinary headers it wasn't asked to sign. It refuses a second credential and any unsigned x-amz- header, and the fake now does the same. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 14235d0a..32a86dd3 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -45,7 +45,7 @@ defmodule PostHog.EmailsTest.FakePostHog do # PostHog's email routes and its file storage as they behave, behind the SDK's real requests. # The upload route signs an upload for each file PostHog doesn't have, the storage keeps a file # only when its upload matches what was signed, as S3 checks a presigned PUT: the signed URL, - # the signed headers and no others but the transport's, and the bytes. The send route + # the signed headers, no second credential or unsigned x-amz- header, and the bytes. The send route # accepts an email only when PostHog has every file it names. A test reads back the stored # files, the accepted emails and what each route received, and can make a route's next request # fail, wait or hold. @@ -59,8 +59,7 @@ defmodule PostHog.EmailsTest.FakePostHog do @storage_url "https://files.example.com" @max_body 64_000_000 @signature "X-Amz-Signature=signed-secret" - # What the HTTP client adds on a real socket. Any other header would be outside the signature. - @transport_headers ["host", "user-agent"] + @signed_headers ["content-length", "x-amz-checksum-sha256"] defmodule Server do @moduledoc false @@ -331,6 +330,14 @@ defmodule PostHog.EmailsTest.FakePostHog do List.keymember?(request.headers, "authorization", 0) -> s3_error(conn, 400, "InvalidArgument", "Only one auth mechanism allowed.") + unsigned_amz_header?(request) -> + s3_error( + conn, + 403, + "AccessDenied", + "There were headers present in the request which were not signed." + ) + not signature_matches?(request, signed) -> s3_error(conn, 403, "SignatureDoesNotMatch", "The request signature does not match.") @@ -343,14 +350,17 @@ defmodule PostHog.EmailsTest.FakePostHog do end end - # The signed query and headers, and no headers but those and the transport's. - defp signature_matches?(request, signed) do - allowed = ["content-length", "x-amz-checksum-sha256" | @transport_headers] + # S3 ignores ordinary headers it wasn't asked to sign, but refuses an unsigned x-amz- one. + defp unsigned_amz_header?(request) do + Enum.any?(request.headers, fn {name, _value} -> + String.starts_with?(name, "x-amz-") and name not in @signed_headers + end) + end + defp signature_matches?(request, signed) do request.query == signed.query and header(request, "x-amz-checksum-sha256") == checksum(signed.sha256) and - header(request, "content-length") == to_string(signed.size) and - Enum.all?(request.headers, fn {name, _value} -> name in allowed end) + header(request, "content-length") == to_string(signed.size) end defp header(request, name) do @@ -755,7 +765,7 @@ defmodule PostHog.EmailsTest do ] end - # The storage refuses a header outside the signed ones, such as the client's. + # The storage refuses a second credential, such as the client's authorization. @tag config: [api_client_module: PostHog.EmailsTest.HeaderedClient] test "uploads with only the headers PostHog signed, not the client's", %{ name: name, From a96da42b5b1d68d3b30474a1c81f1e976d70ed79 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:19:18 +0000 Subject: [PATCH 31/38] test(emails): prove an upload ignores the app's Req defaults by what the storage keeps The test ran on accept-all mocks and asserted the header list. It now runs against the fake PostHog over a real socket, with app-wide Req defaults that S3 would refuse on a presigned PUT, and asserts only the result, the stored bytes and the accepted email. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 68 ++++++++++-------------------------- 1 file changed, 19 insertions(+), 49 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 32a86dd3..9b57b90f 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -2000,23 +2000,18 @@ defmodule PostHog.EmailsReqDefaultsTest do # Req.default_options is app-wide, so this runs alone, after the async tests. use ExUnit.Case, async: false - import Mox - - alias PostHog.API alias PostHog.Emails + alias PostHog.EmailsTest.FakePostHog - setup :verify_on_exit! setup {Req.Test, :verify_on_exit!} setup context do - Mox.stub_with(API.Mock, API.Stub) previous = Application.fetch_env(:req, :default_options) - Application.put_env(:req, :default_options, - auth: {:bearer, "app-wide"}, - headers: [{"x-app-wide", "1"}], - plug: {Req.Test, PostHog.EmailsReqDefaultsTest.AppWide} - ) + # Defaults an app might set for its own requests. The storage refuses either on an upload, as + # S3 does on a presigned PUT: a query that isn't the signed one, and gzipped bytes that no + # longer match the signed checksum. + Application.put_env(:req, :default_options, params: [app: "1"], compress_body: true) on_exit(fn -> case previous do @@ -2030,53 +2025,27 @@ defmodule PostHog.EmailsReqDefaultsTest do api_host: "https://us.i.posthog.com", api_key: "phc_project_key", secret_key: "phs_secret_key", - api_client_module: API.Mock, + api_client_module: PostHog.EmailsTest.ReqTestClient, supervisor_name: context.test, enable_local_evaluation: false ] |> PostHog.Config.validate!() - |> Map.put(:email_upload_adapter, {Req.Plug, plug: {Req.Test, __MODULE__}}) + # The installed Req's default adapter over a real socket: Req.Test's plug would unzip a body. + |> Map.put(:email_upload_adapter, :none) start_link_supervised!({PostHog.Supervisor, config}) - %{name: context.test} + %{name: context.test, fake: FakePostHog.start!()} end - test "an upload ignores the app's Req defaults", %{name: name} do - file = "file bytes" - sha256 = :crypto.hash(:sha256, file) |> Base.encode16(case: :lower) + test "an upload ignores the app's Req defaults", %{name: name, fake: fake} do + {:ok, server} = + start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) - expect(API.Mock, :request, fn _client, :post, _url, _opts -> - {:ok, - %{ - status: 200, - headers: %{}, - body: %{ - "files" => [ - %{ - "sha256" => sha256, - "status" => "upload", - "url" => "https://files.example.com/a?X-Amz-Signature=signed", - "method" => "PUT", - "headers" => %{"Content-Length" => "10"} - } - ] - } - }} - end) - - expect(API.Mock, :request, fn _client, :post, _url, _opts -> - {:ok, %{status: 202, headers: %{}, body: %{"message_id" => "m"}}} - end) - - test_pid = self() - - Req.Test.expect(__MODULE__, fn conn -> - {:ok, body, conn} = Plug.Conn.read_body(conn) - send(test_pid, {:stored, conn.req_headers, body}) - Plug.Conn.send_resp(conn, 200, "") - end) + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{port}") + file = "file bytes" - assert {:ok, _} = + assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, to: "user@example.com", template_key: "receipt", @@ -2085,7 +2054,8 @@ defmodule PostHog.EmailsReqDefaultsTest do attachments: [%{filename: "a.txt", content: file}] ) - assert_received {:stored, headers, ^file} - assert Enum.map(headers, &elem(&1, 0)) == ["content-length"] + sha256 = :crypto.hash(:sha256, file) |> Base.encode16(case: :lower) + assert FakePostHog.files(fake) == %{sha256 => file} + assert [{^id, %{"attachments" => [%{"sha256" => ^sha256}]}}] = FakePostHog.messages(fake) end end From f124592d0fb62c9b49676976beeafdb50f59074d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:27:50 +0000 Subject: [PATCH 32/38] test(emails): let the fake PostHog size an email as PostHog does The send route accepted any email, so the test of a send without files expected 40 MB of html to go through. The fake now counts the message as PostHog does and refuses one over 40 MB, and the test checks that the whole email reaches PostHog and PostHog's refusal comes back. Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 9b57b90f..f836e835 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -43,12 +43,12 @@ end defmodule PostHog.EmailsTest.FakePostHog do @moduledoc false # PostHog's email routes and its file storage as they behave, behind the SDK's real requests. - # The upload route signs an upload for each file PostHog doesn't have, the storage keeps a file + # The upload route signs an upload for each file PostHog doesn't have. The storage keeps a file # only when its upload matches what was signed, as S3 checks a presigned PUT: the signed URL, - # the signed headers, no second credential or unsigned x-amz- header, and the bytes. The send route - # accepts an email only when PostHog has every file it names. A test reads back the stored - # files, the accepted emails and what each route received, and can make a route's next request - # fail, wait or hold. + # the signed headers, no second credential or unsigned x-amz- header, and the bytes. The send + # route accepts an email only when PostHog has every file it names and the email fits PostHog's + # 40 MB estimate. A test reads back the stored files, the accepted emails and what each route + # received, and can make a route's next request fail, wait or hold. # # The routes are :uploads and :send on PostHog, and :storage. Uploads are signed for # https://files.example.com unless set_storage_url/2 names another host. @@ -58,6 +58,7 @@ defmodule PostHog.EmailsTest.FakePostHog do @send_path "/api/projects/@current/messaging/send/email/" @storage_url "https://files.example.com" @max_body 64_000_000 + @max_message_bytes 40_000_000 @signature "X-Amz-Signature=signed-secret" @signed_headers ["content-length", "x-amz-checksum-sha256"] @@ -272,10 +273,31 @@ defmodule PostHog.EmailsTest.FakePostHog do ) ) else - json(conn, 202, %{"message_id" => record_message(fake, email)}) + if message_bytes(email, files) > @max_message_bytes do + json(conn, 400, %{ + "type" => "invalid_request", + "code" => "message_too_large", + "detail" => "This email is too large to send, so PostHog didn't send it.", + "attr" => nil + }) + else + json(conn, 202, %{"message_id" => record_message(fake, email)}) + end end end + # PostHog's estimate of the sent message: each file and body as folded Base64, 64 KiB for + # headers and MIME boundaries, and the subject as 75 bytes per 42 started. + defp message_bytes(email, files) do + bodies = for key <- ["html", "text"], body = email[key], is_binary(body), do: byte_size(body) + sizes = for %{"sha256" => sha256} <- email["attachments"] || [], do: byte_size(files[sha256]) + subject = byte_size(email["subject"] || "") + + Enum.reduce(bodies ++ sizes, 64 * 1024 + div(subject + 41, 42) * 75, fn size, total -> + total + div(div(size + 2, 3) * 4 * 78 + 75, 76) + end) + end + defp record_message(fake, email) do Agent.get_and_update(fake, fn state -> number = state.messages |> length() |> Kernel.+(1) |> Integer.to_string() @@ -1186,10 +1208,11 @@ defmodule PostHog.EmailsTest do start_link_supervised!(Supervisor.child_spec({PostHog.Supervisor, config}, id: name)) html = :binary.copy("a", 40_000_000) - assert {:ok, %Emails.SendResult{id: id}} = + assert {:error, %Error{code: "message_too_large", status_code: 400}} = Emails.send(name, Keyword.put(@code_send, :html, html)) - assert [{^id, %{"html" => ^html, "subject" => "Hi"}}] = FakePostHog.messages(fake) + assert [%{"html" => ^html, "subject" => "Hi"}] = FakePostHog.received(fake, :send) + assert FakePostHog.messages(fake) == [] end # With no subject and an empty body, these sizes leave 3, 73, 77, 147 and 151 bytes of the From 0447cd08574afda3456ba0b4f4c1b2732c1b8b49 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:34:57 +0000 Subject: [PATCH 33/38] test(emails): count the upload route's requests, not the sleep hook, for retries Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index f836e835..e793c3af 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -948,7 +948,6 @@ defmodule PostHog.EmailsTest do status_code: 403 }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - refute_received {:slept, _} assert [_one_put] = FakePostHog.received(fake, :storage) assert FakePostHog.messages(fake) == [] end @@ -1106,7 +1105,7 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429, field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - refute_received {:slept, _} + assert [_one_request] = FakePostHog.received(fake, :uploads) assert FakePostHog.messages(fake) == [] FakePostHog.fail_next(fake, :uploads, {503, %{"code" => "delivery_unavailable"}}) @@ -1114,7 +1113,8 @@ defmodule PostHog.EmailsTest do assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert_received {:slept, _} + # The quota request, then the unavailable one and its retry. + assert [_quota, _unavailable, _retry] = FakePostHog.received(fake, :uploads) assert [{^id, _email}] = FakePostHog.messages(fake) end From 357c5d3a2ddc648d61be2a5c74f93a82f3bb489a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:48:06 +0000 Subject: [PATCH 34/38] fix(emails): count the text PostHog derives and upload only over https Without text, PostHog derives the plain text from the HTML and counts it in the message size, so an HTML-only email near the limit uploaded its files and was then refused. The SDK now counts the HTML twice, since the derived text is never longer. A plain-HTTP upload URL from misconfigured storage would carry the file and its signed URL unencrypted. The SDK now uploads nothing and sends nothing unless the app's own app_host is plain HTTP, as in local development. The fake PostHog in the tests counts the derived text too. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-attachments.md | 2 +- lib/posthog/emails.ex | 24 +++++++-- lib/posthog/emails/attachments.ex | 24 +++++++++ test/posthog/emails_test.exs | 71 +++++++++++++++++++++++-- 4 files changed, 114 insertions(+), 7 deletions(-) diff --git a/.sampo/changesets/emails-attachments.md b/.sampo/changesets/emails-attachments.md index c8b8888f..d9ab8dcb 100644 --- a/.sampo/changesets/emails-attachments.md +++ b/.sampo/changesets/emails-attachments.md @@ -2,4 +2,4 @@ hex/posthog: minor --- -`PostHog.Emails.send/2` takes `cc`, `bcc` and `attachments` on a send with `template_key`. `cc` and `bcc` are one address or a list. Each attachment is a map with `filename`, `content` (the file's raw bytes) and optionally `content_type` and `content_id`. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The SDK hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, without the secret key, and then sends the email with references to them. If an upload fails, it sends nothing and returns `attachment_upload_failed`. An empty `reply_to`, `cc`, `bcc` or `attachments` list is now left out of the request. +`PostHog.Emails.send/2` takes `cc`, `bcc` and `attachments` on a send with `template_key`. `cc` and `bcc` are one address or a list. Each attachment is a map with `filename`, `content` (the file's raw bytes) and optionally `content_type` and `content_id`. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The SDK hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, over https and without the secret key, and then sends the email with references to them. Plain-HTTP uploads are allowed only when `app_host` is plain HTTP, as in local development. If an upload fails, it sends nothing and returns `attachment_upload_failed`. An empty `reply_to`, `cc`, `bcc` or `attachments` list is now left out of the request. diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index b4985e8b..15016483 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -89,7 +89,8 @@ defmodule PostHog.Emails do An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The SDK checks both before any request of a send with files, counting each file and body as Base64 - and the subject as 75 bytes per 42 started, as PostHog does. + and the subject as 75 bytes per 42 started, as PostHog does. Without `:text`, PostHog derives + the text from `:html` and counts it too, so the SDK counts the HTML twice. A send with files takes three steps: @@ -101,6 +102,11 @@ defmodule PostHog.Emails do nothing again. 3. Once every upload succeeded, it sends the email, which names each file by its hash. + Uploads go only over https, so a file and its signed URL never travel unencrypted. When + PostHog names a plain-HTTP upload URL, the SDK uploads nothing, sends nothing, and returns + `api_error` on the attachment. An `:app_host` on plain HTTP, as in local development, allows + plain-HTTP uploads too. + Your app must be able to reach the storage host that PostHog's answer names directly: an upload doesn't go through a proxy, and it ignores your app's `Req.default_options`, which are meant for other requests. If an upload fails, the SDK sends nothing and returns @@ -366,11 +372,15 @@ defmodule PostHog.Emails do defp upload(connection, prepared) do max_retries = prepared.max_retries || connection.max_retries - with {:ok, uploads} <- ask_where_to_upload(connection, prepared.files, max_retries) do + with {:ok, uploads} <- ask_where_to_upload(connection, prepared.files, max_retries), + :ok <- Attachments.check_https(uploads, plain_http?(connection)) do put_files(connection, uploads, max_retries) end end + # An app that talks to PostHog over plain HTTP, as in local development, may upload over it. + defp plain_http?(connection), do: URI.parse(connection.app_host).scheme == "http" + defp put_files(_connection, [], _max_retries), do: :ok defp put_files(connection, [{file, upload} | uploads], max_retries) do @@ -519,11 +529,19 @@ defmodule PostHog.Emails do defp attachments(options) do Attachments.prepare( Map.get(options, :attachments, []), - Enum.flat_map([:html, :text], &List.wrap(options[&1])), + bodies(options), Map.get(options, :subject) || "" ) end + # Without text, PostHog derives it from the HTML and counts it too. The derived text drops tags + # and shortens entities and links, so it is never longer than the HTML. + defp bodies(options) do + html = List.wrap(options[:html]) + text = if options[:text] in [nil, ""], do: html, else: [options[:text]] + html ++ text + end + defp check_template_key(template_key) do if is_binary(template_key) and template_key =~ ~r/\A[a-zA-Z0-9_-]{1,400}\z/ do :ok diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 7a9b0455..0892bd73 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -67,6 +67,30 @@ defmodule PostHog.Emails.Attachments do end end + @doc """ + Refuses the uploads when one would go over plain HTTP, unless `plain_http?` allows it. The + error names the attachment and holds no URL. + """ + @spec check_https([{file(), upload()}], boolean()) :: :ok | {:error, Error.t()} + def check_https(_uploads, true), do: :ok + + def check_https(uploads, false) do + case Enum.find(uploads, fn {_file, upload} -> URI.parse(upload.url).scheme != "https" end) do + nil -> + :ok + + {file, _upload} -> + {:error, + Error.new( + "api_error", + "PostHog named an upload URL that isn't https, so #{file.filename} would travel unencrypted. No email was sent.", + field: file.field, + fix: + "Fix the file storage's URL in PostHog so it uses https. For local development over plain HTTP, set an http :app_host." + )} + end + end + @doc """ Names the option a refusal of the upload request is about: `attachments`, or one of them. PostHog names the upload request's `files`, and the caller passed `attachments`. A refusal of diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index e793c3af..15929ef7 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -287,9 +287,17 @@ defmodule PostHog.EmailsTest.FakePostHog do end # PostHog's estimate of the sent message: each file and body as folded Base64, 64 KiB for - # headers and MIME boundaries, and the subject as 75 bytes per 42 started. + # headers and MIME boundaries, and the subject as 75 bytes per 42 started. Without a text body, + # PostHog derives one from the HTML, here by dropping its tags, and counts it too. defp message_bytes(email, files) do - bodies = for key <- ["html", "text"], body = email[key], is_binary(body), do: byte_size(body) + html = email["html"] || "" + + text = + if email["text"] in [nil, ""], + do: String.replace(html, ~r/<[^>]*>/, ""), + else: email["text"] + + bodies = [byte_size(html), byte_size(text)] sizes = for %{"sha256" => sha256} <- email["attachments"] || [], do: byte_size(files[sha256]) subject = byte_size(email["subject"] || "") @@ -801,8 +809,13 @@ defmodule PostHog.EmailsTest do assert sha == sha256(@pdf) end - # The installed Req's default adapter, over a real socket, as in production. + # The installed Req's default adapter, over a real socket, as in production. The local storage + # is plain HTTP, which an app on a plain-HTTP PostHog may upload to. @tag upload_adapter: :none + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + app_host: "http://localhost:8010" + ] test "uploads over HTTP with Req's default adapter", %{name: name, fake: fake} do {:ok, server} = start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) @@ -1187,6 +1200,56 @@ defmodule PostHog.EmailsTest do assert [{_id, _email}] = FakePostHog.messages(fake) end + # PostHog derives the text from the HTML and counts it too, never longer than the HTML. + test "counts the HTML twice without a text body", %{name: name, fake: fake} do + html = :binary.copy("a", 1_000_000) + files = [%{filename: "a.bin", content: :binary.copy(<<1>>, 28_000_000)}] + html_only = Keyword.put(with_files(files), :html, html) + + assert {:error, %Error{code: "message_too_large", field: "attachments"}} = + Emails.send(name, html_only) + + assert FakePostHog.received(fake, :uploads) == [] + + assert {:ok, %Emails.SendResult{}} = Emails.send(name, html_only ++ [text: "Thanks"]) + end + + test "uploads nothing to a plain-HTTP storage URL", %{name: name, fake: fake} do + FakePostHog.set_storage_url(fake, "http://files.example.com") + + assert {:error, %Error{code: "api_error", field: "attachments__0"} = error} = + Emails.send( + name, + with_files([ + %{filename: "a.pdf", content: @pdf}, + %{filename: "logo.png", content: @logo} + ]) + ) + + assert error.message =~ "a.pdf" + assert error.fix =~ "https" + refute inspect(error) =~ "files.example.com" + assert FakePostHog.received(fake, :storage) == [] + assert FakePostHog.messages(fake) == [] + end + + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + app_host: "http://localhost:8010" + ] + test "uploads over plain HTTP when the app talks to PostHog over plain HTTP", %{ + name: name, + fake: fake + } do + FakePostHog.set_storage_url(fake, "http://localhost:9000") + + assert {:ok, %Emails.SendResult{id: id}} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} + assert [{^id, _email}] = FakePostHog.messages(fake) + end + test "leaves the size of an email without files to PostHog", %{test: test, fake: fake} do {:ok, server} = start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) @@ -2048,6 +2111,8 @@ defmodule PostHog.EmailsReqDefaultsTest do api_host: "https://us.i.posthog.com", api_key: "phc_project_key", secret_key: "phs_secret_key", + # The local storage is plain HTTP, which an app on a plain-HTTP PostHog may upload to. + app_host: "http://localhost:8010", api_client_module: PostHog.EmailsTest.ReqTestClient, supervisor_name: context.test, enable_local_evaluation: false From 6dfb73c52b5ee133de70027842f3016e623c1078 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:18:26 +0000 Subject: [PATCH 35/38] fix(emails): point an oversized HTML-only email at :text, and test uploads by what PostHog keeps An email refused for size without :text now says PostHog counts the derived text too, and that a short :text counts only that. The https fix says plain HTTP is for local development only. The tests now prove "uploads once" and "never retried" by a daily upload quota and by what the storage keeps, not by counting requests, and serve the fake on a real socket as a function plug. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 14 +-- lib/posthog/emails/attachments.ex | 37 +++++-- test/posthog/emails_test.exs | 174 ++++++++++++++++++------------ 3 files changed, 131 insertions(+), 94 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 15016483..67cc2a0d 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -527,19 +527,7 @@ defmodule PostHog.Emails do end defp attachments(options) do - Attachments.prepare( - Map.get(options, :attachments, []), - bodies(options), - Map.get(options, :subject) || "" - ) - end - - # Without text, PostHog derives it from the HTML and counts it too. The derived text drops tags - # and shortens entities and links, so it is never longer than the HTML. - defp bodies(options) do - html = List.wrap(options[:html]) - text = if options[:text] in [nil, ""], do: html, else: [options[:text]] - html ++ text + Attachments.prepare(Map.get(options, :attachments, []), options) end defp check_template_key(template_key) do diff --git a/lib/posthog/emails/attachments.ex b/lib/posthog/emails/attachments.ex index 0892bd73..6d1437e5 100644 --- a/lib/posthog/emails/attachments.ex +++ b/lib/posthog/emails/attachments.ex @@ -24,20 +24,21 @@ defmodule PostHog.Emails.Attachments do @type upload() :: %{url: String.t(), headers: [{String.t(), String.t()}]} @doc """ - Checks the files and the size of the email they make with `bodies` and `subject`, and hashes - each file. An email without files is left to PostHog to size, as nothing is uploaded first. + Checks the files and the size of the email they make with its `:html`, `:text` and `:subject`, + and hashes each file. An email without files is left to PostHog to size, as nothing is + uploaded first. """ - @spec prepare(term(), [String.t()], String.t()) :: {:ok, [file()]} | {:error, Error.t()} - def prepare(attachments, bodies, subject) when is_list(attachments) do + @spec prepare(term(), map()) :: {:ok, [file()]} | {:error, Error.t()} + def prepare(attachments, email) when is_list(attachments) do # Hashing reads every byte, so it waits until the files are known to fit. with :ok <- check_count(attachments), {:ok, files} <- check_each(attachments), - :ok <- check_size(files, bodies, subject) do + :ok <- check_size(files, email) do {:ok, Enum.map(files, &hash/1)} end end - def prepare(_attachments, _bodies, _subject), + def prepare(_attachments, _email), do: invalid("attachments must be a list of files.", "attachments") @doc "The body of the upload request: each distinct file once." @@ -86,7 +87,7 @@ defmodule PostHog.Emails.Attachments do "PostHog named an upload URL that isn't https, so #{file.filename} would travel unencrypted. No email was sent.", field: file.field, fix: - "Fix the file storage's URL in PostHog so it uses https. For local development over plain HTTP, set an http :app_host." + "Fix the file storage's URL in PostHog so it uses https. Plain-HTTP uploads are allowed only in local development, where :app_host is plain HTTP too." )} end end @@ -165,10 +166,16 @@ defmodule PostHog.Emails.Attachments do end end - defp check_size([], _bodies, _subject), do: :ok + defp check_size([], _email), do: :ok - defp check_size(files, bodies, subject) do + defp check_size(files, email) do + html = List.wrap(email[:html]) + derived_text? = email[:text] in [nil, ""] and html != [] + # Without text, PostHog derives it from the HTML and counts it too. The derived text drops + # tags and shortens entities and links, so it is never longer than the HTML. + bodies = if derived_text?, do: html ++ html, else: html ++ List.wrap(email[:text]) sizes = Enum.map(files, & &1.size) ++ Enum.map(bodies, &byte_size/1) + subject = email[:subject] || "" if base64_message_bytes(sizes) + encoded_subject_bytes(subject) > @max_message_bytes do {:error, @@ -178,14 +185,22 @@ defmodule PostHog.Emails.Attachments do field: "attachments", why: "PostHog sends emails up to 40 MB. Base64 makes each file and body about a third larger, and the subject counts 75 bytes per 42 bytes started.", - fix: - "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." + fix: too_large_fix(derived_text?) )} else :ok end end + @too_large_fix "Keep the email under 40 MB after Base64 encoding, about 29 MB of files: attach fewer or smaller files, or link to large files from the email body." + + defp too_large_fix(false), do: @too_large_fix + + defp too_large_fix(true), + do: + @too_large_fix <> + " Without :text, PostHog derives the text from :html and counts it too, so pass a short :text to count only that." + defp base64_message_bytes(sizes) do Enum.reduce(sizes, @headers_and_mime_boundaries_bytes, fn size, total -> total + ceil_div(ceil_div(size, 3) * 4 * 78, 76) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 15929ef7..f6fc9e0b 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -47,7 +47,9 @@ defmodule PostHog.EmailsTest.FakePostHog do # only when its upload matches what was signed, as S3 checks a presigned PUT: the signed URL, # the signed headers, no second credential or unsigned x-amz- header, and the bytes. The send # route accepts an email only when PostHog has every file it names and the email fits PostHog's - # 40 MB estimate. A test reads back the stored files, the accepted emails and what each route + # 40 MB estimate. Like PostHog, it can hold a project to a daily upload quota: the upload route + # refuses to sign uploads beyond what is left of it, and every file the storage keeps counts + # against it. A test reads back the stored files, the accepted emails and what the send route # received, and can make a route's next request fail, wait or hold. # # The routes are :uploads and :send on PostHog, and :storage. Uploads are signed for @@ -62,20 +64,6 @@ defmodule PostHog.EmailsTest.FakePostHog do @signature "X-Amz-Signature=signed-secret" @signed_headers ["content-length", "x-amz-checksum-sha256"] - defmodule Server do - @moduledoc false - # The fake as a plug, to serve PostHog and the storage on a real socket. - @behaviour Plug - - alias PostHog.EmailsTest.FakePostHog - - @impl true - def init(fake), do: fake - - @impl true - def call(conn, fake), do: FakePostHog.serve(conn, fake) - end - @doc "Starts a fake for the calling test and routes the test's PostHog and storage requests to it." def start! do state = %{ @@ -84,6 +72,8 @@ defmodule PostHog.EmailsTest.FakePostHog do files: %{}, signed: %{}, messages: [], + upload_quota: nil, + uploaded_bytes: 0, received: %{uploads: [], storage: [], send: []}, next: %{uploads: [], storage: [], send: []} } @@ -110,6 +100,9 @@ defmodule PostHog.EmailsTest.FakePostHog do def delete_file(fake, bytes), do: Agent.update(fake, &%{&1 | files: Map.delete(&1.files, sha256(bytes))}) + @doc "Holds the project to a daily upload quota of `bytes`, counting the files stored today." + def set_upload_quota(fake, bytes), do: Agent.update(fake, &%{&1 | upload_quota: bytes}) + @doc "Signs later uploads for the storage at `url`." def set_storage_url(fake, url), do: Agent.update(fake, &%{&1 | storage_url: url}) @@ -141,10 +134,6 @@ defmodule PostHog.EmailsTest.FakePostHog do """ def received(fake, route), do: Agent.get(fake, &Enum.reverse(&1.received[route])) - @doc "Whether any route received a request." - def received_any?(fake), - do: Agent.get(fake, fn state -> Enum.any?(state.received, &(elem(&1, 1) != [])) end) - @doc "The answer PostHog gives for a file it doesn't have, signed for the default storage." def upload_slot(bytes), do: upload_slot(@storage_url, sha256(bytes), byte_size(bytes)) @@ -162,7 +151,7 @@ defmodule PostHog.EmailsTest.FakePostHog do } end - @doc false + @doc "Serves PostHog and the storage, as a function plug for a real socket." def serve(conn, fake) do case conn.request_path do @uploads_path -> posthog(conn, :uploads, fake) @@ -231,20 +220,43 @@ defmodule PostHog.EmailsTest.FakePostHog do end defp sign_uploads(conn, %{"files" => files}, fake) when is_list(files) do - answers = - Agent.get_and_update( - fake, - &Enum.map_reduce(files, &1, fn file, state -> sign(file, state) end) - ) + case Agent.get_and_update(fake, &sign_within_quota(files, &1)) do + {:ok, answers} -> + json(conn, 200, %{"files" => answers}) - json(conn, 200, %{"files" => answers}) + :quota_exceeded -> + json( + conn, + 429, + refusal( + "upload_quota_exceeded", + "This project uploaded as many file bytes as it can today.", + nil + ) + ) + end end defp sign_uploads(conn, _request, _fake), do: json(conn, 400, refusal("required", "This field is required.", "files")) + # Each file to upload counts in full, before any is signed. + defp sign_within_quota(files, state) do + to_upload = + for %{"size" => size, "sha256" => sha256} <- files, not stored?(state, sha256), do: size + + if state.upload_quota && state.uploaded_bytes + Enum.sum(to_upload) > state.upload_quota do + {:quota_exceeded, state} + else + {answers, state} = Enum.map_reduce(files, state, &sign/2) + {{:ok, answers}, state} + end + end + + defp stored?(state, sha256), do: Map.has_key?(state.files, sha256) + defp sign(%{"sha256" => sha256, "size" => size}, state) do - if Map.has_key?(state.files, sha256) do + if stored?(state, sha256) do {%{"sha256" => sha256, "status" => "present", "expires_at" => "2026-12-16T09:00:00Z"}, state} else @@ -375,7 +387,14 @@ defmodule PostHog.EmailsTest.FakePostHog do s3_error(conn, 400, "BadDigest", "The SHA256 you specified did not match.") true -> - Agent.update(fake, &put_in(&1.files[signed.sha256], request.body)) + Agent.update(fake, fn state -> + %{ + state + | files: Map.put(state.files, signed.sha256, request.body), + uploaded_bytes: state.uploaded_bytes + byte_size(request.body) + } + end) + Plug.Conn.send_resp(conn, 200, "") end end @@ -684,6 +703,10 @@ defmodule PostHog.EmailsTest do end end + # What a refused send leaves behind at PostHog: no email and no file. + defp nothing_sent?(fake), + do: FakePostHog.messages(fake) == [] and FakePostHog.files(fake) == %{} + describe "send/2 with cc and bcc" do @describetag config: [api_client_module: PostHog.EmailsTest.ReqTestClient] @code_send [to: "user@example.com", template_key: "receipt", subject: "Hi", html: "

"] @@ -705,7 +728,7 @@ defmodule PostHog.EmailsTest do assert email["bcc"] == ["audit@example.com", "archive@example.com"] end - test "leaves out empty lists, and uploads nothing for no files", %{name: name, fake: fake} do + test "leaves out empty lists", %{name: name, fake: fake} do assert {:ok, _} = Emails.send( name, @@ -714,10 +737,9 @@ defmodule PostHog.EmailsTest do assert [{_id, email}] = FakePostHog.messages(fake) refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(email, &1)) - assert FakePostHog.received(fake, :uploads) == [] end - test "a template send refuses them and attachments before any request", %{ + test "a template send refuses them and attachments, and sends nothing", %{ name: name, fake: fake } do @@ -738,7 +760,7 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "template_key" end - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end test "refuses cc and bcc that are not addresses", %{name: name, fake: fake} do @@ -749,7 +771,7 @@ defmodule PostHog.EmailsTest do assert field_name == to_string(field) end - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end end @@ -818,7 +840,7 @@ defmodule PostHog.EmailsTest do ] test "uploads over HTTP with Req's default adapter", %{name: name, fake: fake} do {:ok, server} = - start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) + start_supervised({Bandit, plug: {&FakePostHog.serve/2, fake}, ip: :loopback, port: 0}) {:ok, {_ip, port}} = ThousandIsland.listener_info(server) FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{port}") @@ -830,20 +852,25 @@ defmodule PostHog.EmailsTest do assert [{^id, _email}] = FakePostHog.messages(fake) end + # With no upload quota left, any upload would be refused. test "uploads nothing when PostHog has every file", %{name: name, fake: fake} do FakePostHog.put_files(fake, [@pdf]) + FakePostHog.set_upload_quota(fake, 0) assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert FakePostHog.received(fake, :storage) == [] + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} assert [{_id, %{"attachments" => [%{"sha256" => sha}]}}] = FakePostHog.messages(fake) assert sha == sha256(@pdf) end + # The quota leaves room for one upload of each file. test "uploads a file attached twice once, and sends both references", %{ name: name, fake: fake } do + FakePostHog.set_upload_quota(fake, byte_size(@pdf) + byte_size(@logo)) + assert {:ok, _} = Emails.send( name, @@ -853,7 +880,6 @@ defmodule PostHog.EmailsTest do ]) ) - assert [_one_put] = FakePostHog.received(fake, :storage) assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} assert [{_id, email}] = FakePostHog.messages(fake) @@ -864,6 +890,9 @@ defmodule PostHog.EmailsTest do email["attachments"] assert sha == sha256(@pdf) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "logo.png", content: @logo}])) + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf, sha256(@logo) => @logo} end test "sends nothing after a failed upload, and returns the storage's answer", %{ @@ -903,8 +932,8 @@ defmodule PostHog.EmailsTest do refute inspect(error) =~ "signed-secret" refute inspect(error) =~ @pdf + assert FakePostHog.files(fake) == %{sha256(@logo) => @logo} assert FakePostHog.messages(fake) == [] - assert FakePostHog.received(fake, :send) == [] end @tag config: [ @@ -951,6 +980,7 @@ defmodule PostHog.EmailsTest do assert FakePostHog.files(fake) == %{sha256(file) => file} end + # The refusal is the storage's next answer only, so a retry would succeed. test "doesn't retry an upload the storage refused", %{name: name, fake: fake} do FakePostHog.fail_next(fake, :storage, {403, "AccessDenied", "Request has expired."}) @@ -961,8 +991,7 @@ defmodule PostHog.EmailsTest do status_code: 403 }} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [_one_put] = FakePostHog.received(fake, :storage) - assert FakePostHog.messages(fake) == [] + assert nothing_sent?(fake) end test "a storage 401 doesn't blame the PostHog key", %{name: name, fake: fake} do @@ -985,8 +1014,7 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "reach files.example.com directly" assert error.field == "attachments__0" refute inspect(error) =~ "signed-secret" - assert FakePostHog.files(fake) == %{} - assert FakePostHog.received(fake, :send) == [] + assert nothing_sent?(fake) end test "a send whose instance stops between its steps sends nothing more", %{ @@ -1004,8 +1032,7 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "client_closed"} = error} = Task.await(sending) assert error.message =~ "no email was sent" - assert FakePostHog.received(fake, :storage) == [] - assert FakePostHog.received(fake, :send) == [] + assert nothing_sent?(fake) end test "a template send takes empty lists as left out", %{name: name, fake: fake} do @@ -1023,16 +1050,18 @@ defmodule PostHog.EmailsTest do refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(email, &1)) end + # The quota leaves room for one upload of each file. test "a send retry doesn't upload again", %{name: name, fake: fake} do + FakePostHog.set_upload_quota(fake, byte_size(@pdf) + byte_size(@logo)) FakePostHog.fail_next(fake, :send, {503, %{"code" => "service_unavailable"}}) assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [_first, _retry] = FakePostHog.received(fake, :send) - assert [_one_ask] = FakePostHog.received(fake, :uploads) - assert [_one_put] = FakePostHog.received(fake, :storage) assert [{^id, _email}] = FakePostHog.messages(fake) + + assert {:ok, _} = Emails.send(name, with_files([%{filename: "logo.png", content: @logo}])) + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf, sha256(@logo) => @logo} end test "returns PostHog's refusal of the uploads, naming the attachment", %{ @@ -1088,8 +1117,7 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "authentication_failed", field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert FakePostHog.received(fake, :storage) == [] - assert FakePostHog.received(fake, :send) == [] + assert nothing_sent?(fake) end test "a PostHog without the upload route sends nothing", %{name: name, fake: fake} do @@ -1102,9 +1130,10 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "not_found", status_code: 404, field: "attachments"}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert FakePostHog.received(fake, :send) == [] + assert nothing_sent?(fake) end + # Each refusal is the upload route's next answer only, so a retry succeeds. test "doesn't retry the daily upload quota, and retries unavailable storage", %{ name: name, fake: fake @@ -1118,17 +1147,15 @@ defmodule PostHog.EmailsTest do assert {:error, %Error{code: "upload_quota_exceeded", status_code: 429, field: nil}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - assert [_one_request] = FakePostHog.received(fake, :uploads) - assert FakePostHog.messages(fake) == [] + assert nothing_sent?(fake) FakePostHog.fail_next(fake, :uploads, {503, %{"code" => "delivery_unavailable"}}) assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) - # The quota request, then the unavailable one and its retry. - assert [_quota, _unavailable, _retry] = FakePostHog.received(fake, :uploads) assert [{^id, _email}] = FakePostHog.messages(fake) + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} end test "an answer the SDK can't use sends nothing", %{name: name, fake: fake} do @@ -1149,11 +1176,10 @@ defmodule PostHog.EmailsTest do Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) end - assert FakePostHog.received(fake, :storage) == [] - assert FakePostHog.received(fake, :send) == [] + assert nothing_sent?(fake) end - test "refuses more than 10 files before any request", %{name: name, fake: fake} do + test "refuses more than 10 files, and sends nothing", %{name: name, fake: fake} do files = for i <- 1..11, do: %{filename: "#{i}.txt", content: "file #{i}"} assert {:error, @@ -1164,10 +1190,10 @@ defmodule PostHog.EmailsTest do } = error} = Emails.send(name, with_files(files)) assert error.fix =~ "10" - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end - test "refuses an email over 40 MB after Base64 encoding before any request", %{ + test "refuses an email over 40 MB after Base64 encoding, and sends nothing", %{ name: name, fake: fake } do @@ -1187,7 +1213,7 @@ defmodule PostHog.EmailsTest do assert error.fix =~ "29 MB" end - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) FakePostHog.put_files(fake, [file]) @@ -1206,12 +1232,21 @@ defmodule PostHog.EmailsTest do files = [%{filename: "a.bin", content: :binary.copy(<<1>>, 28_000_000)}] html_only = Keyword.put(with_files(files), :html, html) - assert {:error, %Error{code: "message_too_large", field: "attachments"}} = + assert {:error, %Error{code: "message_too_large", field: "attachments"} = error} = Emails.send(name, html_only) - assert FakePostHog.received(fake, :uploads) == [] + assert error.fix =~ ":text" + assert nothing_sent?(fake) - assert {:ok, %Emails.SendResult{}} = Emails.send(name, html_only ++ [text: "Thanks"]) + with_text = html_only ++ [text: "Thanks"] + larger_files = [%{filename: "a.bin", content: :binary.copy(<<1>>, 29_500_000)}] + + assert {:error, %Error{code: "message_too_large"} = error} = + Emails.send(name, Keyword.put(with_text, :attachments, larger_files)) + + refute error.fix =~ ":text" + + assert {:ok, %Emails.SendResult{}} = Emails.send(name, with_text) end test "uploads nothing to a plain-HTTP storage URL", %{name: name, fake: fake} do @@ -1229,8 +1264,7 @@ defmodule PostHog.EmailsTest do assert error.message =~ "a.pdf" assert error.fix =~ "https" refute inspect(error) =~ "files.example.com" - assert FakePostHog.received(fake, :storage) == [] - assert FakePostHog.messages(fake) == [] + assert nothing_sent?(fake) end @tag config: [ @@ -1252,7 +1286,7 @@ defmodule PostHog.EmailsTest do test "leaves the size of an email without files to PostHog", %{test: test, fake: fake} do {:ok, server} = - start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) + start_supervised({Bandit, plug: {&FakePostHog.serve/2, fake}, ip: :loopback, port: 0}) {:ok, {_ip, port}} = ThousandIsland.listener_info(server) host = "http://127.0.0.1:#{port}" @@ -1301,7 +1335,7 @@ defmodule PostHog.EmailsTest do end assert {:error, %Error{code: "message_too_large"}} = send_file.(unquote(refused)) - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) FakePostHog.put_files(fake, [:binary.copy(<<0>>, unquote(fits))]) @@ -1336,7 +1370,7 @@ defmodule PostHog.EmailsTest do Emails.send(name, with_files(attachments)) end - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end test "asks for atom keys when an attachment has string keys", %{name: name, fake: fake} do @@ -1352,7 +1386,7 @@ defmodule PostHog.EmailsTest do with_files([%{"filename" => "a.txt", "content" => "a", "cid" => "logo"}]) ) - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end test "refuses an attachment key that has no string form", %{name: name, fake: fake} do @@ -1362,7 +1396,7 @@ defmodule PostHog.EmailsTest do Emails.send(name, with_files([attachment])) assert error.message =~ "{:extra, :field}" - refute FakePostHog.received_any?(fake) + assert nothing_sent?(fake) end test "a path points to reading the file", %{name: name} do @@ -2127,7 +2161,7 @@ defmodule PostHog.EmailsReqDefaultsTest do test "an upload ignores the app's Req defaults", %{name: name, fake: fake} do {:ok, server} = - start_supervised({Bandit, plug: {FakePostHog.Server, fake}, ip: :loopback, port: 0}) + start_supervised({Bandit, plug: {&FakePostHog.serve/2, fake}, ip: :loopback, port: 0}) {:ok, {_ip, port}} = ThousandIsland.listener_info(server) FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{port}") From d4dabd7ef12dc26e830d7074d55a4cb4bba879e9 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:33:14 +0000 Subject: [PATCH 36/38] fix(emails): read only the start of the storage's answer to an upload An upload now keeps at most 16 KB of the storage's answer, enough for an S3 error, and stops reading there. A storage that answers without end can no longer fill the app's memory or hold the upload until its deadline. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 15 +++++++- test/posthog/emails_test.exs | 67 +++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index 67cc2a0d..db30d56a 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -191,6 +191,8 @@ defmodule PostHog.Emails do "https://eu.posthog.com" => @eu_app_host } @max_retry_delay_ms 30_000 + # Enough of the storage's answer for an S3 error, whose XML is well under 1 KB. + @max_storage_answer_bytes 16_384 @log_metadata [posthog_skip_capture: true] @shared_fields [:to, :distinct_id, :idempotency_key, :max_retries] @@ -838,7 +840,8 @@ defmodule PostHog.Emails do # A bare request: not through the api_client, whose defaults, such as headers a custom client # adds, are meant for PostHog, and without Req's steps and the app's Req.default_options, such as # an auth or compress_body set for other requests. The storage gets the bytes and the headers - # PostHog signed the upload with, nothing else. + # PostHog signed the upload with, nothing else. It reads only the start of the storage's answer, + # so a storage that answers without end can't fill the app's memory. defp send_upload(connection, upload, content, timeout_ms) do request = Req.Request.new( @@ -848,6 +851,7 @@ defmodule PostHog.Emails do body: content, options: [receive_timeout: timeout_ms] ) + |> Map.put(:into, &keep_start/2) |> with_upload_adapter(connection.upload_adapter) run_request(connection, timeout_ms, fn -> @@ -858,6 +862,15 @@ defmodule PostHog.Emails do end) end + defp keep_start({:data, data}, {request, response}) do + body = response.body <> data + + if byte_size(body) < @max_storage_answer_bytes, + do: {:cont, {request, %{response | body: body}}}, + else: + {:halt, {request, %{response | body: binary_part(body, 0, @max_storage_answer_bytes)}}} + end + # Req's own default adapter, which differs between Req versions, unless a test sets one. defp with_upload_adapter(request, :none), do: request diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index f6fc9e0b..00615473 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -109,7 +109,8 @@ defmodule PostHog.EmailsTest.FakePostHog do @doc """ Answers the route's next request with `reply` instead: `{status, json}` on PostHog's routes, and `{status, s3_code, s3_message}` or a transport error's reason, such as `:econnrefused`, on - the storage. + the storage. `{status, s3_code, s3_message, :endless}` follows the storage's error with + padding that doesn't end until the client stops reading, a little every 10 ms. """ def fail_next(fake, route, reply), do: script(fake, route, {:reply, reply}) @@ -350,6 +351,9 @@ defmodule PostHog.EmailsTest.FakePostHog do } case receive_request(fake, :storage, request) do + {{:reply, {status, code, message, :endless}}, _test} -> + endless_s3_error(conn, status, code, message) + {{:reply, {status, code, message}}, _test} -> s3_error(conn, status, code, message) @@ -422,12 +426,35 @@ defmodule PostHog.EmailsTest.FakePostHog do defp s3_error(conn, status, code, message) do conn |> Plug.Conn.put_resp_content_type("application/xml") - |> Plug.Conn.send_resp( - status, - ~s(\n#{code}#{message}/team/1/file) - ) + |> Plug.Conn.send_resp(status, s3_error_xml(code, message)) + end + + defp endless_s3_error(conn, status, code, message) do + conn = + conn + |> Plug.Conn.put_resp_content_type("application/xml") + |> Plug.Conn.send_chunked(status) + + {:ok, conn} = Plug.Conn.chunk(conn, s3_error_xml(code, message)) + pad(conn, :binary.copy(" ", 65_536), 3_000) + end + + # Stops when the client closes the connection, or after 30 seconds. + defp pad(conn, _padding, 0), do: conn + + defp pad(conn, padding, chunks_left) do + Process.sleep(10) + + case Plug.Conn.chunk(conn, padding) do + {:ok, conn} -> pad(conn, padding, chunks_left - 1) + {:error, _closed} -> conn + end end + defp s3_error_xml(code, message), + do: + ~s(\n#{code}#{message}/team/1/file) + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) defp checksum(sha256), do: sha256 |> Base.decode16!(case: :lower) |> Base.encode64() end @@ -980,6 +1007,36 @@ defmodule PostHog.EmailsTest do assert FakePostHog.files(fake) == %{sha256(file) => file} end + # Reading the whole answer would outlast the upload's deadline of about a second. + @tag upload_adapter: :none + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + app_host: "http://localhost:8010", + email_request_timeout_ms: 1_000 + ] + test "reads only the start of a storage answer that doesn't end", %{name: name, fake: fake} do + {:ok, server} = + start_supervised({Bandit, plug: {&FakePostHog.serve/2, fake}, ip: :loopback, port: 0}) + + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{port}") + + FakePostHog.fail_next( + fake, + :storage, + {403, "AccessDenied", "Request has expired.", :endless} + ) + + assert {:error, %Error{code: "attachment_upload_failed", status_code: 403} = error} = + Emails.send( + name, + with_files([%{filename: "a.pdf", content: @pdf}]) ++ [max_retries: 0] + ) + + assert error.message =~ "AccessDenied: Request has expired" + assert nothing_sent?(fake) + end + # The refusal is the storage's next answer only, so a retry would succeed. test "doesn't retry an upload the storage refused", %{name: name, fake: fake} do FakePostHog.fail_next(fake, :storage, {403, "AccessDenied", "Request has expired."}) From 3ffb13d621a81800d0aa7b27badb474b6e6ea5ac Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:59:44 +0000 Subject: [PATCH 37/38] test(emails): prove an upload never follows the storage's redirect The upload is a bare Req request without Req's steps, so it never follows a redirect, which would carry the file to another URL. A test has the storage answer the PUT with a 307 and a 308 to another host. The other host gets nothing, the upload fails on the attachment without naming the new URL, and nothing is sent. With Req's redirect step, the test fails. The docs say so. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.ex | 8 ++++-- test/posthog/emails_test.exs | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/lib/posthog/emails.ex b/lib/posthog/emails.ex index db30d56a..fb4df524 100644 --- a/lib/posthog/emails.ex +++ b/lib/posthog/emails.ex @@ -105,7 +105,8 @@ defmodule PostHog.Emails do Uploads go only over https, so a file and its signed URL never travel unencrypted. When PostHog names a plain-HTTP upload URL, the SDK uploads nothing, sends nothing, and returns `api_error` on the attachment. An `:app_host` on plain HTTP, as in local development, allows - plain-HTTP uploads too. + plain-HTTP uploads too. An upload never follows a redirect, which would carry the file to + another URL: the storage's redirect fails the upload with `attachment_upload_failed`. Your app must be able to reach the storage host that PostHog's answer names directly: an upload doesn't go through a proxy, and it ignores your app's `Req.default_options`, which are @@ -840,8 +841,9 @@ defmodule PostHog.Emails do # A bare request: not through the api_client, whose defaults, such as headers a custom client # adds, are meant for PostHog, and without Req's steps and the app's Req.default_options, such as # an auth or compress_body set for other requests. The storage gets the bytes and the headers - # PostHog signed the upload with, nothing else. It reads only the start of the storage's answer, - # so a storage that answers without end can't fill the app's memory. + # PostHog signed the upload with, nothing else. Without Req's steps it never follows a redirect, + # so the file goes only to the URL PostHog signed. It reads only the start of the storage's + # answer, so a storage that answers without end can't fill the app's memory. defp send_upload(connection, upload, content, timeout_ms) do request = Req.Request.new( diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 00615473..81817c0e 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -40,6 +40,16 @@ defmodule PostHog.EmailsTest.HeaderedClient do defdelegate request(client, method, url, opts), to: Client end +defmodule PostHog.EmailsTest.OtherHost do + @moduledoc false + # A host the storage redirects to. It tells the test about every request it gets. + + def serve(conn, test) do + send(test, {:other_host, conn.method, conn.request_path}) + Plug.Conn.send_resp(conn, 200, "") + end +end + defmodule PostHog.EmailsTest.FakePostHog do @moduledoc false # PostHog's email routes and its file storage as they behave, behind the SDK's real requests. @@ -111,6 +121,7 @@ defmodule PostHog.EmailsTest.FakePostHog do and `{status, s3_code, s3_message}` or a transport error's reason, such as `:econnrefused`, on the storage. `{status, s3_code, s3_message, :endless}` follows the storage's error with padding that doesn't end until the client stops reading, a little every 10 ms. + `{:redirect, status, location}` redirects the storage's next request to `location`. """ def fail_next(fake, route, reply), do: script(fake, route, {:reply, reply}) @@ -351,6 +362,11 @@ defmodule PostHog.EmailsTest.FakePostHog do } case receive_request(fake, :storage, request) do + {{:reply, {:redirect, status, location}}, _test} -> + conn + |> Plug.Conn.put_resp_header("location", location) + |> Plug.Conn.send_resp(status, "") + {{:reply, {status, code, message, :endless}}, _test} -> endless_s3_error(conn, status, code, message) @@ -1007,6 +1023,44 @@ defmodule PostHog.EmailsTest do assert FakePostHog.files(fake) == %{sha256(file) => file} end + @tag upload_adapter: :none + @tag config: [ + api_client_module: PostHog.EmailsTest.ReqTestClient, + app_host: "http://localhost:8010" + ] + test "doesn't follow the storage's redirect with the file", %{name: name, fake: fake} do + {:ok, storage} = + start_supervised({Bandit, plug: {&FakePostHog.serve/2, fake}, ip: :loopback, port: 0}) + + {:ok, other_host} = + start_supervised( + Supervisor.child_spec( + {Bandit, + plug: {&PostHog.EmailsTest.OtherHost.serve/2, self()}, ip: :loopback, port: 0}, + id: :other_host + ) + ) + + {:ok, {_ip, storage_port}} = ThousandIsland.listener_info(storage) + {:ok, {_ip, other_port}} = ThousandIsland.listener_info(other_host) + FakePostHog.set_storage_url(fake, "http://127.0.0.1:#{storage_port}") + elsewhere = "http://127.0.0.1:#{other_port}/elsewhere" + + for status <- [307, 308] do + FakePostHog.fail_next(fake, :storage, {:redirect, status, elsewhere}) + + assert {:error, %Error{code: "attachment_upload_failed", status_code: ^status} = error} = + Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) + + assert error.field == "attachments__0" + assert error.message =~ "a.pdf" + refute inspect(error) =~ "elsewhere" + refute inspect(error) =~ "signed-secret" + refute_received {:other_host, _method, _path} + assert nothing_sent?(fake) + end + end + # Reading the whole answer would outlast the upload's deadline of about a second. @tag upload_adapter: :none @tag config: [ From f02dc0b5e626862b617a2d91d6d19c6b140afc0b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:24:04 +0000 Subject: [PATCH 38/38] test(emails): drop the upload quota from the fake PostHog PostHog charges the daily upload quota when it signs the uploads, not when the storage keeps a file, so a fake that charged stored files proved "uploads once" through behaviour PostHog doesn't have. The fake now has no quota. The tests assert what a caller sees: a file PostHog has goes by reference, a file attached twice is two references to one stored file, and a retried send goes out once. Refs PostHog/michaelr-context#86 Co-Authored-By: Claude Opus 5.5 --- test/posthog/emails_test.exs | 78 ++++++++---------------------------- 1 file changed, 17 insertions(+), 61 deletions(-) diff --git a/test/posthog/emails_test.exs b/test/posthog/emails_test.exs index 81817c0e..b012dbef 100644 --- a/test/posthog/emails_test.exs +++ b/test/posthog/emails_test.exs @@ -57,9 +57,7 @@ defmodule PostHog.EmailsTest.FakePostHog do # only when its upload matches what was signed, as S3 checks a presigned PUT: the signed URL, # the signed headers, no second credential or unsigned x-amz- header, and the bytes. The send # route accepts an email only when PostHog has every file it names and the email fits PostHog's - # 40 MB estimate. Like PostHog, it can hold a project to a daily upload quota: the upload route - # refuses to sign uploads beyond what is left of it, and every file the storage keeps counts - # against it. A test reads back the stored files, the accepted emails and what the send route + # 40 MB estimate. A test reads back the stored files, the accepted emails and what the send route # received, and can make a route's next request fail, wait or hold. # # The routes are :uploads and :send on PostHog, and :storage. Uploads are signed for @@ -82,8 +80,6 @@ defmodule PostHog.EmailsTest.FakePostHog do files: %{}, signed: %{}, messages: [], - upload_quota: nil, - uploaded_bytes: 0, received: %{uploads: [], storage: [], send: []}, next: %{uploads: [], storage: [], send: []} } @@ -110,9 +106,6 @@ defmodule PostHog.EmailsTest.FakePostHog do def delete_file(fake, bytes), do: Agent.update(fake, &%{&1 | files: Map.delete(&1.files, sha256(bytes))}) - @doc "Holds the project to a daily upload quota of `bytes`, counting the files stored today." - def set_upload_quota(fake, bytes), do: Agent.update(fake, &%{&1 | upload_quota: bytes}) - @doc "Signs later uploads for the storage at `url`." def set_storage_url(fake, url), do: Agent.update(fake, &%{&1 | storage_url: url}) @@ -232,43 +225,20 @@ defmodule PostHog.EmailsTest.FakePostHog do end defp sign_uploads(conn, %{"files" => files}, fake) when is_list(files) do - case Agent.get_and_update(fake, &sign_within_quota(files, &1)) do - {:ok, answers} -> - json(conn, 200, %{"files" => answers}) + answers = + Agent.get_and_update( + fake, + &Enum.map_reduce(files, &1, fn file, state -> sign(file, state) end) + ) - :quota_exceeded -> - json( - conn, - 429, - refusal( - "upload_quota_exceeded", - "This project uploaded as many file bytes as it can today.", - nil - ) - ) - end + json(conn, 200, %{"files" => answers}) end defp sign_uploads(conn, _request, _fake), do: json(conn, 400, refusal("required", "This field is required.", "files")) - # Each file to upload counts in full, before any is signed. - defp sign_within_quota(files, state) do - to_upload = - for %{"size" => size, "sha256" => sha256} <- files, not stored?(state, sha256), do: size - - if state.upload_quota && state.uploaded_bytes + Enum.sum(to_upload) > state.upload_quota do - {:quota_exceeded, state} - else - {answers, state} = Enum.map_reduce(files, state, &sign/2) - {{:ok, answers}, state} - end - end - - defp stored?(state, sha256), do: Map.has_key?(state.files, sha256) - defp sign(%{"sha256" => sha256, "size" => size}, state) do - if stored?(state, sha256) do + if Map.has_key?(state.files, sha256) do {%{"sha256" => sha256, "status" => "present", "expires_at" => "2026-12-16T09:00:00Z"}, state} else @@ -407,13 +377,7 @@ defmodule PostHog.EmailsTest.FakePostHog do s3_error(conn, 400, "BadDigest", "The SHA256 you specified did not match.") true -> - Agent.update(fake, fn state -> - %{ - state - | files: Map.put(state.files, signed.sha256, request.body), - uploaded_bytes: state.uploaded_bytes + byte_size(request.body) - } - end) + Agent.update(fake, &put_in(&1.files[signed.sha256], request.body)) Plug.Conn.send_resp(conn, 200, "") end @@ -895,10 +859,9 @@ defmodule PostHog.EmailsTest do assert [{^id, _email}] = FakePostHog.messages(fake) end - # With no upload quota left, any upload would be refused. - test "uploads nothing when PostHog has every file", %{name: name, fake: fake} do + # PostHog answers present with no upload URL, so the file can only be sent by reference. + test "sends a file PostHog already has by reference", %{name: name, fake: fake} do FakePostHog.put_files(fake, [@pdf]) - FakePostHog.set_upload_quota(fake, 0) assert {:ok, _} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) @@ -907,13 +870,10 @@ defmodule PostHog.EmailsTest do assert sha == sha256(@pdf) end - # The quota leaves room for one upload of each file. - test "uploads a file attached twice once, and sends both references", %{ + test "sends a file attached twice as two references to one stored file", %{ name: name, fake: fake } do - FakePostHog.set_upload_quota(fake, byte_size(@pdf) + byte_size(@logo)) - assert {:ok, _} = Emails.send( name, @@ -933,9 +893,6 @@ defmodule PostHog.EmailsTest do email["attachments"] assert sha == sha256(@pdf) - - assert {:ok, _} = Emails.send(name, with_files([%{filename: "logo.png", content: @logo}])) - assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf, sha256(@logo) => @logo} end test "sends nothing after a failed upload, and returns the storage's answer", %{ @@ -1161,18 +1118,17 @@ defmodule PostHog.EmailsTest do refute Enum.any?(["reply_to", "cc", "bcc", "attachments"], &Map.has_key?(email, &1)) end - # The quota leaves room for one upload of each file. - test "a send retry doesn't upload again", %{name: name, fake: fake} do - FakePostHog.set_upload_quota(fake, byte_size(@pdf) + byte_size(@logo)) + test "retries an unavailable send with files, and the email goes out once", %{ + name: name, + fake: fake + } do FakePostHog.fail_next(fake, :send, {503, %{"code" => "service_unavailable"}}) assert {:ok, %Emails.SendResult{id: id}} = Emails.send(name, with_files([%{filename: "a.pdf", content: @pdf}])) assert [{^id, _email}] = FakePostHog.messages(fake) - - assert {:ok, _} = Emails.send(name, with_files([%{filename: "logo.png", content: @logo}])) - assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf, sha256(@logo) => @logo} + assert FakePostHog.files(fake) == %{sha256(@pdf) => @pdf} end test "returns PostHog's refusal of the uploads, naming the attachment", %{