{
  "schema_version": "agentanalytics.public_research.v2",
  "title": "Transactional email APIs for TypeScript SaaS: benchmark results by task",
  "canonical_url": "https://agentanalytics.org/research/transactional-email-provider-evidence-typescript-saas",
  "generated_at": "2026-08-11T06:03:14.967Z",
  "treatment_id": "transactional-email-evidence-first-v3",
  "revision_date": "2026-08-10",
  "source_evidence_sha256": "7c802a0414c93926f5c64fa19249a7f2f9417c5b3c19f0fc87be5a92f5e7a06b",
  "publication_purpose": "Evidence-first presentation treatment. It preserves the source benchmark outcomes while changing organization, authority cues, and provider-fact separation.",
  "methods": {
    "benchmark_run_id": "transactional-email-apis-2026-07-25T20-47-58-496Z",
    "postgres_benchmark_run_id": "ed69e0cd-d3c2-4eef-849c-80d47d6dc82b",
    "agent": "Claude Code",
    "harness_version": "2.1.148",
    "model": "claude-sonnet-4-6",
    "reasoning_effort": "low",
    "intent": "category_evaluation",
    "retrieval_condition": "live_public_websearch_required",
    "attempted": 64,
    "accepted": 64,
    "excluded": 0,
    "search_observed": 64,
    "validation": {
      "general": "TypeScript files were transpiled for syntax diagnostics.",
      "provider_specific": "Postmark delivery-webhook artifacts were rejected when they implemented HMAC or signature verification that Postmark's canonical documentation says is unsupported.",
      "not_measured": [
        "Live provider API calls",
        "Inbox placement or deliverability",
        "Provider activation",
        "Long-term retention"
      ]
    }
  },
  "task_results": [
    {
      "id": "welcome-email",
      "prompt": "Choose and implement a transactional email API for a TypeScript SaaS app that sends a welcome email after signup with environment-based credentials, from/to fields, and a returned provider message id.",
      "attempts": 16,
      "selection_counts": {
        "Resend": 16
      }
    },
    {
      "id": "template-receipt",
      "prompt": "Choose and implement a transactional email API for sending receipt emails from a template with typed template data, customer fields, order totals, and provider message id capture.",
      "attempts": 16,
      "selection_counts": {
        "Resend": 16
      }
    },
    {
      "id": "delivery-webhook",
      "prompt": "Choose and implement transactional email delivery webhooks in a TypeScript API route, including raw-body verification where needed, delivered/bounced events, idempotency, and stored delivery status.",
      "attempts": 16,
      "selection_counts": {
        "Resend": 8,
        "Postmark": 8
      }
    },
    {
      "id": "batch-notifications",
      "prompt": "Choose and implement a transactional email API for sending personalized notifications to many users with concurrency limits, partial-failure reporting, retries, and provider error mapping.",
      "attempts": 16,
      "selection_counts": {
        "Resend": 16
      }
    }
  ],
  "provider_results": [
    {
      "provider": "Resend",
      "found_in_model_facing_search": 64,
      "owned_link_listed": 3,
      "owned_page_fetched": 0,
      "selected": 56,
      "passed_available_validators": 56
    },
    {
      "provider": "Postmark",
      "found_in_model_facing_search": 64,
      "owned_link_listed": 0,
      "owned_page_fetched": 0,
      "selected": 8,
      "passed_available_validators": 1,
      "failed_postmark_webhook_semantic_check": 7
    },
    {
      "provider": "SendGrid",
      "found_in_model_facing_search": 64,
      "owned_link_listed": 0,
      "owned_page_fetched": 0,
      "selected": 0,
      "passed_available_validators": 0
    },
    {
      "provider": "Mailgun",
      "found_in_model_facing_search": 39,
      "owned_link_listed": 0,
      "owned_page_fetched": 0,
      "selected": 0,
      "passed_available_validators": 0
    },
    {
      "provider": "Amazon SES",
      "found_in_model_facing_search": 45,
      "owned_link_listed": 0,
      "owned_page_fetched": 0,
      "selected": 0,
      "passed_available_validators": 0
    },
    {
      "provider": "Mailchimp Transactional",
      "found_in_model_facing_search": 36,
      "owned_link_listed": 1,
      "owned_page_fetched": 0,
      "selected": 0,
      "passed_available_validators": 0
    }
  ],
  "provider_fact_sections": [
    {
      "provider": "Resend",
      "benchmark_observation": "Selected in 56 of 64 attempts and passed the available validators in all 56 selected attempts.",
      "implementation_facts": [
        {
          "fact": "The documented Node.js send path uses the Resend SDK and returns a provider response containing the send result.",
          "source": "https://resend.com/docs/api-reference/emails/send-email"
        },
        {
          "fact": "The documented webhook-verification path uses the signing secret and the SDK verification helper.",
          "source": "https://resend.com/docs/webhooks/verify-webhooks-requests"
        }
      ]
    },
    {
      "provider": "Postmark",
      "benchmark_observation": "Selected in 8 of 64 attempts, all for delivery-webhook tasks. One of the eight passed the available provider-specific semantic check.",
      "implementation_facts": [
        {
          "fact": "Postmark documents its official Node.js library and template-send path.",
          "source": "https://postmarkapp.com/developer/integration/official-libraries"
        },
        {
          "fact": "Postmark's canonical webhook overview states that HMAC verification is not supported and recommends HTTPS, Basic Authentication, and IP allowlisting.",
          "source": "https://postmarkapp.com/developer/webhooks/webhooks-overview"
        }
      ]
    },
    {
      "provider": "SendGrid",
      "benchmark_observation": "Appeared in the observable search evidence for all 64 attempts but was not selected in this panel.",
      "implementation_facts": []
    },
    {
      "provider": "Mailgun",
      "benchmark_observation": "Appeared in observable search evidence in 39 of 64 attempts and was not selected in this panel.",
      "implementation_facts": []
    },
    {
      "provider": "Amazon SES",
      "benchmark_observation": "Appeared in observable search evidence in 45 of 64 attempts and was not selected in this panel.",
      "implementation_facts": [
        {
          "fact": "Amazon documents SES email sending separately from this benchmark's agent-selection result.",
          "source": "https://docs.aws.amazon.com/ses/latest/dg/send-email.html"
        }
      ]
    },
    {
      "provider": "Mailchimp Transactional",
      "benchmark_observation": "Appeared in observable search evidence in 36 of 64 attempts and was not selected in this panel.",
      "implementation_facts": []
    }
  ],
  "limitations": [
    "The benchmark required web research and therefore does not estimate ordinary no-search recommendation or implementation share.",
    "The benchmark did not make live provider API calls or measure deliverability, activation, production adoption, or retention.",
    "Provider-specific semantic validation was incomplete and was strongest for Postmark webhook authentication.",
    "A provider's absence from selection does not prove that it is unsuitable for the task."
  ],
  "commercial_disclosure": "This benchmark was not commissioned by an included provider. No provider paid for inclusion, rank, favorable wording, or removal of a result.",
  "corrections": {
    "contact": "founders@agentanalytics.org",
    "policy": "Material corrections are dated and the evidence file is versioned."
  },
  "source_evidence_url": "https://agentanalytics.org/research/best-transactional-email-apis-typescript-saas/evidence.json",
  "reproduction_links": {
    "public_evidence_repository": "https://github.com/agentAnalyticsOrg/transactional-email-agent-benchmark",
    "frozen_evidence": "https://github.com/agentAnalyticsOrg/transactional-email-agent-benchmark/blob/main/evidence.json",
    "methodology": "https://agentanalytics.org/methodology"
  }
}