Merge pull request #1875 from tomusher/feature/mailchimp-integration

Replace Campaign Monitor with Mailchimp

home/management/commands/sync_campaign_monitor.py → home/management/commands/sync_mailchimp.py

@@ -1,5 +1,5 @@
 from django.core.management.base import BaseCommand
-from subscribe.campaign_monitor import update_segments
+from subscribe.mailchimp import update_segments
 
 class Command(BaseCommand):
 

home/wagtail_hooks.py

@@ -37,8 +37,8 @@ def register_command_urls():
 @hooks.register("register_admin_menu_item")
 def register_commands_menu_item():
     sync_menu_item = AdminOnlyMenuItem(
-        "Sync Campaign Monitor",
-        reverse("campaign_monitor:sync"),
+        "Sync Mailchimp",
+        reverse("mailchimp:sync"),
         classname="icon icon-mail",
         order=10,
     )

newamericadotorg/api/subscribe/views.py

@@ -6,7 +6,7 @@
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 
-from subscribe.campaign_monitor import update_subscriber
+from subscribe.mailchimp import update_subscriber
 from subscribe.models import MailingListSegment
 
 
@@ -29,24 +29,16 @@ def subscribe(request):
         return Response({"status": "UNVERIFIED"})
 
     subscriptions = params.getlist("subscriptions[]", [])
-    subscription_titles = MailingListSegment.objects.filter(
+    subscription_titles = list(MailingListSegment.objects.filter(
         pk__in=subscriptions,
-    ).values_list('title', flat=True)
-    job_title = params.get("job_title", None)
-    org = params.get("organization", None)
+    ).values_list('title', flat=True))
+
     zipcode = params.get("zipcode", None)
-    custom_fields = []
+    custom_fields = {}
 
-    if job_title:
-        custom_fields.append({"key": "JobTitle", "value": job_title})
-    if org:
-        custom_fields.append({"key": "Organization", "value": org})
     if zipcode:
-        custom_fields.append({"key": "MailingZip/PostalCode", "value": zipcode})
-    if subscription_titles:
-        for s in subscription_titles:
-            custom_fields.append({"key": "Subscriptions", "value": s})
+        custom_fields['ZIP'] = zipcode
 
-    status = update_subscriber(params.get("email"), params.get("name"), custom_fields)
+    status = update_subscriber(params.get("email"), params.get("name"), subscription_titles, custom_fields)
 
     return Response({"status": status})

newamericadotorg/assets/react/home-panels/pages/Subscribe.js

@@ -27,8 +27,6 @@ export class HomeSubscribe extends Subscribe {
               <div className="subscribe__fields__sticky-wrapper">
                 <Text name="email" label="Email" value={params.email} onChange={this.change} />
                 <Text name="name" label="First Name & Last Name" value={params.name} onChange={this.change} />
-                <Text name="organization" label="Organization" value={params.organization} onChange={this.change} required={false} />
-                <Text name="job_title" label="Job Title" value={params.job_title} onChange={this.change} required={false} />
                 <Text name="zipcode" label="Zipcode" value={params.zipcode} onChange={this.change} required={false} />
                 <Recaptcha
                   ref={e => recaptchaInstance = e}

newamericadotorg/assets/react/program-page/components/Subscribe.js

@@ -68,8 +68,6 @@ export default class Subscribe extends Component {
       params: {
         email: email || '',
         name: '',
-        organization: '',
-        job_title: '',
         zipcode: '',
       },
       subscriptions
@@ -210,8 +208,6 @@ export default class Subscribe extends Component {
               <div className={`subscribe__fields ${subscriptions.length > 1 && 'col-md-6'}`}>
                 <Text name="email" label="Email" value={params.email} onChange={this.change} />
                 <Text name="name" label="First Name & Last Name" value={params.name} onChange={this.change} />
-                <Text name="organization" label="Organization" value={params.organization} onChange={this.change} required={false} />
-                <Text name="job_title" label="Job Title" value={params.job_title} onChange={this.change} required={false} required={false} />
                 <Text name="zipcode" label="Zipcode" value={params.zipcode} onChange={this.change} required={false} />
               <Recaptcha
                 ref={e => recaptchaInstance = e}

newamericadotorg/settings/base.py

@@ -408,8 +408,8 @@
 CSP_REPORT_URI = os.environ.get('CSP_REPORT_URI')
 CSP_REPORT_PERCENTAGE = 0.05
 
-CREATESEND_API_KEY = os.getenv("CREATESEND_API_KEY")
-CREATESEND_CLIENTID = os.getenv("CREATESEND_CLIENTID")
-CREATESEND_LISTID = os.getenv("CREATESEND_LISTID")
+MAILCHIMP_HOST = os.getenv("MAILCHIMP_HOST")
+MAILCHIMP_API_KEY = os.getenv("MAILCHIMP_API_KEY")
+MAILCHIMP_LIST_ID = os.getenv("MAILCHIMP_LIST_ID")
 
 RECAPTCHA_SECRET_KEY = os.getenv("NEW_RECAPTCHA_SECRET_KEY")

pyproject.toml

@@ -25,7 +25,6 @@ dependencies = [
 
     # Other packages
     'Wand >= 0.6.11',
-    'createsend >= 7.0, <8',
     'psycopg2 >=2.8, <3',
     'WeasyPrint==51',
     'python-docx >= 0.8.11',

requirements.txt

@@ -32,8 +32,6 @@ cffi==1.15.1
     #   weasyprint
 charset-normalizer==3.1.0
     # via requests
-createsend==7.0.0
-    # via newamerica-cms (pyproject.toml)
 cssselect2==0.7.0
     # via
     #   cairosvg
@@ -167,7 +165,6 @@ sentry-sdk==1.19.1
     # via newamerica-cms (pyproject.toml)
 six==1.16.0
     # via
-    #   createsend
     #   html5lib
     #   l18n
     #   python-dateutil

subscribe/mailchimp.py

@@ -0,0 +1,143 @@
+import traceback
+import urllib.parse
+import requests
+import hashlib
+from enum import Enum
+from django.conf import settings
+from urllib.parse import urljoin
+import logging
+from subscribe.models import MailingListSegment
+
+logger = logging.getLogger(__name__)
+
+
+class StatusEnum(Enum):
+    OK = "OK"
+    BAD_REQUEST = "BAD_REQUEST"
+
+
+class MailchimpError(Exception):
+    pass
+
+
+class MailchimpClient:
+    def __init__(self, *, host, api_key):
+        self.api_key = api_key
+        self.host = host
+
+    def _build_url(self, path):
+        return urljoin(f"https://{self.host}", path)
+
+    def _do_request(self, method, path, **kwargs):
+        try:
+            response = requests.request(
+                method,
+                self._build_url(path),
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                timeout=10,
+                **kwargs,
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise MailchimpError(f"Request to Mailchimp failed") from e
+
+    def get(self, path):
+        return self._do_request("GET", path)
+
+    def post(self, path, data):
+        return self._do_request("POST", path, json=data)
+
+    def put(self, path, data):
+        return self._do_request("PUT", path, json=data)
+
+    def _get_subscriber_hash(self, email):
+        return hashlib.md5(email.lower().strip().encode()).hexdigest()
+
+    def get_tags(self, list_id):
+        tags = []
+        offset = 0
+        count = 100
+
+        while True:
+            response = self.get(
+                f"/3.0/lists/{list_id}/tag-search?count={count}&offset={offset}"
+            )
+            tags.extend(response["tags"])
+
+            total_items = response["total_items"]
+            if offset + count >= total_items:
+                break
+
+            offset += count
+
+        return tags
+
+    def subscribe(self, *, list_id, email, name, merge_fields, tags):
+        subscriber_hash = self._get_subscriber_hash(email)
+        return self.put(
+            f"/3.0/lists/{list_id}/members/{subscriber_hash}",
+            {
+                "email_address": email,
+                "status": "pending",
+                "merge_fields": merge_fields,
+                "tags": tags,
+            },
+        )
+
+    def get_subscriber(self, *, list_id, email):
+        return self.get(f"/3.0/lists/{list_id}/members/{email}")
+
+
+def _get_client():
+    host = settings.MAILCHIMP_HOST
+    api_key = settings.MAILCHIMP_API_KEY
+
+    if not host:
+        raise ValueError(
+            "MAILCHIMP_HOST is not set. Set this to the base API host for your Mailchimp account, for example: us22.api.mailchimp.com"
+        )
+    if not api_key:
+        raise ValueError("MAILCHIMP_API_KEY is not set. Set this to your Mailchimp API key.")
+
+    return MailchimpClient(host=host, api_key=api_key)
+
+
+def update_segments():
+    client = _get_client()
+    list_id = settings.MAILCHIMP_LIST_ID
+
+    tags = client.get_tags(list_id)
+    existing_tag_names = set(MailingListSegment.objects.values_list("title", flat=True))
+
+    # Process tags and create/update MailingListSegment objects
+    current_tag_names = set()
+    for tag in tags:
+        tag_name = tag["name"]
+        current_tag_names.add(tag_name)
+
+        MailingListSegment.objects.update_or_create(
+            title=tag_name,
+        )
+
+    # Delete MailingListSegment objects for tags that no longer exist
+    tags_to_delete = existing_tag_names - current_tag_names
+    MailingListSegment.objects.filter(title__in=tags_to_delete).delete()
+
+    return len(current_tag_names)
+
+
+def update_subscriber(email, name, tags, custom_fields):
+    client = _get_client()
+    try:
+        client.subscribe(
+            list_id=settings.MAILCHIMP_LIST_ID,
+            email=email,
+            name=name,
+            merge_fields=custom_fields,
+            tags=tags,
+        )
+        return StatusEnum.OK.value
+    except MailchimpError:
+        logger.warning("Failed to create subscriber in Mailchimp", exc_info=True)
+        return StatusEnum.BAD_REQUEST.value

subscribe/migrations/0005_rename_sync_permission.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.18 on 2024-09-05 08:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('subscribe', '0004_remove_legacy_subscribe_page_models'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='mailinglistsegment',
+            options={'permissions': [('can_sync_from_mailchimp', 'Can sync from Mailchimp')]},
+        ),
+    ]

subscribe/models.py

@@ -12,7 +12,7 @@ class MailingListSegment(models.Model):
 
     class Meta:
         permissions = [
-            ("can_sync_from_campaign_monitor", "Can sync from Campaign Monitor"),
+            ("can_sync_from_mailchimp", "Can sync from Mailchimp"),
         ]
 
     def __str__(self):

subscribe/templates/wagtailadmin/campaign_monitor_sync.html → subscribe/templates/wagtailadmin/mailchimp_sync.html

@@ -1,9 +1,9 @@
 {% extends "wagtailadmin/base.html" %}
 
-{% block titletag %}Commands: Sync Campaign Monitor{% endblock %}
+{% block titletag %}Commands: Sync Mailchimp{% endblock %}
 
 {% block content %}
-    {% include "wagtailadmin/shared/header.html" with title="Sync Campaign Monitor" icon="mail" %}
+    {% include "wagtailadmin/shared/header.html" with title="Sync Mailchimp" icon="mail" %}
     <section class="nice-padding">
         <form action="" method="post">{% csrf_token %}
             <input type="submit" value="Sync Now" class="button" />