diff --git a/.github/workflows/email_call.yaml b/.github/workflows/email_call.yaml deleted file mode 100644 index 593ab8cd..00000000 --- a/.github/workflows/email_call.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Sends an email from the bioimageiobot - -name: email - -on: - workflow_dispatch: # dipatch only for testing - inputs: - user_email: - description: "Email address of user" - required: true - type: string - message: - description: "email body" - required: false - type: string - - workflow_call: - inputs: - user_email: - description: "Email address of user" - required: true - type: string - message: - description: "email body" - required: false - type: string - -jobs: - send-email: - runs-on: ubuntu-latest - steps: - - name: Send mail - uses: dawidd6/action-send-mail@v3 - with: - # Specify connection via URL (replaces server_address, server_port, secure, - # username and password) - # - # Format: - # - # * smtp://user:password@server:port - # * smtp+starttls://user:password@server:port - connection_url: ${{secrets.MAIL_CONNECTION}} - # Required mail server address if not connection_url: - server_address: smtp.gmail.com - # Server port, default 25: - server_port: 465 - # Optional whether this connection use TLS (default is true if server_port is 465) - secure: true - # Optional (recommended) mail server username: - username: bioimageiobot@gmail.com - # Optional (recommended) mail server password: - password: ${{secrets.MAIL_PASSWORD}} - # Required mail subject: - subject: Github Actions job result - # Required recipients' addresses: - to: ${{ inputs.user_email }} - # Required sender full name (address can be skipped): - from: bioimageiobot@gmail.com - # Optional plain body: - body: ${{ inputs.message }} - # Optional HTML body read from file: - # html_body: file://README.html - # Optional carbon copy recipients: - # cc: kyloren@example.com,leia@example.com - # Optional blind carbon copy recipients: - # bcc: r2d2@example.com,hansolo@example.com - # Optional recipient of the email response: - # reply_to: luke@example.com - # Optional Message ID this message is replying to: - # in_reply_to: - # Optional unsigned/invalid certificates allowance: - ignore_cert: true - # Optional converting Markdown to HTML (set content_type to text/html too): - convert_markdown: true - # Optional attachments: - # attachments: # TODO Can we attach a test log? - # Optional priority: 'high', 'normal' (default) or 'low' - # priority: low diff --git a/.github/workflows/emails_to_chat.yaml b/.github/workflows/emails_to_chat.yaml new file mode 100644 index 00000000..03cbdf0d --- /dev/null +++ b/.github/workflows/emails_to_chat.yaml @@ -0,0 +1,27 @@ +name: forward emails to chat + +on: + schedule: + - cron: "0 * * * *" # every hour at minute 0 + +concurrency: forward-emails-to-chat + +env: + S3_HOST: ${{vars.S3_HOST}} + S3_BUCKET: ${{vars.S3_BUCKET}} + S3_FOLDER: ${{vars.S3_FOLDER}} + S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} + S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} + MAIL_PASSWORD: ${{secrets.MAIL_PASSWORD}} + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" # caching pip dependencies + - run: pip install . + - run: backoffice forward-emails-to-chat diff --git a/.github/workflows/publish_call.yaml b/.github/workflows/publish_call.yaml index 1f4cb6c0..f809333f 100644 --- a/.github/workflows/publish_call.yaml +++ b/.github/workflows/publish_call.yaml @@ -46,8 +46,3 @@ jobs: cache: "pip" # caching pip dependencies - run: pip install . - run: backoffice publish "${{ inputs.resource_id }}" "staged/${{ inputs.stage_number }}" - # - name: Publish to Zenodo - # run: | - # python .github/scripts/update_status.py "${{ inputs.resource_path }}" "Publishing to Zenodo" "5" - # python .github/scripts/upload_model_to_zenodo.py --resource_path "${{inputs.resource_path}}" - # python .github/scripts/update_status.py "${{ inputs.resource_path }}" "Publishing complete" "6" diff --git a/backoffice/_backoffice.py b/backoffice/_backoffice.py index 52c9701f..7ac85f61 100644 --- a/backoffice/_backoffice.py +++ b/backoffice/_backoffice.py @@ -9,6 +9,7 @@ from backoffice.backup import backup from backoffice.generate_collection_json import generate_collection_json from backoffice.gh_utils import set_gh_actions_outputs +from backoffice.mailroom import forward_emails_to_chat, notify_uploader from backoffice.remote_resource import ( PublishedVersion, RemoteResource, @@ -90,6 +91,12 @@ def await_review(self, resource_id: str, version: str): f"Cannot await review for already published {resource_id} {version}" ) rv.await_review() + notify_uploader( + rv, + "is awaiting review ⌛", + f"Thank you for proposing {rv.id} {rv.version}!\n" + + "Our maintainers will take a look shortly!", + ) def request_changes(self, resource_id: str, version: str, reason: str): """mark a (staged) resource version as needing changes""" @@ -100,6 +107,13 @@ def request_changes(self, resource_id: str, version: str, reason: str): ) rv.request_changes(reason=reason) + notify_uploader( + rv, + "needs changes 📑", + f"Thank you for proposing {rv.id} {rv.version}!\n" + + "We kindly ask you to upload an updated version, because: \n" + + f"{reason}\n", + ) def publish(self, resource_id: str, version: str): """publish a (staged) resource version""" @@ -111,6 +125,13 @@ def publish(self, resource_id: str, version: str): published: PublishedVersion = rv.publish() assert isinstance(published, PublishedVersion) + self.generate_collection_json() + notify_uploader( + rv, + "was published! 🎉", + f"Thank you for contributing {published.id} {published.version} to bioimage.io!\n" + + "Check it out at https://bioimage.io/#/?id={published.id}\n", # TODO: link to version + ) def backup(self, destination: Optional[str] = None): """backup the whole collection (to zenodo.org)""" @@ -121,3 +142,6 @@ def generate_collection_json( ): """generate the collection.json file --- a summary of the whole collection""" generate_collection_json(self.client, collection_template=collection_template) + + def forward_emails_to_chat(self): + forward_emails_to_chat(self.client, last_n_days=7) diff --git a/backoffice/mailroom/__init__.py b/backoffice/mailroom/__init__.py new file mode 100644 index 00000000..0bc685b1 --- /dev/null +++ b/backoffice/mailroom/__init__.py @@ -0,0 +1,3 @@ +from ._forward_emails_to_chat import forward_emails_to_chat as forward_emails_to_chat +from ._send_email import notify_uploader as notify_uploader +from ._send_email import send_email as send_email diff --git a/backoffice/mailroom/_forward_emails_to_chat.py b/backoffice/mailroom/_forward_emails_to_chat.py new file mode 100644 index 00000000..eb3d41bd --- /dev/null +++ b/backoffice/mailroom/_forward_emails_to_chat.py @@ -0,0 +1,162 @@ +import email.message +import email.parser +import imaplib +import os +from contextlib import contextmanager +from datetime import datetime, timedelta +from email.utils import parsedate_to_datetime +from typing import Any + +from dotenv import load_dotenv +from loguru import logger + +from backoffice.mailroom.constants import ( + BOT_EMAIL, + IMAP_PORT, + REPLY_HINT, + SMTP_SERVER, + STATUS_UPDATE_SUBJECT, +) +from backoffice.remote_resource import get_remote_resource_version +from backoffice.s3_client import Client +from backoffice.s3_structure.chat import Chat, Message + +_ = load_dotenv() + + +FORWARDED_TO_CHAT_FLAT = "forwarded-to-bioimageio-chat" + + +def forward_emails_to_chat(s3_client: Client, last_n_days: int): + cutoff_datetime = datetime.now().astimezone() - timedelta(days=last_n_days) + with _get_imap_client() as imap_client: + _update_chats(s3_client, imap_client, cutoff_datetime) + + +@contextmanager +def _get_imap_client(): + imap_client = imaplib.IMAP4_SSL(SMTP_SERVER, IMAP_PORT) + _ = imap_client.login(BOT_EMAIL, os.environ["MAIL_PASSWORD"]) + yield imap_client + _ = imap_client.logout() + + +def _get_body(msg: email.message.Message): + if msg.is_multipart(): + for part in msg.walk(): + ctype = part.get_content_type() + cdispo = str(part.get("Content-Disposition")) + + # skip any text/plain (txt) attachments + if ctype == "text/plain" and "attachment" not in cdispo: + msg_part = part + break + else: + logger.error("faild to get body from multipart message: {}", msg) + return None + else: + # not multipart - i.e. plain text, no attachments, keeping fingers crossed + msg_part = msg + + msg_bytes = msg_part.get_payload(decode=True) + try: + body = str(msg_bytes, "utf-8") # pyright: ignore[reportArgumentType] + except Exception as e: + logger.error("failed to decode email body: {}", e) + return None + else: + return body + + +def _update_chats( + s3_client: Client, imap_client: imaplib.IMAP4_SSL, cutoff_datetime: datetime +): + _ = imap_client.select("inbox") + for msg_id, rid, rv, msg, dt in _iterate_relevant_emails( + imap_client, cutoff_datetime + ): + ok, flag_data = imap_client.fetch(str(msg_id), "(FLAGS)") + if ok != "OK" or len(flag_data) != 1: + logger.error("failed to get flags for {}", msg_id) + continue + try: + assert isinstance(flag_data[0], bytes), type(flag_data[0]) + flags = str(flag_data[0], "utf-8") + except Exception as e: + logger.error("failed to interprete flags '{}': {}", flag_data[0], e) + continue + + if FORWARDED_TO_CHAT_FLAT in flags: + continue # already processed + + body = _get_body(msg) + if body is None: + continue + + sender = msg["from"] + text = "[forwarded from email]\n" + body.replace("> " + REPLY_HINT, "").replace( + REPLY_HINT, "" + ) + rr = get_remote_resource_version(s3_client, rid, rv) + if not rr.exists(): + logger.error("Cannot comment on non-existing resource {} {}", rid, rv) + continue + + rr.extend_chat(Chat(messages=[Message(author=sender, text=text, timestamp=dt)])) + _ = imap_client.store(str(msg_id), "+FLAGS", FORWARDED_TO_CHAT_FLAT) + + +def _iterate_relevant_emails(imap_client: imaplib.IMAP4_SSL, cutoff_datetime: datetime): + for msg_id, msg, dt in _iterate_emails(imap_client, cutoff_datetime): + subject = str(msg["subject"]) + if STATUS_UPDATE_SUBJECT not in subject: + logger.debug("ignoring subject: '{}'", subject) + continue + + try: + _, id_version = subject.split(STATUS_UPDATE_SUBJECT) + resource_id, resource_version = id_version.strip().split(" ") + except Exception: + logger.warning("failed to process subject: {}", subject) + continue + + yield msg_id, resource_id, resource_version, msg, dt + + +def _iterate_emails(imap_client: imaplib.IMAP4_SSL, cutoff_datetime: datetime): + data = imap_client.search(None, "ALL") + mail_ids = data[1] + id_list = mail_ids[0].split() + first_email_id = int(id_list[0]) + latest_email_id = int(id_list[-1]) + + for msg_id in range(latest_email_id, first_email_id, -1): + ok, msg_data = imap_client.fetch(str(msg_id), "(RFC822)") + if ok != "OK": + logger.error("failed to fetch email {}", msg_id) + continue + + parts = [p for p in msg_data if isinstance(p, tuple)] + if len(parts) == 0: + logger.error("found email with without any parts") + continue + elif len(parts) > 1: + logger.error( + "found email with multiple parts. I'll only look at the first part" + ) + + _, msg_part = parts[0] + + msg = email.message_from_string(str(msg_part, "utf-8")) + dt: Any = parsedate_to_datetime(msg["date"]) + if isinstance(dt, datetime): + if dt < cutoff_datetime: + break + else: + logger.error("failed to parse email datetime '{}'", msg["date"]) + + yield msg_id, msg, dt + + +if __name__ == "__main__": + forward_emails_to_chat(Client(), 7) diff --git a/backoffice/mailroom/_send_email.py b/backoffice/mailroom/_send_email.py new file mode 100644 index 00000000..fb5a9757 --- /dev/null +++ b/backoffice/mailroom/_send_email.py @@ -0,0 +1,57 @@ +import os +import smtplib +from email.mime.text import MIMEText + +from dotenv import load_dotenv +from loguru import logger + +from backoffice.mailroom.constants import ( + BOT_EMAIL, + REPLY_HINT, + SMTP_PORT, + SMTP_SERVER, + STATUS_UPDATE_SUBJECT, +) +from backoffice.remote_resource import PublishedVersion, StagedVersion + +_ = load_dotenv() + + +def notify_uploader(rv: StagedVersion | PublishedVersion, subject_end: str, msg: str): + email, name = rv.get_uploader() + if email is None: + logger.error("missing uploader email for {} {}", rv.id, rv.version) + else: + send_email( + subject=f"{STATUS_UPDATE_SUBJECT}{rv.id} {rv.version} {subject_end.strip()}", + body=( + f"Dear {name},\n" + + f"{msg.strip()}\n" + + "Kind regards,\n" + + "The bioimage.io bot 🦒\n" + + REPLY_HINT + ), + recipients=[email], + ) + + +def send_email(subject: str, body: str, recipients: list[str]): + from_addr = BOT_EMAIL + to_addr = ", ".join(recipients) + msg = MIMEText(body) + msg["From"] = from_addr + msg["To"] = to_addr + msg["Subject"] = subject + with smtplib.SMTP_SSL(SMTP_SERVER, SMTP_PORT) as smtp_server: + _ = smtp_server.login(BOT_EMAIL, os.environ["MAIL_PASSWORD"]) + _ = smtp_server.sendmail(BOT_EMAIL, recipients, msg.as_string()) + + logger.info("Email '{}' sent to {}", subject, recipients) + + +if __name__ == "__main__": + send_email( + subject=STATUS_UPDATE_SUBJECT + " lazy-bug staged/2", + body="Staged version 2 of your model 'lazy-bug' is now under review.", + recipients=["thefynnbe@gmail.com"], + ) diff --git a/backoffice/mailroom/constants.py b/backoffice/mailroom/constants.py new file mode 100644 index 00000000..691e1e0c --- /dev/null +++ b/backoffice/mailroom/constants.py @@ -0,0 +1,6 @@ +BOT_EMAIL = "bioimageiobot@gmail.com" +IMAP_PORT = 993 +SMTP_PORT = 465 +SMTP_SERVER = "smtp.gmail.com" +STATUS_UPDATE_SUBJECT = "bioimage.io status update: " +REPLY_HINT = "IMPORTANT NOTE: Any reply to this email will be published as a public comment on the respective bioimage.io resource version." diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 2ba4687e..e3313da8 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -5,7 +5,7 @@ import zipfile from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Generic, Optional, Type, TypeVar +from typing import Generic, NamedTuple, Optional, Type, TypeVar from bioimageio.spec.utils import identify_bioimageio_yaml_file_name from loguru import logger @@ -130,6 +130,11 @@ def _extend_json(self, extension: JsonFileT, path: str): self.client.put_pydantic(path, current) +class Uploader(NamedTuple): + email: Optional[str] + name: str + + @dataclass class RemoteResourceVersion(RemoteResource, Generic[NumberT], ABC): """Base class for a resource version (`StagedVersion` or `PublishedVersion`)""" @@ -171,6 +176,28 @@ def extend_log( """extend log file""" self._extend_version_specific_json(extension) + def extend_chat( + self, + extension: Chat, + ): + """extend chat file""" + self._extend_version_specific_json(extension) + + def get_uploader(self): + rdf = yaml.load(self.client.load_file(f"{self.folder}files/rdf.yaml")) + try: + uploader = rdf["uploader"] + email = uploader["email"] + name = uploader.get( + "name", f"{rdf.get('type', 'bioimage.io resource')} contributor" + ) + except Exception as e: + logger.error("failed to extract uploader from rdf: {}", e) + email = None + name = "bioimage.io resource contributor" + + return Uploader(email=email, name=name) + @dataclass class StagedVersion(RemoteResourceVersion[StageNumber]): @@ -187,10 +214,10 @@ def version_prefix(self): def unpack(self, package_url: str): # ensure we have a chat.json - self._extend_version_specific_json(self._get_version_specific_json(Chat)) + self.extend_chat(Chat()) # ensure we have a logs.json - self._extend_version_specific_json(self._get_version_specific_json(Logs)) + self.extend_log(Logs()) # set first status (this also write versions.json) self._set_status( diff --git a/backoffice/validate_format.py b/backoffice/validate_format.py index 924cf83e..dbf6ceb0 100644 --- a/backoffice/validate_format.py +++ b/backoffice/validate_format.py @@ -225,6 +225,20 @@ def validate_format(staged: StagedVersion): staged.set_testing_status("Validating RDF format") rdf_source = staged.rdf_url rd = load_description(rdf_source, format_version="discover") + if not isinstance(rd, InvalidDescr): + rd.validation_summary.add_detail( + ValidationDetail( + name="Check that uploader is specified", + status="failed" if rd.uploader is None else "passed", + errors=[ + ErrorEntry( + loc=("uploader", "email"), + msg="missing uploader email", + type="error", + ) + ], + ) + ) dynamic_test_cases: list[dict[Literal["weight_format"], WeightsFormat]] = [] conda_envs: dict[WeightsFormat, CondaEnv] = {} if not isinstance(rd, InvalidDescr):