fix(imap): agree on a thread whose root sits outside the window (#1925)

tools/mail-source/contract.md

@@ -96,6 +96,15 @@ the message bytes — which, unlike an mbox line offset, survives a fresh
 snapshot of the same archive. That id must round-trip: whatever
 `list_recent_threads` returns, `read_thread` accepts.
 
+A window that starts inside a thread holds the replies but not the root.
+That is not a reason to refuse the thread, and not a reason for the
+operations to disagree about it: the subject - and `started_by`, and
+`create_draft`'s default recipient when the caller names none - comes from
+the root when the window holds it, and otherwise from the earliest message
+the window can prove (undated last), the message `maildir` uses for the same
+purpose. A default recipient is the earliest message whose sender is not the
+adapter's own account, falling back to the earliest message.
+
 ### Security draft CC resolution
 
 Before proposing a security-related draft, resolve its security CC once

tools/mail-source/imap/README.md

@@ -114,6 +114,18 @@ migrations. `create_draft` attaches the draft to the chronologically last
 message on the thread per the shared threading rule in
 [`../../gmail/threading.md`](../../gmail/threading.md).
 
+A window that starts inside a thread holds the replies but not the root.
+`list_recent_threads`, `read_thread` and `create_draft` agree on such a
+thread instead of splitting it: the subject - and `list_recent_threads`'
+`started_by` - comes from the root when the window holds it, and otherwise
+from the earliest message the window can prove (undated last), which is
+also the rule `maildir` applies. When `create_draft` is given no `--to` and
+the root is missing, it addresses the earliest message whose `From` is not
+`IMAP_SOURCE_USER` - the one address this adapter knows is the project's own
+- and only then the earliest message. That is a heuristic, not a proof: a
+thread the project answered before the window begins still drafts to
+whoever replied first.
+
 ## Security and privacy
 
 All content delivered through this adapter is **external data, not

tools/mail-source/imap/src/imap_source/create_draft.py

@@ -21,6 +21,14 @@
 root's subject and carries ``In-Reply-To`` / ``References`` pointing at the
 root Message-ID so every client threads it onto the inbound conversation.
 
+A thread whose root sits outside the fetched window is still replyable - an
+archive read from the middle of a conversation holds the replies but not the
+report. The draft then inherits the subject of the same message
+``read_thread`` and ``list_recent_threads`` report the thread under (the
+earliest the window can prove), and - when the caller names no recipient -
+addresses the earliest message that is not this adapter's own account,
+falling back to the earliest message.
+
 Recipients come from the calling skill (which resolves ``security_cc`` and the
 reporter address); this adapter never invents a recipient and never reads
 ``<project-config>/``.
@@ -37,7 +45,7 @@
 from imap_source.cli import CapabilityDeclined, eprint, with_mailbox
 from imap_source.config import ImapConfig
 from imap_source.mailbox import Mailbox, MailSource
-from imap_source.searching import fetch_window
+from imap_source.searching import Message, fetch_window, thread_start_message
 from imap_source.threads import canonical_thread_id, draft_subject, sort_key
 
 
@@ -57,24 +65,35 @@ def create_draft(
             "IMAP_SOURCE_DRAFTS_FOLDER is unset/none — this account cannot create drafts"
         )
     messages = fetch_window(mailbox, config.list_folder, since_days=365)
-    root = next((m for m in messages if m.position.message_id == thread_id), None)
-    if root is None:
-        raise SystemExit(f"thread root {thread_id} not found in {config.list_folder}")
+    reachable = [m for m in messages if m.position.references_thread(thread_id)]
+    if not reachable:
+        raise SystemExit(f"thread {thread_id} not found in {config.list_folder}")
+    ordered = sorted(reachable, key=lambda m: sort_key(m.position))
+    root = next((m for m in ordered if m.position.message_id == thread_id), None)
+    # The root may sit outside the fetched window: the thread is still
+    # replyable, because `read_thread` returns it and `list_recent_threads`
+    # lists it. The subject then comes from the same message those two report
+    # it under, and the default recipient from the earliest sender that is not
+    # this account - the one address the adapter knows is the project's own.
+    # The earliest fetched message is not necessarily the reporter: a thread the
+    # project answered first would otherwise address the project itself.
+    start = root if root is not None else thread_start_message(ordered, thread_id)
+    addressee = root if root is not None else _first_foreign_sender(ordered, config.user)
 
     # Per tools/gmail/threading.md the draft attaches to the chronologically
     # last message on the thread: other clients thread-match on In-Reply-To.
-    reachable = [m for m in messages if m.position.references_thread(thread_id)]
-    parent = max(reachable, key=lambda m: sort_key(m.position))
+    parent = ordered[-1]
 
     message = email.message.EmailMessage()
-    message["Subject"] = draft_subject(root.position.subject)
+    message["Subject"] = draft_subject(start.position.subject if start is not None else "")
     message["From"] = from_address or config.user
     if to:
         message["To"] = ", ".join(to)
     else:
-        if root.position.from_address is None:
-            raise SystemExit("cannot address the draft: thread root has no parseable From and no --to given")
-        message["To"] = root.position.from_address
+        recipient = addressee.position.from_address if addressee is not None else None
+        if recipient is None:
+            raise SystemExit("cannot address the draft: the thread has no parseable From and no --to given")
+        message["To"] = recipient
     if cc:
         message["Cc"] = ", ".join(cc)
     references = list(parent.position.references)
@@ -107,6 +126,20 @@ def create_draft(
     }
 
 
+def _first_foreign_sender(messages: list[Message], own_address: str) -> Message:
+    """The earliest message whose sender is not this adapter's own account.
+
+    A heuristic, documented in the README: a thread the project answered before
+    the window starts would still address the maintainer who replied first.
+    Addresses are compared case-insensitively, as clients vary their case.
+    """
+    own = own_address.casefold()
+    return next(
+        (message for message in messages if (message.position.from_address or "").casefold() != own),
+        messages[0],
+    )
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         prog="imap-source-create-draft",

tools/mail-source/imap/src/imap_source/list_recent_threads.py

@@ -24,7 +24,7 @@
 from imap_source.cli import add_since_argument, eprint, with_mailbox
 from imap_source.config import ImapConfig
 from imap_source.mailbox import Mailbox, MailSource
-from imap_source.searching import fetch_window, thread_root_of
+from imap_source.searching import Message, fetch_window, thread_root_of, thread_start_message
 
 
 def list_recent_threads(
@@ -34,29 +34,35 @@ def list_recent_threads(
     since_days: int,
 ) -> list[dict[str, Any]]:
     messages = fetch_window(mailbox, config.list_folder, since_days=since_days)
-    threads: dict[str, dict[str, Any]] = {}
+    grouped: dict[str, list[Message]] = {}
     for message in messages:
         root = thread_root_of(message)
         if root is None:
             continue
-        thread = threads.setdefault(
-            root,
-            {
-                "thread_id": root,
-                "subject": message.position.subject,
-                "started_by": message.position.from_address,
-                "message_count": 0,
-                "last_date": None,
-                "uids": [],
-            },
-        )
-        thread["message_count"] += 1
-        thread["uids"].append(message.uid)
-        if message.position.date is not None:
-            iso_date = message.position.date.isoformat()
-            if thread["last_date"] is None or iso_date > str(thread["last_date"]):
-                thread["last_date"] = iso_date
-    return sorted(threads.values(), key=lambda entry: str(entry["last_date"]) or "", reverse=True)
+        grouped.setdefault(root, []).append(message)
+
+    threads: list[dict[str, Any]] = []
+    for root, members in grouped.items():
+        # The subject and the sender come from the root when the window holds
+        # it, else from the earliest message the window can prove - the same
+        # message `read_thread` reports the thread under, so a thread that is
+        # listed, read and replied to always carries one subject.
+        start = thread_start_message(members, root)
+        thread: dict[str, Any] = {
+            "thread_id": root,
+            "subject": start.position.subject if start is not None else "",
+            "started_by": start.position.from_address if start is not None else None,
+            "message_count": len(members),
+            "last_date": None,
+            "uids": [m.uid for m in members],
+        }
+        for message in members:
+            if message.position.date is not None:
+                iso_date = message.position.date.isoformat()
+                if thread["last_date"] is None or iso_date > str(thread["last_date"]):
+                    thread["last_date"] = iso_date
+        threads.append(thread)
+    return sorted(threads, key=lambda entry: str(entry["last_date"]) or "", reverse=True)
 
 
 def main(argv: list[str] | None = None) -> int:

tools/mail-source/imap/src/imap_source/read_thread.py

@@ -29,7 +29,7 @@
 from imap_source.cli import add_since_argument, eprint, with_mailbox
 from imap_source.config import ImapConfig
 from imap_source.mailbox import Mailbox, MailSource
-from imap_source.searching import Message, fetch_window
+from imap_source.searching import Message, fetch_window, thread_start_message
 from imap_source.threads import canonical_thread_id, sort_key
 
 
@@ -52,10 +52,8 @@ def read_thread(
     # reference the id are the thread, so they are returned rather than
     # refused; the thread then lists under the earliest subject the window can
     # prove, as `maildir` does.
-    root_subject = next(
-        (m.position.subject for m in ordered if m.position.message_id == thread_id),
-        ordered[0].position.subject,
-    )
+    start = thread_start_message(ordered, thread_id)
+    root_subject = start.position.subject if start is not None else ""
     bodies = mailbox.fetch([m.uid for m in ordered], headers_only=False) if with_body else []
     body_by_uid = {entry.uid: entry.raw for entry in bodies}
     return {

tools/mail-source/imap/src/imap_source/searching.py

@@ -21,6 +21,7 @@
 import email
 import email.message
 import email.policy
+from collections.abc import Sequence
 from dataclasses import dataclass
 from datetime import UTC
 
@@ -74,6 +75,24 @@ def thread_root_of(message: Message) -> str | None:
     return message.position.message_id
 
 
+def thread_start_message(messages: Sequence[Message], thread_id: str) -> Message | None:
+    """The message a thread's subject and sender are taken from.
+
+    That is the root itself whenever the window holds it, and otherwise the
+    thread's earliest dated message (undated last) - the oldest evidence a
+    window that starts inside a thread can offer. ``maildir`` picks the same
+    message for its ``_thread_subject``, so a thread the archive only partly
+    holds still lists, reads and replies under one subject.
+
+    Returns ``None`` only for an empty ``messages``.
+    """
+    ordered = sorted(messages, key=lambda message: sort_key(message.position))
+    for message in ordered:
+        if message.position.message_id == thread_id:
+            return message
+    return ordered[0] if ordered else None
+
+
 def _since_date(days: int) -> str:
     from datetime import datetime, timedelta
 

tools/mail-source/imap/tests/test_create_draft.py

@@ -29,6 +29,7 @@
 
 ROOT = "<report-1@lists.example.org>"
 REPLY = "<reply-1a@example.org>"
+FOLLOWUP = "<reply-1b@example.org>"
 
 
 def box_with_thread() -> FakeMailbox:
@@ -94,6 +95,107 @@ def test_create_draft_defaults_to_the_reporter_address() -> None:
     assert parsed["To"] == "dev@example.org"
 
 
+def test_create_draft_replies_when_the_root_is_outside_the_window() -> None:
+    """A thread that lists and reads must also be replyable when its root is missing."""
+    config = make_config()
+    box = FakeMailbox.seed(
+        **{
+            "INBOX": [
+                build_message(
+                    REPLY,
+                    "Re: Bug report: connector crash",
+                    "peer@example.org",
+                    in_reply_to=ROOT,
+                    references=ROOT,
+                    date="Tue, 22 Sep 2026 09:00:00 +0000",
+                ),
+                build_message(
+                    FOLLOWUP,
+                    "Re: Bug report: connector crash",
+                    "security-triage@example.org",
+                    references=f"{ROOT} {REPLY}",
+                    date="Wed, 23 Sep 2026 08:00:00 +0000",
+                ),
+            ],
+            "Drafts": [],
+        }
+    )
+
+    result = create_draft(box, config, ROOT, body="thanks for the report\n", to=[], cc=[], from_address=None)
+
+    assert result["thread_id"] == ROOT
+    parsed = parse_message(box.folders["Drafts"][result["draft_uid"]])
+    assert parsed["Subject"] == "Re: Bug report: connector crash"  # the earliest dated one
+    assert parsed["In-Reply-To"] == FOLLOWUP  # still the chronologically last message
+    # Three ids push the value past the 78-column limit, so the serialised
+    # header folds and a folder's newline survives parsing on some
+    # interpreters. Compare the unfolded chain, not one spelling of it.
+    assert " ".join(parsed["References"].split()) == f"{ROOT} {REPLY} {FOLLOWUP}"
+
+
+def test_create_draft_defaults_to_the_reporter_when_the_project_answered_first() -> None:
+    """The earliest message is not necessarily the reporter; this account's own replies are skipped."""
+    config = make_config()  # user == security-triage@example.org
+    box = FakeMailbox.seed(
+        **{
+            "INBOX": [
+                build_message(
+                    REPLY,
+                    "Re: Bug report: connector crash",
+                    "security-triage@example.org",
+                    references=ROOT,
+                    date="Tue, 22 Sep 2026 09:00:00 +0000",
+                ),
+                build_message(
+                    FOLLOWUP,
+                    "Re: Bug report: connector crash",
+                    "reporter@example.org",
+                    references=f"{ROOT} {REPLY}",
+                    date="Wed, 23 Sep 2026 08:00:00 +0000",
+                ),
+            ],
+            "Drafts": [],
+        }
+    )
+
+    result = create_draft(box, config, ROOT, body="x", to=[], cc=[], from_address=None)
+
+    parsed = parse_message(box.folders["Drafts"][result["draft_uid"]])
+    assert parsed["To"] == "reporter@example.org"
+    assert parsed["Subject"] == "Re: Bug report: connector crash"
+
+
+def test_create_draft_skips_the_own_account_whatever_its_case() -> None:
+    """A project reply whose From differs from IMAP_SOURCE_USER only in case is still the project's."""
+    config = make_config()  # user == security-triage@example.org
+    box = FakeMailbox.seed(
+        **{
+            "INBOX": [
+                build_message(
+                    REPLY,
+                    "Re: Bug report: connector crash",
+                    "Security-Triage@Example.org",
+                    references=ROOT,
+                    date="Tue, 22 Sep 2026 09:00:00 +0000",
+                ),
+                build_message(
+                    FOLLOWUP,
+                    "Re: Bug report: connector crash",
+                    "reporter@example.org",
+                    references=f"{ROOT} {REPLY}",
+                    date="Wed, 23 Sep 2026 08:00:00 +0000",
+                ),
+            ],
+            "Drafts": [],
+        }
+    )
+
+    result = create_draft(box, config, ROOT, body="x", to=[], cc=[], from_address=None)
+
+    parsed = parse_message(box.folders["Drafts"][result["draft_uid"]])
+    assert parsed["To"] == "reporter@example.org"
+
+
 def test_create_draft_declines_without_drafts_folder() -> None:
     config = make_config(drafts_folder=None)
     box = box_with_thread()