tenant.openagents/omega

1#!/usr/bin/env python3
2"""
3Track the effectiveness of the duplicate-detection bot by classifying issues
4into outcome categories on a GitHub Projects v2 board.
5
6Subcommands:
7    classify-closed <issue_number> <closer_login> <state_reason>
8        Classify a closed issue and add it to the project board.
9
10    classify-open
11        Classify open, triaged, bot-commented issues and add them to
12        the project board as Noise.
13
14Requires:
15    requests (pip install requests)
16
17Environment variables:
18    GITHUB_TOKEN     - GitHub App token
19    PROJECT_NUMBER   - GitHub Projects v2 board number (default: 76, override for local testing)
20"""
21
22import argparse
23import functools
24import os
25import re
26import sys
27import time
28from datetime import datetime, timezone
29
30import requests
31
32GITHUB_API = "https://api.github.com"
33GRAPHQL_URL = "https://api.github.com/graphql"
34REPO_OWNER = "zed-industries"
35REPO_NAME = "zed"
36STAFF_TEAM_SLUG = "staff"
37BOT_LOGIN = "zed-community-bot[bot]"
38BOT_APP_SLUG = "zed-community-bot"
39# Strings that identify a comment posted by the duplicate-detection bot. Any
40# match counts as a bot comment for classification purposes. A single comment
41# can contain both markers (v3+ produces this when there are both confident
42# duplicates and lower-confidence triage context).
43BOT_COMMENT_MARKERS = (
44    "This issue appears to be a duplicate of",  # user-facing duplicate alert
45    "Additional recent context for triagers",  # v3+ collapsed triage section
46)
47BOT_START_DATE = "2026-02-18"
48NEEDS_TRIAGE_LABEL = "state:needs triage"
49DEFAULT_PROJECT_NUMBER = 76
50VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"}
51# HTTP statuses we'll retry on for GET requests
52TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504}
53# Add a new tuple when you deploy a new version of the bot that you want to
54# keep track of (e.g. the prompt gets a rewrite or the model gets swapped).
55# Newest first, please. The datetime is for the deployment time (merge to main).
56BOT_VERSION_TIMELINE = [
57    ("v4", datetime(2026, 6, 16, 12, 43, tzinfo=timezone.utc)),
58    ("v3", datetime(2026, 5, 25, 14, 30, tzinfo=timezone.utc)),
59    ("v2", datetime(2026, 2, 26, 14, 9, tzinfo=timezone.utc)),
60    ("v1", datetime(2026, 2, 18, tzinfo=timezone.utc)),
61]
62
63
64def bot_version_for_time(date_string):
65    """Return the bot version that was active at the given ISO 8601 timestamp."""
66    timestamp = datetime.fromisoformat(date_string.replace("Z", "+00:00"))
67    for version, deployed in BOT_VERSION_TIMELINE:
68        if timestamp >= deployed:
69            return version
70    return BOT_VERSION_TIMELINE[-1][0]
71
72
73def github_api_get(path, params=None):
74    """Fetch JSON from the GitHub REST API, retrying transient failures. Raises on non-2xx status."""
75    url = f"{GITHUB_API}/{path.lstrip('/')}"
76    for attempt in range(3):
77        try:
78            response = requests.get(url, headers=GITHUB_HEADERS, params=params)
79            response.raise_for_status()
80            return response.json()
81        except requests.RequestException as e:
82            transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or (
83                isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES
84            )
85            if not transient or attempt == 2:
86                raise
87            wait = 2 ** attempt
88            print(f"  Transient GitHub API error ({e}); retrying in {wait}s")
89            time.sleep(wait)
90
91
92def github_search_issues(query):
93    """Search issues, returning most recently created first."""
94    # not handling pagination on purpose: the oldest issues are on the board already
95    params = {"q": query, "sort": "created", "order": "desc", "per_page": 100}
96    return github_api_get("/search/issues", params).get("items", [])
97
98
99def is_staff_member(username):
100    """Check if user is an active member of the staff team."""
101    try:
102        data = github_api_get(
103            f"/orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/memberships/{username}"
104        )
105        return data.get("state") == "active"
106    except requests.HTTPError as error:
107        if error.response.status_code == 404:
108            return False
109        raise
110
111
112def fetch_issue(issue_number):
113    data = github_api_get(f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}")
114    return {
115        "number": issue_number,
116        "node_id": data["node_id"],
117        "author": (data.get("user") or {}).get("login", ""),
118        "type_name": (data.get("type") or {}).get("name"),
119        "created_at": data.get("created_at", ""),
120    }
121
122
123def is_bot_dupe_comment(body):
124    """True if the comment body looks like one posted by the duplicate-detection bot."""
125    return any(marker in body for marker in BOT_COMMENT_MARKERS)
126
127
128def get_bot_comment_with_time(issue_number):
129    """Get the bot's duplicate-detection comment and its timestamp from an issue.
130
131    Recognizes both the user-facing duplicate alert and the v3+ triage-only
132    comment formats. Returns {"body": str, "created_at": str} if found, else None.
133    """
134    comments_path = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments"
135    page = 1
136    while comments := github_api_get(comments_path, {"per_page": 100, "page": page}):
137        for comment in comments:
138            author = (comment.get("user") or {}).get("login", "")
139            body = comment.get("body", "")
140            if author == BOT_LOGIN and is_bot_dupe_comment(body):
141                return {"body": body, "created_at": comment.get("created_at", "")}
142        page += 1
143    return None
144
145
146def parse_suggested_issues(comment_body):
147    """Extract issue numbers from the bot's comment (lines like '- #12345')."""
148    return [int(match) for match in re.findall(r"^- #(\d+)", comment_body, re.MULTILINE)]
149
150
151def github_api_graphql(query, variables=None, partial_errors_ok=False):
152    """Execute a GitHub GraphQL query. Raises on errors unless partial_errors_ok is set."""
153    response = requests.post(
154        GRAPHQL_URL,
155        headers=GITHUB_HEADERS,
156        json={"query": query, "variables": variables or {}},
157    )
158    response.raise_for_status()
159    data = response.json()
160    if "errors" in data:
161        if not partial_errors_ok or "data" not in data:
162            raise RuntimeError(f"GraphQL errors: {data['errors']}")
163        print(f"  GraphQL partial errors (ignored): {data['errors']}")
164    return data["data"]
165
166
167def find_canonical_among(duplicate_number, candidates):
168    """Check if any candidate issue has duplicate_number marked as a duplicate.
169
170    The MarkedAsDuplicateEvent lives on the canonical issue's timeline, not the
171    duplicate's. So to find which canonical issue our duplicate was closed against,
172    we check each candidate's timeline for a MarkedAsDuplicateEvent whose
173    `duplicate` field matches our issue.
174
175    Returns the matching canonical issue number, or None.
176    """
177    if not candidates:
178        return None
179
180    # candidate issue numbers are baked into the query body via field aliases
181    # (GraphQL doesn't let you parametrize alias names), so $numbers isn't needed.
182    data = github_api_graphql(
183        """
184        query($owner: String!, $repo: String!) {
185          repository(owner: $owner, name: $repo) {
186            PLACEHOLDER
187          }
188        }
189        """.replace("PLACEHOLDER", "\n            ".join(
190            f'issue_{number}: issue(number: {number}) {{'
191            f' timelineItems(last: 50, itemTypes: [MARKED_AS_DUPLICATE_EVENT]) {{'
192            f' nodes {{ ... on MarkedAsDuplicateEvent {{ duplicate {{ ... on Issue {{ number }} }} }} }} }} }}'
193            for number in candidates
194        )),
195        {"owner": REPO_OWNER, "repo": REPO_NAME},
196        partial_errors_ok=True,
197    )
198
199    repo = data["repository"]
200    for candidate in candidates:
201        issue_data = repo.get(f"issue_{candidate}")
202        if not issue_data:
203            continue
204        for node in issue_data["timelineItems"]["nodes"]:
205            dup_number = (node.get("duplicate") or {}).get("number")
206            if dup_number == duplicate_number:
207                return candidate
208    return None
209
210
211@functools.lru_cache
212def get_project_config():
213    """Fetch the project board's ID, field IDs, and option IDs."""
214    data = github_api_graphql(
215        """
216        query($org: String!, $number: Int!) {
217          organization(login: $org) {
218            projectV2(number: $number) {
219              id
220              fields(first: 30) {
221                nodes {
222                  ... on ProjectV2SingleSelectField { id name options { id name } }
223                  ... on ProjectV2Field { id name }
224                }
225              }
226            }
227          }
228        }
229        """,
230        {"org": REPO_OWNER, "number": PROJECT_NUMBER},
231    )
232    project = data["organization"]["projectV2"]
233
234    config = {"project_id": project["id"], "fields": {}}
235    for field_node in project["fields"]["nodes"]:
236        name = field_node.get("name")
237        if not name:
238            continue
239        field_info = {"id": field_node["id"]}
240        if "options" in field_node:
241            field_info["options"] = {
242                option["name"]: option["id"] for option in field_node["options"]
243            }
244        config["fields"][name] = field_info
245
246    print(f"  Project config loaded: {len(config['fields'])} fields")
247    return config
248
249
250def find_project_item(issue_node_id):
251    """Check if an issue is already on our project board.
252
253    Returns the project item ID if found, or None.
254    """
255    data = github_api_graphql(
256        "query($id: ID!) { node(id: $id) { ... on Issue { projectItems(first: 20) { nodes { id project { number } } } } } }",
257        {"id": issue_node_id},
258    )
259    for item in data["node"]["projectItems"]["nodes"]:
260        if item["project"]["number"] == PROJECT_NUMBER:
261            return item["id"]
262    return None
263
264
265def add_project_item(issue_node_id):
266    """Add an issue to the project board. Returns the new item ID."""
267    config = get_project_config()
268    data = github_api_graphql(
269        """
270        mutation($projectId: ID!, $contentId: ID!) {
271          addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) {
272            item { id }
273          }
274        }
275        """,
276        {"projectId": config["project_id"], "contentId": issue_node_id},
277    )
278    return data["addProjectV2ItemById"]["item"]["id"]
279
280
281def set_field_value(item_id, field_name, value):
282    """Set a single field value on a project board item."""
283    config = get_project_config()
284    field = config["fields"].get(field_name)
285    if not field:
286        print(f"  Warning: field '{field_name}' not found on project board")
287        return
288
289    if "options" in field:
290        # single-select field
291        option_id = field["options"].get(value)
292        if not option_id:
293            print(f"  Warning: option '{value}' not found for field '{field_name}'")
294            return
295        field_value = {"singleSelectOptionId": option_id}
296    else:
297        # text field
298        field_value = {"text": str(value)}
299
300    github_api_graphql(
301        """
302        mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: ProjectV2FieldValue!) {
303          updateProjectV2ItemFieldValue(input: {
304            projectId: $projectId
305            itemId: $itemId
306            fieldId: $fieldId
307            value: $value
308          }) {
309            projectV2Item { id }
310          }
311        }
312        """,
313        {
314            "projectId": config["project_id"],
315            "itemId": item_id,
316            "fieldId": field["id"],
317            "value": field_value,
318        },
319    )
320
321
322def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="Auto-classified", notes=None, bot_comment_time=None):
323    """Add an issue to the project board (or update it if already there), setting field values."""
324    item_id = find_project_item(issue_node_id)
325    if item_id:
326        print(f"  Issue already on board, updating (item {item_id})")
327    else:
328        item_id = add_project_item(issue_node_id)
329        print(f"  Added to project board (item {item_id})")
330
331    set_field_value(item_id, "Outcome", outcome)
332    set_field_value(item_id, "Status", status)
333
334    if closed_as and closed_as in VALID_CLOSED_AS_VALUES:
335        set_field_value(item_id, "Closed as", closed_as)
336
337    if notes:
338        set_field_value(item_id, "Notes", notes)
339
340    if bot_comment_time:
341        set_field_value(item_id, "Bot version", bot_version_for_time(bot_comment_time))
342
343    return item_id
344
345
346def classify_closed(issue_number, closer_login, state_reason):
347    """Classify a closed issue and add/update it on the project board."""
348    state_reason = state_reason or "unknown"
349    print(f"Classifying closed issue #{issue_number}")
350    print(f"  Closer: {closer_login}, state_reason: {state_reason}")
351
352    issue = fetch_issue(issue_number)
353    author = issue["author"]
354    print(f"  Author: {author}, type: {issue['type_name']}")
355
356    if is_staff_member(author):
357        print(f"  Skipping: author '{author}' is a staff member")
358        return
359
360    bot_comment = get_bot_comment_with_time(issue_number)
361    bot_commented = bot_comment is not None
362    print(f"  Bot commented: {bot_commented}")
363
364    closer_is_author = closer_login == author
365
366    if bot_commented and closer_is_author:
367        classify_as_success(issue, bot_comment, state_reason)
368    elif bot_commented and not closer_is_author:
369        # Only authors, staff, and triagers can close issues, so
370        # a non-author closer is always someone with elevated permissions.
371        classify_non_author_closed(issue, bot_comment, state_reason)
372    elif not bot_commented and state_reason == "duplicate":
373        classify_as_missed_opportunity(issue)
374    else:
375        print("  Skipping: no bot comment and not closed as duplicate")
376
377
378def classify_as_success(issue, bot_comment, state_reason):
379    """Author closed their own issue after the bot commented."""
380    if state_reason == "duplicate":
381        status = "Auto-classified"
382        notes = None
383    else:
384        # could be closed for an unrelated reason; flag for review
385        status = "Needs review"
386        notes = f"Author closed as {state_reason}"
387
388    if status == "Auto-classified":
389        print(f"  -> Success (closed as {state_reason})")
390    else:
391        print(f"  -> Possible Success, needs review ({notes})")
392    add_or_update_project_item(
393        issue["node_id"],
394        outcome="Success",
395        closed_as=state_reason,
396        status=status,
397        notes=notes,
398        bot_comment_time=bot_comment["created_at"],
399    )
400
401
402def classify_non_author_closed(issue, bot_comment, state_reason):
403    """Non-author (staff or triager) closed an issue the bot had commented on."""
404    if state_reason == "duplicate":
405        classify_as_assist(issue, bot_comment)
406    else:
407        notes = f"Closed by staff/triager as {state_reason}, not duplicate"
408        print(f"  -> Possible Noise, needs review ({notes})")
409        add_or_update_project_item(
410            issue["node_id"],
411            outcome="Noise",
412            closed_as=state_reason,
413            status="Needs review",
414            notes=notes,
415            bot_comment_time=bot_comment["created_at"],
416        )
417
418
419def classify_as_assist(issue, bot_comment):
420    """Staff member closed as duplicate after the bot commented. Check if the dup matches."""
421    suggested = parse_suggested_issues(bot_comment["body"])
422    if not suggested:
423        print("  -> Assist, needs review (could not parse bot suggestions)")
424        add_or_update_project_item(
425            issue["node_id"], outcome="Assist", closed_as="duplicate",
426            status="Needs review", notes="Could not parse bot suggestions",
427            bot_comment_time=bot_comment["created_at"])
428        return
429
430    # Let exceptions from find_canonical_among propagate — a query failure here is
431    # not the same as "no canonical match" and shouldn't be silently downgraded to
432    # a Needs review entry. Failing the workflow surfaces the problem immediately.
433    original = find_canonical_among(issue["number"], suggested)
434
435    if original:
436        status = "Auto-classified"
437        notes = None
438        print(f"  -> Assist (original #{original} matches bot suggestion)")
439    else:
440        status = "Needs review"
441        suggested_str = ", ".join(f"#{number}" for number in suggested)
442        notes = f"Bot suggested {suggested_str}; none matched as canonical"
443        print(f"  -> Possible Assist, needs review ({notes})")
444
445    add_or_update_project_item(
446        issue["node_id"], outcome="Assist", closed_as="duplicate", status=status, notes=notes,
447        bot_comment_time=bot_comment["created_at"])
448
449
450def classify_as_missed_opportunity(issue):
451    """Issue closed as duplicate but the bot never commented."""
452    print("  -> Missed opportunity")
453    add_or_update_project_item(
454        issue["node_id"], outcome="Missed opportunity", closed_as="duplicate", status="Auto-classified",
455        bot_comment_time=issue["created_at"])
456
457
458def classify_open():
459    """Classify open, triaged, bot-commented issues as Noise."""
460    print("Classifying open issues")
461
462    query = (
463        f"repo:{REPO_OWNER}/{REPO_NAME} is:issue is:open "
464        f"commenter:app/{BOT_APP_SLUG} "
465        f'-label:"{NEEDS_TRIAGE_LABEL}" '
466        f"created:>={BOT_START_DATE}"
467    )
468    print(f"  Search query: {query}")
469
470    results = github_search_issues(query)
471    print(f"  Found {len(results)} candidate issues")
472
473    added, skipped, errors = 0, 0, 0
474    for item in results:
475        number = item["number"]
476        try:
477            type_name = (item.get("type") or {}).get("name")
478            author = (item.get("user") or {}).get("login", "")
479            node_id = item["node_id"]
480
481            skip_reason = (
482                f"type is {type_name}" if type_name and type_name not in ("Bug", "Crash")
483                else f"author {author} is staff" if is_staff_member(author)
484                else "already on the board" if find_project_item(node_id)
485                else "no bot duplicate comment found" if not (bot_comment := get_bot_comment_with_time(number))
486                else None
487            )
488
489            if skip_reason:
490                print(f"  #{number}: skipping, {skip_reason}")
491                skipped += 1
492                continue
493
494            print(f"  #{number}: adding as Noise")
495            add_or_update_project_item(node_id, outcome="Noise", status="Auto-classified",
496                                       bot_comment_time=bot_comment["created_at"])
497            added += 1
498        except Exception as error:  # broad catch: one issue failing shouldn't stop the sweep
499            print(f"  #{number}: error processing issue, skipping: {error}")
500            errors += 1
501
502    print(f"  Done: added {added}, skipped {skipped}, errors {errors}")
503    if errors > 0:
504        sys.exit(1)
505
506
507if __name__ == "__main__":
508    parser = argparse.ArgumentParser(
509        description="Track duplicate bot effectiveness on a GitHub project board.",
510    )
511    subparsers = parser.add_subparsers(dest="command", required=True)
512
513    classify_parser = subparsers.add_parser(
514        "classify-closed",
515        help="Classify a closed issue and add it to the project board.",
516    )
517    classify_parser.add_argument("issue_number", type=int)
518    classify_parser.add_argument("closer_login")
519    classify_parser.add_argument("state_reason")
520
521    subparsers.add_parser(
522        "classify-open",
523        help="Classify open, triaged, bot-commented issues as Noise.",
524    )
525
526    args = parser.parse_args()
527
528    GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "")
529    if not GITHUB_TOKEN:
530        print("Error: GITHUB_TOKEN environment variable is required")
531        sys.exit(1)
532
533    raw_project_number = os.environ.get("PROJECT_NUMBER", "")
534    if raw_project_number:
535        try:
536            PROJECT_NUMBER = int(raw_project_number)
537        except ValueError:
538            print(f"Error: PROJECT_NUMBER must be an integer, got '{raw_project_number}'")
539            sys.exit(1)
540    else:
541        PROJECT_NUMBER = DEFAULT_PROJECT_NUMBER
542
543    GITHUB_HEADERS = {
544        "Authorization": f"token {GITHUB_TOKEN}",
545        "Accept": "application/vnd.github+json",
546    }
547
548    if args.command == "classify-closed":
549        classify_closed(args.issue_number, args.closer_login, args.state_reason)
550    elif args.command == "classify-open":
551        classify_open()
552