CI pipeline monitor label fixes

.github/skills/ci-pipeline-monitor/.gitignore

@@ -13,3 +13,4 @@ temp/
 # Intermediate JSON output files (piped between scripts)
 failing_builds.json
 failed_tests.json
+cached_labels.json

.github/skills/ci-pipeline-monitor/scripts/update_github.py

@@ -85,13 +85,15 @@ def _one_failure(self, fail, go):
         else:
             print("GitHub Issue: NEW — creating new issue")
             gh_issue_command.append("create")
+            gh_issue_command.append("--label")
+            gh_issue_command.append("blocking-clean-ci-optional")
             creating_new_issue = True
 
         if fail["labels"]:
             if creating_new_issue:
                 for label in fail["labels"].split(','):
                     stripped_label = label.strip()
-                    if stripped_label:
+                    if stripped_label and (stripped_label != "blocking-clean-ci-optional"):
                         gh_issue_command.append('--label')
                         gh_issue_command.append(stripped_label)
 

.github/skills/ci-pipeline-monitor/scripts/validate_results.py

@@ -18,6 +18,8 @@
 import urllib.parse
 import urllib.request
 
+LABELS_API_ENDPOINT = "https://api.github.com/repos/dotnet/runtime/labels"
+LABELS_CACHE_FILE = os.path.join(__file__, "..", "cached_labels.json")
 
 def check(name, passed, detail="", warn_only=False):
     if not passed and warn_only:
@@ -574,6 +576,61 @@ def _build_sig(error_message, stack_trace):
         if not ok:
             failures += 1
 
+    # 16h. Every failure's suggested labels are valid labels on the dotnet/runtime repo
+    # Otherwise, update_github will fail to file the issue
+
+    total += 1
+    all_label_names = set()
+    try:
+        # TODO: If dotnet/runtime ever has more than 999 labels, increase this page limit. At time of script authoring
+        #  we have around 300 so this is more than enough.
+        for page_index in range(1, 11):
+            req = urllib.request.Request(f"{LABELS_API_ENDPOINT}?per_page=100&page={page_index}", headers={
+                "Accept": "application/vnd.github+json",
+                "User-Agent": "ci-pipeline-monitor-validator",
+            })
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                data = json.loads(resp.read())
+                page_label_names = set([item["name"] for item in data])
+                # Don't bother querying additional empty pages after this one.
+                if len(page_label_names) <= 0:
+                    break
+
+                for name in page_label_names:
+                    all_label_names.add(name.lower())
+
+        with open(LABELS_CACHE_FILE, "w") as f:
+            f.write(json.dumps(sorted(all_label_names)))
+    except urllib.error.HTTPError:
+        print("  [WARN] Failed to fetch labels list from GitHub due to an HTTP error. Loading cached labels list.")
+        with open(LABELS_CACHE_FILE) as f:
+            labels_json = f.read()
+            labels_list = json.loads(labels_json)
+            all_label_names = set(labels_list)
+
+    all_failure_rows = conn.execute("""
+        SELECT id, labels FROM failures
+    """).fetchall()
+    bad_labels = []
+    for failure_row in all_failure_rows:
+        raw_labels = failure_row["labels"]
+        failure_labels = raw_labels.split(',') if raw_labels else []
+        for failure_label in failure_labels:
+            failure_label = failure_label.strip().lower()
+            if not failure_label:
+                continue
+            if not (failure_label in all_label_names):
+                print(f"  [FAIL] Invalid label '{failure_label}' for failure {failure_row['id']}")
+                bad_labels.append(failure_row)
+
+    ok = check("All failure labels are valid labels from dotnet/runtime repo",
+            len(bad_labels) == 0,
+            f"{len(bad_labels)} label(s) were invalid (see above)" if len(bad_labels) else "")
+    if not ok:
+        print("Full set of valid labels follows:")
+        print(", ".join(sorted(all_label_names)))
+        failures += 1
+
     # Report checks (only if --report provided)
     if args.report:
         print("\n=== Report Sanity ===")