TheRock/compiler/amd-llvm/.github/workflows/build_metadata_extractor.py

"""
Module Name: build_metadata_extractor.py

Description:
    This module facilitates the extraction of build metadata from nightly
    builds. It reads a manifest file, collects data about submodules,
    generates tables for artifacts and build logs, and lists any failed
    jobs from GitHub actions. The results are saved in a structured JSON
    file for further analysis and reporting.

Key Classes and Functions:
    - BuildMetadataExtractor: Main class containing methods to extract and
      compile metadata.
        - __init__(...): Initializes with required parameters for accessing
          GitHub and reading manifests.
        - read_manifest_file(): Reads and validates the manifest JSON file.
        - extract_submodule_table(manifest_data): Generates a table of
          submodules and their URLs.
        - generate_manifest_artifact_logs_table(): Creates a table for Rock
          Manifest, Artifacts, and Build Logs.
        - list_failures(): Fetches and lists all failure jobs, handling
          potential pagination.
        - save_results_to_file(...): Saves the results to a file in
          structured JSON format.

Environment Variables:
    - ORG_NAME: Organization name on GitHub.
    - PROJECT_NAME: Project name within the organization.
    - RUN_ID: Identifier for the GitHub Actions workflow run.
    - GITHUB_TOKEN: GitHub API token for authentication.
    - MANIFEST_FILE: Path to the manifest file to be read.
    - ROCK_MANIFEST_URL: URL for the Rock manifest.
    - ARTIFACTS_URL: URL for the build artifacts.
    - BUILD_LOGS_URL: URL for accessing build logs.
    - OUTPUT_FILE: File name for saving extracted metadata results.
"""

import json
import os
import requests


class BuildMetadataExtractor:
    def __init__(self, org_name, project_name, run_id, github_token, manifest_path, artifacts_url, build_logs_url, output_file):
        """Initialize the extractor with organization, project info, and additional inputs."""
        self.org_name = org_name
        self.project_name = project_name
        self.run_id = run_id
        self.github_token = github_token
        self.manifest_path = manifest_path
        self.artifacts_url = artifacts_url
        self.build_logs_url = build_logs_url
        self.output_file = output_file
        self.headers = {'Authorization': f'token {self.github_token}'}
        self.base_url = f"https://api.github.com/repos/{self.org_name}/{self.project_name}"

    def read_manifest_file(self):
        """Reads and validates the manifest JSON file."""
        if not os.path.exists(self.manifest_path):
            print(f"Manifest file '{self.manifest_path}' does not exist.")
            return None
        submodules = {}
        with open(self.manifest_path, 'r') as f:
           for component in f:
              component = component.strip()
              if not component:
                 continue
              component_list = component.split(None, 1)  # this will split on component name and sha
              if len(component_list) == 2:
                 submodule_name = component_list[0]
                 pin_sha = component_list[1]
                 submodules[submodule_name] = pin_sha
        return submodules

    def extract_submodule_table(self, manifest_data):
        """Generates a table detailing submodules and URLs."""
        #the_rock_commit = manifest_data.get('the_rock_commit', '')
        #commit_url = f"{self.base_url}/commit/{the_rock_commit}"
        table = '| Submodule | URL |\n|-----------|-----|\n'
        commit_base_url = f"https://github.com/{self.org_name}"
        for submodule_name,pin_sha in manifest_data.items():
            if submodule_name == 'rccl':  # Filter out specific submodules
               continue
            submodule_url = f"{commit_base_url}/{submodule_name}"
            commit_url = f"{submodule_url}/commit/{pin_sha}"
            table += f'| {submodule_name} | ({commit_url}) |\n'

        return table

    def generate_manifest_artifact_logs_table(self):
        """Creates a table for Rock Manifest, Artifacts, and Build Logs."""
        table = '| Description | URL |\n|-------------|-----|\n'
        #table += f'| Rock Manifest | ({self.rock_manifest_url}) |\\n'
        table += f'| Artifacts | ({self.artifacts_url}) |\\n'
        table += f'| Build Logs | ({self.build_logs_url}) |\\n'

        return table

    def list_failures(self):
        """Fetches and lists all failure jobs, handling potential pagination."""
        jobs_url = f"{self.base_url}/actions/runs/{self.run_id}/jobs"
        failure_jobs = []
        page = 1

        while True:
            response = requests.get(f"{jobs_url}?page={page}", headers=self.headers)
            jobs_data = response.json()
            if 'jobs' not in jobs_data or not jobs_data['jobs']:
                break

            failure_jobs.extend([job for job in jobs_data['jobs'] if job.get('conclusion') == 'failure'])
            page += 1
        if not failure_jobs:
            return None  # No failures, return None

        failure_table = '| Failure Job Name | Job URL |\n|------------------|---------|\n'
        for job in failure_jobs:
            job_name = job['name']
            job_url = job['html_url']
            failure_table += f'| {job_name} | ({job_url}) |\\n'
        return failure_table

    def save_results_to_file(self,submodule_table, manifest_artifacts_table, failure_table):
        """Saves the results to a file in structured JSON format."""
        results = {
            "submodule_table": submodule_table,
            "manifest_artifacts_table": manifest_artifacts_table,
            "failure_table": failure_table or "No failures found"
        }
        with open(self.output_file, 'w') as f:
            json.dump(results, f, indent=2)
        print(f"Results have been saved to {self.output_file}")

if __name__ == "__main__":
    # Initialize variables from environment
    ORG_NAME = os.getenv("ORG_NAME", "ROCm")
    PROJECT_NAME = os.getenv("PROJECT_NAME", "llvm-project")
    RUN_ID = os.getenv("RUN_ID", "")
    GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
    MANIFEST_FILE = os.getenv("MANIFEST_FILE", "manifest.txt")
    #ROCK_MANIFEST_URL = os.getenv("ROCK_MANIFEST_URL")
    ARTIFACTS_URL = os.getenv("ARTIFACTS_URL")
    BUILD_LOGS_URL = os.getenv("BUILD_LOGS_URL")
    OUTPUT_FILE = os.getenv("OUTPUT_FILE", "results.json")

    # Initialize extractor
    #extractor = BuildMetadataExtractor(
    #    ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, MANIFEST_FILE, 
    #    ROCK_MANIFEST_URL, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE
    #)

    extractor = BuildMetadataExtractor(
        ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, MANIFEST_FILE, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE
    )


    # Process the manifest file
    manifest_data = extractor.read_manifest_file()
    if manifest_data:
      submodule_table = extractor.extract_submodule_table(manifest_data)
      manifest_artifacts_table = extractor.generate_manifest_artifact_logs_table()
      failure_table = extractor.list_failures()

    # Save results to an output file
    extractor.save_results_to_file(submodule_table, manifest_artifacts_table, failure_table)