TheRock/rocm-libraries/projects/miopen/Jenkinsfile

def projDir = "projects"
def miopenDir = "${projDir}/miopen"
def ckDir = "${projDir}/composablekernel"

def rocmnode(name) {
    return '(rocmtest || miopen) && (' + name + ')'
}

// Returns the original PR branch tip SHA (pullHash) or branch HEAD SHA (hash)
// from the Jenkins SCMRevisionAction, which is recorded before any local merge.
// Returns null for non-SCM triggers; caller should handle fall back.
@NonCPS
String getGitHubCommitHash(def build)
{
    def scmAction = build?.actions.find { action ->
        action instanceof jenkins.scm.api.SCMRevisionAction
    }
    if (scmAction?.revision instanceof org.jenkinsci.plugins.github_branch_source.PullRequestSCMRevision)
    {
        return scmAction.revision.pullHash
    }
    else if (scmAction?.revision instanceof jenkins.plugins.git.AbstractGitSCMSource$SCMRevisionImpl)
    {
        return scmAction.revision.hash
    }
    return null
}

def get_branch_name(){
    def shared_library_branch = scm.branches[0].name
    if (shared_library_branch .contains("*/")) {
        shared_library_branch  = shared_library_branch.split("\\*/")[1]
    }
    echo "${shared_library_branch}"
    return shared_library_branch
}

/// Stage name format:
/// [DataType] Backend[/Compiler] BuildType [TestSet] [Target]
///
/// The only mandatory elements are Backend and BuildType; others are optional.
///
/// DataType := { Fp16 | Bf16 | Int8 | Fp32 }
/// Backend := { Hip | HipNoGPU}
/// Compiler := { Clang* | GCC* }
///   * "Clang" is the default for the Hip backend, and implies hip-clang compiler.
///   * The default compiler is usually not specified.
/// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ]
///   * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find
///                            NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... }
/// TestSet := { All | Smoke* | <Performance Dataset> | Build-only }
///   * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On".
///   * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified.
///   * "Performance Dataset" is a performance test with a specified dataset.
/// Target := { gfx908 | gfx90a | gfx942 } [ Xnack+ ]

utils = null

def show_node_info() {
    sh """
        echo "NODE_NAME = \$NODE_NAME"
        lsb_release -sd
        uname -r
        cat /sys/module/amdgpu/version
        ls /opt/ -la
    """
}

def cloneUpdateRefRepo() {
    def refRepoPath = "/var/jenkins/ref-repo/rocm-libraries"
    def lockLabel = "git ref repo lock - ${env.NODE_NAME}"
    def folderExists = sh(
        script: "test -d ${refRepoPath}/refs",
        returnStatus: true
    ) == 0

    if (!folderExists) {
        echo "rocm-libraries repo does not exist at ${refRepoPath}, creating mirror clone..."
        echo "locking on label: ${lockLabel}"
        lock(lockLabel) {
            def cloneCommand = """
                set -ex
                rm -rf ${refRepoPath} && mkdir -p ${refRepoPath}
                git clone --mirror https://github.com/ROCm/rocm-libraries.git ${refRepoPath}
            """
            sh(script: cloneCommand, label: "clone ref repo")
        }
        echo "Completed git clone, lock released"
    }
    echo "rocm-libraries repo exists at ${refRepoPath}, performing git remote update..."
    echo "locking on label: ${lockLabel}"
    lock(lockLabel) {
        def fetchCommand = """
            set -ex
            cd ${refRepoPath}
            git remote prune origin
            git remote update
        """
        sh(script: fetchCommand, label: "update ref repo")
    }
    echo "Completed git ref repo fetch, lock released"
}

def checkoutRepo()
{
    //update ref repo
    cloneUpdateRefRepo()
    def scmVars = checkout scm
    // getGitHubCommitHash reads SCMRevisionAction recorded before any local merge,
    // giving the true PR branch tip (pullHash) or branch HEAD (hash).
    // Falls back to ORIG_HEAD (pre-merge HEAD set by git merge) when SCMRevisionAction
    // is unavailable, then to HEAD for branch builds where no merge occurred.
    env.GIT_COMMIT = getGitHubCommitHash(currentBuild.rawBuild) ?: sh(returnStdout: true, script: '''
        git rev-parse ORIG_HEAD 2>/dev/null || git rev-parse HEAD
    ''').trim()
}

def withWorkingDir(Closure body) {
    show_node_info()
    checkoutRepo()
    dir("${env.WORKSPACE}/${env.MIOPEN_DIR}") {
        if (utils == null) {
            utils = load "vars/utils.groovy"
        }
        body()
    }
}

// Loads utils if a "Restart from Stage" skipped all earlier withWorkingDir calls.
def ensureUtils() {
    if (utils == null) {
        node(rocmnode("nogpu")) {
            withWorkingDir {}
        }
    }
}

// Posts GitHub status from the pipeline post section, loading utils if needed.
def postGithubStatus(String context, String state, String description, Closure extraActions = null) {
    if (utils == null) {
        node(rocmnode("nogpu")) {
            withWorkingDir {
                utils.setGithubStatus(context, state, description)
                if (extraActions) {
                    extraActions()
                }
            }
        }
    } else {
        node {
            utils.setGithubStatus(context, state, description)
            if (extraActions) {
                extraActions()
            }
        }
    }
}

def runDbSyncJob(def flags, def gpu_family)
{
    script {
        withWorkingDir {
            utils.buildHipClangJob(dvc_pull: true,
                                setup_flags: "-DMIOPEN_TEST_DBSYNC=1" + flags,
                                make_targets: 'test_db_sync',
                                execute_cmd: './bin/test_db_sync',
                                needs_gpu:false,
                                build_install: true,
                                gpu_family: gpu_family
                                )
        }
    }
}

def runBuildAndSingleGtestJob(Map conf=[:])
{
    def flags = conf.get('flags', '')
    def build_timeout_minutes = (conf.get('build_timeout_minutes', 420) as Integer)
    def gpu_family = conf.get('gpu_family', null)

    // catchError prevents exceptions from propagating so junit always runs.
    def buildReached = false
    catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
        withWorkingDir {
            def single_gtest_flags = " -DMIOPEN_TEST_DISCRETE=OFF -DGTEST_PARALLEL_LEVEL=4 "
            buildReached = true
            utils.buildHipClangJob(
                setup_flags: single_gtest_flags + flags,
                build_cmd: "LLVM_PATH=/opt/rocm/llvm CTEST_PARALLEL_LEVEL=4 ninja -j\$(nproc) install miopen_gtest check",
                build_install:true,
                build_timeout:build_timeout_minutes,
                gpu_family: gpu_family)
        }
    }

    // Post failure if catchError swallowed an exception before buildHipClangJob ran.
    if (!buildReached && utils != null) {
        utils.setGithubStatus(env.STAGE_NAME, 'failure', 'Stage failed before build started')
    }

    // Collect JUnit XML even on failure.
    def xmlGlob = "${env.MIOPEN_DIR}/build/test_results/miopen_gtest_shard*.xml"
    junit allowEmptyResults: true, testResults: xmlGlob
}

def runDockerBuild(String gpuFamily) {
    node(rocmnode("docker")) {
        try {
            withWorkingDir {
                def (dockerImage, imageName) = utils.getDockerImageWithStatus(gpu_family: gpuFamily, ensure_only: true)
                // Expose the built CI image name so Publish Dev Image can retag it directly.
                if (gpuFamily == "ci") {
                    env.CI_DOCKER_IMAGE = imageName
                }
            }
        } finally {
            cleanWs()
        }
    }
}

def runTheRockDockerBuild() {
    node(rocmnode("docker")) {
        try {
            withWorkingDir {
                def result = utils.buildTheRockDockerImage()
                // Expose values to subsequent stages via pipeline-scoped env vars.
                env.THEROCK_CANDIDATE_IMAGE = result.image    ?: ""
                env.THEROCK_FULL_HASH       = result.fullHash ?: ""
                env.THEROCK_SHORT_HASH      = result.shortHash ?: ""
                // "true" means the hash is already live on :therock - Promote stage is skipped.
                env.THEROCK_SKIP_UPDATE     = result.skip ? "true" : "false"
            }
        } finally {
            cleanWs()
        }
    }
}

def runDevImagePublish() {
    node(rocmnode("docker")) {
        try {
            withWorkingDir {
                utils.publishDevDockerImage()
            }
        } finally {
            cleanWs()
        }
    }
}

def runTheRockDockerPromote() {
    node(rocmnode("docker")) {
        try {
            withWorkingDir {
                utils.promoteTheRockDockerImage(
                    env.THEROCK_CANDIDATE_IMAGE,
                    env.THEROCK_FULL_HASH
                )
            }
        } finally {
            cleanWs()
        }
    }
}

//launch develop branch nightly/weekly jobs
CRON_SETTINGS = BRANCH_NAME == "develop" ? '''
0 0 * * * % RUN_NIGHTLY_TESTS=true;BUILD_PACKAGE_AND_CHECKS=false;BUILD_FULL_TESTS=false;TARGET_GFX908=false;TARGET_GFX90A=true;TARGET_GFX942=true
0 1 * * * % NONCRITICAL_HW_NIGHTLY=true;BUILD_PACKAGE_AND_CHECKS=false;BUILD_FULL_TESTS=false;TARGET_GFX908=true;TARGET_NAVI35=true;TARGET_GFX90A=false;TARGET_GFX942=false
0 22 * * * % BUILD_THEROCK_DOCKER=true
''' : ""


pipeline {
    agent none
    options {
        skipDefaultCheckout()
        // Disabled: selective-rerun needs all parallel stages to finish.
        // parallelsAlwaysFailFast()
    }
    triggers{
        parameterizedCron(CRON_SETTINGS)
    }
    parameters {
        booleanParam(
            name: "BUILD_DOCKER",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "BUILD_THEROCK_DOCKER",
            defaultValue: false,
            description: "")
        booleanParam(
            name: "BUILD_FULL_TESTS",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "RUN_HIP_TIDY",
            defaultValue: true,
            description: "Control the Hip Tidy (clang-analyze) stage in Full Tests")
        booleanParam(
            name: "BUILD_PACKAGE_AND_CHECKS",
            defaultValue: true,
            description: "Control the Package and Static Checks stage")
        booleanParam(
            name: "TARGET_NOGPU",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "TARGET_GFX908",
            defaultValue: false,
            description: "")
        booleanParam(
            name: "TARGET_GFX90A",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "TARGET_GFX942",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "TARGET_NAVI32",
            defaultValue: false,
            description: "")
        booleanParam(
            name: "TARGET_NAVI35",
            defaultValue: false,
            description: "Navi3.5 Strix Halo")
        booleanParam(
            name: "TARGET_NAVI4",
            defaultValue: false,
            description: "")
        booleanParam(
            name: "DATATYPE_NA",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "DATATYPE_FP32",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "DATATYPE_TF32",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "DATATYPE_FP16",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "DATATYPE_BF16",
            defaultValue: true,
            description: "")
        booleanParam(
            name: "DBSYNC_TEST",
            defaultValue: true,
            description: "Control DB Sync test execution in Full Tests")
        string(name: "DOCKER_IMAGE_OVERRIDE",
            defaultValue: '',
            description: "Specify a custom Docker image to use for CI for debugging")
        booleanParam(
            name: "USE_SCCACHE_DOCKER",
            defaultValue: true,
            description: "Use the sccache for building CK in the Docker Image (default: ON)")
        booleanParam(
            name: "RUN_NIGHTLY_TESTS",
            defaultValue: false,
            description: "Run the nightly tests (default: OFF)")
        booleanParam(
            name: "NONCRITICAL_HW_NIGHTLY",
            defaultValue: false,
            description: "Nightly builds for non-critical hardware (gfx908, Navi 3.5) (default: OFF)")

    }

    environment{
        extra_log_env   = " MIOPEN_LOG_LEVEL=5 "
        Fp16_flags      = " -DMIOPEN_TEST_HALF=On"
        Bf16_flags      = " -DMIOPEN_TEST_BFLOAT16=On"
        Int8_flags      = " -DMIOPEN_TEST_INT8=On"
        Full_test       = " -DMIOPEN_TEST_ALL=On"
        Tf32_flags      = " -DMIOPEN_TEST_TF32=On"

        gfx908_flags    = " -DMIOPEN_INSTALL_GPU_DATABASES=gfx908"
        gfx90a_flags    = " -DMIOPEN_INSTALL_GPU_DATABASES=gfx90a"
        gfx942_flags    = " -DMIOPEN_INSTALL_GPU_DATABASES=gfx942"
        gfx1151_flags   = " -DMIOPEN_INSTALL_GPU_DATABASES=gfx1151"
        gfx1101_flags   = " -DMIOPEN_INSTALL_GPU_DATABASES=gfx1101"

        Smoke_targets   = " check MIOpenDriver"
        NOCOMGR_flags   = " -DMIOPEN_USE_COMGR=Off"
        NOMLIR_flags    = " -DMIOPEN_USE_MLIR=Off"
        CK_DIR          = "${ckDir}"
        MIOPEN_DIR      = "${miopenDir}"
        PROJ_DIR        = "${projDir}"

        Build_timeout_minutes = 420
    }
    stages{
        // Builds rocm/miopen:therock-<hash> if the pinned hash changed; sets THEROCK_CANDIDATE_IMAGE.
        stage('Build TheRock Docker: Build') {
            when {
                expression { params.BUILD_THEROCK_DOCKER }
            }
            steps {
                script {
                    runTheRockDockerBuild()
                }
            }
        }
        // Always runs when BUILD_THEROCK_DOCKER=true, even if the TheRock hash is unchanged.
        // This is intentional: nightly CI catches CK regressions and produces the dev image
        // regardless of whether TheRock itself advanced.
        stage('Build Docker'){
            when {
                expression {
                    params.BUILD_THEROCK_DOCKER \
                    || (params.BUILD_DOCKER && params.TARGET_NOGPU && params.DATATYPE_NA)
                }
            }
            parallel {
                stage('Build Docker for CI') { steps { script { runDockerBuild('ci') } } } // Unified docker build for CI
                // stage('Build Docker gfx90X') { steps { script { runDockerBuild('gfx90X') } } }
                // stage('Build Docker gfx942, gfx950') { steps { script { runDockerBuild('gfx942_gfx950') } } }
                // stage('Build Docker Navi(gfx1101, gfx1151)') { steps { script { runDockerBuild('navi') } } }
            }
        }
        stage("Package and Static checks") {
            when {
                expression {
                    params.BUILD_THEROCK_DOCKER \
                    || (params.BUILD_PACKAGE_AND_CHECKS && params.TARGET_NOGPU && params.DATATYPE_NA)
                }
            }
            steps {
                script {
                    ensureUtils()
                    parallel utils.packageAndStaticCheckStages(params, env, this.&rocmnode, this.&withWorkingDir)
                }
            }
        }
        stage("Full Tests") {
            when {
                expression {
                    params.BUILD_FULL_TESTS || params.BUILD_THEROCK_DOCKER
                }
            }
            steps {
                script {
                    ensureUtils()
                    parallel utils.fullTestStages(params, env, this.&rocmnode, this.&withWorkingDir, this.&runDbSyncJob, this.&runBuildAndSingleGtestJob)
                }
            }
        }
        // Promotes the candidate to :therock only after all CI stages pass.
        stage('Build TheRock Docker: Promote') {
            when {
                expression {
                    params.BUILD_THEROCK_DOCKER && env.THEROCK_SKIP_UPDATE != 'true'
                }
            }
            steps {
                script {
                    runTheRockDockerPromote()
                }
            }
        }
        // Publishes rocm/miopen-dev:multiarch_dev_<date> and :latest after CI passes.
        stage('Publish Dev Image') {
            when {
                expression { params.BUILD_THEROCK_DOCKER }
            }
            steps {
                script {
                    runDevImagePublish()
                }
            }
        }
        stage("Nightly Tests") {
            when {
                expression { params.RUN_NIGHTLY_TESTS }
            }
            steps {
                script {
                    ensureUtils()
                    parallel utils.nightlyTestStages(params, env, this.&rocmnode, this.&withWorkingDir)
                }
            }
        }
        stage("Non-Critical HW Nightly Tests") {
            when {
                expression { params.NONCRITICAL_HW_NIGHTLY }
            }
            steps {
                script {
                    ensureUtils()
                    parallel utils.nonCriticalHWNightlyStages(params, env, this.&rocmnode, this.&withWorkingDir, this.&runDbSyncJob, this.&runBuildAndSingleGtestJob)
                }
            }
        }
    }

    post {
        success {
            script {
                postGithubStatus('Math CI Summary', 'success', 'All checks have passed')
            }
        }
        failure {
            script {
                postGithubStatus('Math CI Summary', 'failure', 'Some checks have failed') {
                    utils.sendTeamsFailureNotification(buildTheRock: params.BUILD_THEROCK_DOCKER)
                }
            }
        }
    }
}