From ae73878c5962ed7e3ca310d40dd1d166d50be7fb Mon Sep 17 00:00:00 2001 From: Mike Ellery Date: Thu, 19 Apr 2018 10:33:43 -0700 Subject: [PATCH] Build an unsigned rpm in dev pipeline --- Jenkinsfile | 523 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 343 insertions(+), 180 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 91722d68f..83b0d274c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,15 +3,19 @@ import groovy.json.JsonOutput import java.text.* -def all_status = [:] -def commit_id = '' +all_status = [:] +commit_id = '' +git_fork = 'ripple' +git_repo = 'rippled' +// // this is not the actual token, but an ID/key into the jenkins // credential store which httpRequest can access. -def github_cred = '6bd3f3b9-9a35-493e-8aef-975403c82d3e' +// +github_cred = '6bd3f3b9-9a35-493e-8aef-975403c82d3e' // // root API url for our repo (default, overriden below) // -String github_repo = 'https://api.github.com/repos/ripple/rippled' +github_api = 'https://api.github.com/repos/ripple/rippled' try { stage ('Startup Checks') { @@ -20,15 +24,7 @@ try { // a filesystem (although we just pass it text) node { checkout scm - commit_id = sh( - script: 'git rev-parse HEAD', - returnStdout: true) - commit_id = commit_id.trim() - echo "commit ID is ${commit_id}" - commit_log = sh ( - script: "git show --name-status ${commit_id}", - returnStdout: true) - printGitInfo (commit_id, commit_log) + commit_id = getCommitID() // // NOTE this getUserRemoteConfigs call requires a one-time // In-process Script Approval (configure jenkins). We need this @@ -37,12 +33,12 @@ try { def remote_url = scm.getUserRemoteConfigs()[0].getUrl() if (remote_url) { echo "GIT URL scm: $remote_url" - def fork = remote_url.tokenize('/')[2] - def repo = remote_url.tokenize('/')[3].split('\\.')[0] - echo "GIT FORK: $fork" - echo "GIT REPO: $repo" - github_repo = "https://api.github.com/repos/${fork}/${repo}" - echo "API URL REPO: $github_repo" + git_fork = remote_url.tokenize('/')[2] + git_repo = remote_url.tokenize('/')[3].split('\\.')[0] + echo "GIT FORK: $git_fork" + echo "GIT REPO: $git_repo" + github_api = "https://api.github.com/repos/${git_fork}/${git_repo}" + echo "API URL REPO: $github_api" } if (env.CHANGE_AUTHOR) { @@ -60,7 +56,7 @@ try { def response = httpRequest( timeout: 10, authentication: github_cred, - url: "${github_repo}/collaborators") + url: "${github_api}/collaborators") def collab_data = readJSON( text: response.content) collab_found = false; @@ -207,63 +203,7 @@ try { // not figure out how to make powershell redirect internally output = powershell ( returnStdout: true, - script: ''' -# Enable streams 3-6 -$WarningPreference = 'Continue' -$VerbosePreference = 'Continue' -$DebugPreference = 'Continue' -$InformationPreference = 'Continue' - -Invoke-BatchFile "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64 -Get-ChildItem env:* | Sort-Object name -cl -cmake --version -New-Item -ItemType Directory -Force -Path "build/$env:BUILD_DIR" -ErrorAction Stop -$sw = [Diagnostics.Stopwatch]::StartNew() -try { - Push-Location "build/$env:BUILD_DIR" - if ($env:NINJA_BUILD -eq "true") { - cmake -G"Ninja" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../.. - } - else { - cmake -G"Visual Studio 15 2017 Win64" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../.. - } - if ($LastExitCode -ne 0) { throw "CMake failed" } - - ## as of 01/2018, DO NOT USE cmake to run the actual build step. for some - ## reason, cmake spawning the build under jenkins causes MSBUILD/ninja to - ## get stuck at the end of the build. Perhaps cmake is spawning - ## incorrectly or failing to pass certain params - - if ($env:NINJA_BUILD -eq "true") { - ninja -j $env:NUMBER_OF_PROCESSORS -v - } - else { - msbuild /fl /m /nr:false /p:Configuration="$env:CONFIG_TYPE" /p:Platform=x64 /p:GenerateFullPaths=True /v:normal /nologo /clp:"ShowCommandLine;DisableConsoleColor" "$env:PROJECT_NAME.vcxproj" - } - if ($LastExitCode -ne 0) { throw "CMake build failed" } - - $exe = "./$env:CONFIG_TYPE/$env:PROJECT_NAME" - if ($env:NINJA_BUILD -eq "true") { - $exe = "./$env:PROJECT_NAME" - } - "Exe is at $exe" - $params = '--unittest', '--quiet', '--unittest-log' - if ($env:PARALLEL_TESTS -eq "true") { - $params = $params += "--unittest-jobs=$env:NUMBER_OF_PROCESSORS" - } - & $exe $params - if ($LastExitCode -ne 0) { throw "Unit tests failed" } -} -catch { - throw -} -finally { - $sw.Stop() - $sw.Elapsed - Pop-Location -} -''') + script: windowsBuildCmd()) // if the powershell command fails (has nonzero exit) // then the command above throws, we don't get our output, // and we never create this output file. @@ -278,43 +218,10 @@ finally { sh "rm -fv ${bldlabel}.txt" // execute the bld command in a redirecting shell // to capture output - sh '''\ -#!/bin/bash -set -ex -log_file=''' + "${bldlabel}.txt" + ''' -exec 3>&1 1>>${log_file} 2>&1 -ccache -s -source /opt/rh/devtoolset-6/enable -/usr/bin/time -p ./bin/ci/ubuntu/build-and-test.sh 2>&1 -ccache -s -''' + sh redhatBuildCmd(bldlabel) } } finally { - def outstr = '' - def loglink = '[console](' + env.BUILD_URL + '/console)' - def logfile = "${bldlabel}.txt" - if (fileExists(logfile)) { - outstr = readFile(logfile) - loglink = "[logfile](" + env.BUILD_URL + "/artifact/${logfile})" - } - def st = getResults(outstr, bldlabel) - def time = getTime(outstr, bldlabel) - def fail_count = getFailures(outstr, bldlabel) - outstr = null - def txtcolor = - fail_count == 0 ? 'DarkGreen' : 'Crimson' - def shortbld = bldlabel - shortbld = shortbld.replace('debug', 'dbg') - shortbld = shortbld.replace('release', 'rel') - shortbld = shortbld.replace('unity', 'un') - manager.addShortText( - "${shortbld}: ${st}, t: ${time}", - txtcolor, - 'white', - '0px', - 'white') - archive("${bldlabel}.txt") if (bldtype == 'docs') { publishHTML( allowMissing: true, @@ -332,9 +239,9 @@ ccache -s if (cmake_txt != '') { cmake_txt = "
" + cmake_txt } + def st = reportStatus(bldlabel, bldtype + cmake_txt + envs, env.BUILD_URL) lock('rippled_dev_status') { - all_status[bldlabel] = - [fail_count == 0, bldtype + cmake_txt + envs, "${st}, t: ${time}", loglink] + all_status[bldlabel] = st } } //try-catch-finally } //withEnv @@ -343,6 +250,50 @@ ccache -s } //builds item } //for variants + // Also add a single build job for doing the RPM build + // on a docker node + builds['rpm'] = { + node('docker') { + def bldlabel = 'rpm' + configFileProvider ( + [configFile( + fileId: 'rippled-commit-signer-public-keys.txt', + variable: 'SIGNER_PUBLIC_KEYS')]) + { + def remote = + (git_fork == 'ripple') ? 'origin' : git_fork + + withCredentials( + [string( + credentialsId: 'RIPPLED_RPM_ROLE_ID', + variable: 'ROLE_ID')]) + { + withEnv([ + 'docker_image=artifactory.ops.ripple.com:6555/rippled-rpm-builder:latest', + "git_commit=${commit_id}", + "git_remote=${remote}", + "rpm_release=${env.BUILD_ID}"]) + { + try { + sh "rm -fv ${bldlabel}.txt" + sh "if [ -d rpm-out ]; then rm -rf rpm-out; fi" + sh rpmBuildCmd(bldlabel) + } + finally { + def st = reportStatus(bldlabel, bldlabel, env.BUILD_URL) + lock('rippled_dev_status') { + all_status[bldlabel] = st + } + archiveArtifacts( + artifacts: 'rpm-out/*.rpm', + allowEmptyArchive: true) + } + } //withEnv + } //withCredentials + } //configFile + } //node + } + // this actually executes all the builds we just defined // above, in parallel as slaves are available parallel builds @@ -352,76 +303,15 @@ finally { // anything here should run always... stage ('Final Status') { node { - def start_time = new Date() - def sdf = new SimpleDateFormat('yyyyMMdd - HH:mm:ss') - def datestamp = sdf.format(start_time) - - def results = """ -## Jenkins Build Summary - -Built from [this commit](https://github.com/ripple/rippled/commit/${commit_id}) - -Built at __${datestamp}__ - -### Test Results - -Build Type | Log | Result | Status ----------- | --- | ------ | ------ -""" - for ( e in all_status) { - results += e.value[1] + ' | ' + e.value[3] + ' | ' + e.value[2] + ' | ' + - (e.value[0] ? 'PASS :white_check_mark: ' : 'FAIL :red_circle: ') + '\n' - } - results += '\n' - echo 'FINAL BUILD RESULTS' - echo results - + def results = makeResultText() try { - def url_comment = '' - if (env.CHANGE_ID && env.CHANGE_ID ==~ /\d+/) { - // - // CHANGE_ID indicates we are building a PR - // find PR comments - // - def resp = httpRequest( - timeout: 10, - authentication: github_cred, - url: "${github_repo}/pulls/$CHANGE_ID") - def result = readJSON(text: resp.content) - // - // follow issue comments link - // - url_comment = result['_links']['issue']['href'] + '/comments' - } - else { - // - // if not a PR, just search comments for our commit ID - // - url_comment = - "${github_repo}/commits/${commit_id}/comments" - } - - def response = httpRequest( - timeout: 10, - authentication: github_cred, - url: url_comment) - def data = readJSON(text: response.content) - def comment_id = 0 - def mode = 'POST' - // see if we can find and existing comment here with - // a heading that matches ours... - for (comment in data) { - if (comment['body'] =~ /(?m)^##\s+Jenkins Build/) { - comment_id = comment['id'] - echo "existing status comment ${comment_id} found" - url_comment = comment['url'] - mode = 'PATCH' - break; - } - } - + def res = getCommentID() //get array return b/c jenkins does not allow multiple direct return/assign + def comment_id = res[0] + def url_comment = res[1] + def mode = 'PATCH' if (comment_id == 0) { echo 'no existing status comment found' + mode = 'POST' } def body = JsonOutput.toJson([ @@ -464,6 +354,89 @@ ${log} """ } +def makeResultText () { + def start_time = new Date() + def sdf = new SimpleDateFormat('yyyyMMdd - HH:mm:ss') + def datestamp = sdf.format(start_time) + + def results = """ +## Jenkins Build Summary + +Built from [this commit](https://github.com/${git_fork}/${git_repo}/commit/${commit_id}) + +Built at __${datestamp}__ + +### Test Results + +Build Type | Log | Result | Status +---------- | --- | ------ | ------ +""" + for ( e in all_status) { + results += e.value[1] + ' | ' + e.value[3] + ' | ' + e.value[2] + ' | ' + + (e.value[0] ? 'PASS :white_check_mark: ' : 'FAIL :red_circle: ') + '\n' + } + results += '\n' + echo 'FINAL BUILD RESULTS' + echo results + results +} + +def getCommentID () { + def url_c = '' + if (env.CHANGE_ID && env.CHANGE_ID ==~ /\d+/) { + // + // CHANGE_ID indicates we are building a PR + // find PR comments + // + def resp = httpRequest( + timeout: 10, + authentication: github_cred, + url: "${github_api}/pulls/$CHANGE_ID") + def result = readJSON(text: resp.content) + // + // follow issue comments link + // + url_c = result['_links']['issue']['href'] + '/comments' + } + else { + // + // if not a PR, just search comments for our commit ID + // + url_c = + "${github_api}/commits/${commit_id}/comments" + } + def response = httpRequest( + timeout: 10, + authentication: github_cred, + url: url_c) + def data = readJSON(text: response.content) + def comment_id = 0 + // see if we can find and existing comment here with + // a heading that matches ours... + for (comment in data) { + if (comment['body'] =~ /(?m)^##\s+Jenkins Build/) { + comment_id = comment['id'] + echo "existing status comment ${comment_id} found" + url_c = comment['url'] + break; + } + } + [comment_id, url_c] +} + +def getCommitID () { + def cid = sh ( + script: 'git rev-parse HEAD', + returnStdout: true) + cid = cid.trim() + echo "commit ID is ${cid}" + commit_log = sh ( + script: "git show --name-status ${cid}", + returnStdout: true) + printGitInfo (cid, commit_log) + cid +} + @NonCPS def getResults(text, label) { // example: @@ -477,6 +450,7 @@ def getResults(text, label) { matcher ? matcher[0][1] + ' cases, ' + matcher[0][3] + ' failed' : 'no test results' } +@NonCPS def getFailures(text, label) { // [see above for format] def matcher = @@ -537,4 +511,193 @@ def upDir(path) { matcher ? matcher[0][1] : path } +// the shell command used for building on redhat +def redhatBuildCmd(bldlabel) { +'''\ +#!/bin/bash +set -ex +log_file=''' + "${bldlabel}.txt" + ''' +exec 3>&1 1>>${log_file} 2>&1 +ccache -s +source /opt/rh/devtoolset-6/enable +/usr/bin/time -p ./bin/ci/ubuntu/build-and-test.sh 2>&1 +ccache -s +''' +} + +// the powershell command used for building an RPM +def windowsBuildCmd() { +''' +# Enable streams 3-6 +$WarningPreference = 'Continue' +$VerbosePreference = 'Continue' +$DebugPreference = 'Continue' +$InformationPreference = 'Continue' + +Invoke-BatchFile "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64 +Get-ChildItem env:* | Sort-Object name +cl +cmake --version +New-Item -ItemType Directory -Force -Path "build/$env:BUILD_DIR" -ErrorAction Stop +$sw = [Diagnostics.Stopwatch]::StartNew() +try { + Push-Location "build/$env:BUILD_DIR" + if ($env:NINJA_BUILD -eq "true") { + cmake -G"Ninja" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../.. + } + else { + cmake -G"Visual Studio 15 2017 Win64" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../.. + } + if ($LastExitCode -ne 0) { throw "CMake failed" } + + ## as of 01/2018, DO NOT USE cmake to run the actual build step. for some + ## reason, cmake spawning the build under jenkins causes MSBUILD/ninja to + ## get stuck at the end of the build. Perhaps cmake is spawning + ## incorrectly or failing to pass certain params + + if ($env:NINJA_BUILD -eq "true") { + ninja -j $env:NUMBER_OF_PROCESSORS -v + } + else { + msbuild /fl /m /nr:false /p:Configuration="$env:CONFIG_TYPE" /p:Platform=x64 /p:GenerateFullPaths=True /v:normal /nologo /clp:"ShowCommandLine;DisableConsoleColor" "$env:PROJECT_NAME.vcxproj" + } + if ($LastExitCode -ne 0) { throw "CMake build failed" } + + $exe = "./$env:CONFIG_TYPE/$env:PROJECT_NAME" + if ($env:NINJA_BUILD -eq "true") { + $exe = "./$env:PROJECT_NAME" + } + "Exe is at $exe" + $params = '--unittest', '--quiet', '--unittest-log' + if ($env:PARALLEL_TESTS -eq "true") { + $params = $params += "--unittest-jobs=$env:NUMBER_OF_PROCESSORS" + } + & $exe $params + if ($LastExitCode -ne 0) { throw "Unit tests failed" } +} +catch { + throw +} +finally { + $sw.Stop() + $sw.Elapsed + Pop-Location +} +''' +} + +// the shell command used for building an RPM +def rpmBuildCmd(bldlabel) { +'''\ +#!/bin/bash +set -ex +log_file=''' + "${bldlabel}.txt" + ''' +exec 3>&1 1>>${log_file} 2>&1 + +# Vault Steps +SECRET_ID=$(cat /.vault/rippled-build-role/secret-id) +export VAULT_TOKEN=$(/usr/local/ripple/ops-toolbox/vault/vault_approle_auth -r ${ROLE_ID} -s ${SECRET_ID} -t) +/usr/local/ripple/ops-toolbox/vault/vault_get_sts_token.py -r rippled-build-role + +head $SIGNER_PUBLIC_KEYS + +mkdir -p rpm-out + +docker pull "${docker_image}" + +echo "Running build container" + +docker run --rm \ +-v $PWD/rpm-out:/opt/rippled-rpm/out \ +-v $SIGNER_PUBLIC_KEYS:/opt/rippled-rpm/public-keys.txt \ +-e "GIT_COMMIT=$git_commit" \ +-e "GIT_REMOTE=$git_remote" \ +-e "RPM_RELEASE=$rpm_release" \ +"${docker_image}" + +. rpm-out/build_vars + +cd rpm-out +tar xvf rippled-*.tar.gz +ls -la *.rpm +################################# +## for now we don't want the src +## and debugsource rpms for testing +## or archiving... +################################# +rm rippled-debugsource*.rpm +rm *.src.rpm +mkdir rpm-main +cp *.rpm rpm-main +cd rpm-main +cd ../.. + +cat > test_rpm.sh << "EOL" +#!/bin/bash + +function error { + echo $1 + exit 1 +} + +yum install -y yum-utils +rpm -i /opt/rippled-rpm/*.rpm +rc=$?; if [[ $rc != 0 ]]; then + error "error installing rpms" +fi + +/opt/ripple/bin/rippled --unittest +rc=$?; if [[ $rc != 0 ]]; then + error "rippled --unittest failed" +fi + +/opt/ripple/bin/validator-keys --unittest +rc=$?; if [[ $rc != 0 ]]; then + error "validator-keys --unittest failed" +fi + +EOL + +chmod +x test_rpm.sh + +echo "Running test container" + +docker run --rm \ +-v $PWD/rpm-out/rpm-main:/opt/rippled-rpm \ +-v $PWD:/opt/rippled --entrypoint /opt/rippled/test_rpm.sh \ +centos:latest +''' +} + +// post processing step after each build: +// * archives the log file +// * adds short description/status to build status +// * returns an array of result info to add to the all_build summary +def reportStatus(label, type, bldurl) { + def outstr = '' + def loglink = "[console](${bldurl}/console)" + def logfile = "${label}.txt" + if ( fileExists(logfile) ) { + archiveArtifacts( artifacts: logfile ) + outstr = readFile(logfile) + loglink = "[logfile](${bldurl}/artifact/${logfile})" + } + def st = getResults(outstr, label) + def time = getTime(outstr, label) + def fail_count = getFailures(outstr, label) + outstr = null + def txtcolor = + fail_count == 0 ? 'DarkGreen' : 'Crimson' + def shortbld = label + shortbld = shortbld.replace('debug', 'dbg') + shortbld = shortbld.replace('release', 'rel') + shortbld = shortbld.replace('unity', 'un') + manager.addShortText( + "${shortbld}: ${st}, t: ${time}", + txtcolor, + 'white', + '0px', + 'white') + [fail_count == 0, type, "${st}, t: ${time}", loglink] +}