Build an unsigned rpm in dev pipeline

This commit is contained in:
Mike Ellery
2018-04-19 10:33:43 -07:00
committed by seelabs
parent cfdc64d7cf
commit ae73878c59

523
Jenkinsfile vendored
View File

@@ -3,15 +3,19 @@
import groovy.json.JsonOutput
import java.text.*
def all_status = [:]
def commit_id = ''
all_status = [:]
commit_id = ''
git_fork = 'ripple'
git_repo = 'rippled'
//
// this is not the actual token, but an ID/key into the jenkins
// credential store which httpRequest can access.
def github_cred = '6bd3f3b9-9a35-493e-8aef-975403c82d3e'
//
github_cred = '6bd3f3b9-9a35-493e-8aef-975403c82d3e'
//
// root API url for our repo (default, overriden below)
//
String github_repo = 'https://api.github.com/repos/ripple/rippled'
github_api = 'https://api.github.com/repos/ripple/rippled'
try {
stage ('Startup Checks') {
@@ -20,15 +24,7 @@ try {
// a filesystem (although we just pass it text)
node {
checkout scm
commit_id = sh(
script: 'git rev-parse HEAD',
returnStdout: true)
commit_id = commit_id.trim()
echo "commit ID is ${commit_id}"
commit_log = sh (
script: "git show --name-status ${commit_id}",
returnStdout: true)
printGitInfo (commit_id, commit_log)
commit_id = getCommitID()
//
// NOTE this getUserRemoteConfigs call requires a one-time
// In-process Script Approval (configure jenkins). We need this
@@ -37,12 +33,12 @@ try {
def remote_url = scm.getUserRemoteConfigs()[0].getUrl()
if (remote_url) {
echo "GIT URL scm: $remote_url"
def fork = remote_url.tokenize('/')[2]
def repo = remote_url.tokenize('/')[3].split('\\.')[0]
echo "GIT FORK: $fork"
echo "GIT REPO: $repo"
github_repo = "https://api.github.com/repos/${fork}/${repo}"
echo "API URL REPO: $github_repo"
git_fork = remote_url.tokenize('/')[2]
git_repo = remote_url.tokenize('/')[3].split('\\.')[0]
echo "GIT FORK: $git_fork"
echo "GIT REPO: $git_repo"
github_api = "https://api.github.com/repos/${git_fork}/${git_repo}"
echo "API URL REPO: $github_api"
}
if (env.CHANGE_AUTHOR) {
@@ -60,7 +56,7 @@ try {
def response = httpRequest(
timeout: 10,
authentication: github_cred,
url: "${github_repo}/collaborators")
url: "${github_api}/collaborators")
def collab_data = readJSON(
text: response.content)
collab_found = false;
@@ -207,63 +203,7 @@ try {
// not figure out how to make powershell redirect internally
output = powershell (
returnStdout: true,
script: '''
# Enable streams 3-6
$WarningPreference = 'Continue'
$VerbosePreference = 'Continue'
$DebugPreference = 'Continue'
$InformationPreference = 'Continue'
Invoke-BatchFile "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
Get-ChildItem env:* | Sort-Object name
cl
cmake --version
New-Item -ItemType Directory -Force -Path "build/$env:BUILD_DIR" -ErrorAction Stop
$sw = [Diagnostics.Stopwatch]::StartNew()
try {
Push-Location "build/$env:BUILD_DIR"
if ($env:NINJA_BUILD -eq "true") {
cmake -G"Ninja" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../..
}
else {
cmake -G"Visual Studio 15 2017 Win64" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../..
}
if ($LastExitCode -ne 0) { throw "CMake failed" }
## as of 01/2018, DO NOT USE cmake to run the actual build step. for some
## reason, cmake spawning the build under jenkins causes MSBUILD/ninja to
## get stuck at the end of the build. Perhaps cmake is spawning
## incorrectly or failing to pass certain params
if ($env:NINJA_BUILD -eq "true") {
ninja -j $env:NUMBER_OF_PROCESSORS -v
}
else {
msbuild /fl /m /nr:false /p:Configuration="$env:CONFIG_TYPE" /p:Platform=x64 /p:GenerateFullPaths=True /v:normal /nologo /clp:"ShowCommandLine;DisableConsoleColor" "$env:PROJECT_NAME.vcxproj"
}
if ($LastExitCode -ne 0) { throw "CMake build failed" }
$exe = "./$env:CONFIG_TYPE/$env:PROJECT_NAME"
if ($env:NINJA_BUILD -eq "true") {
$exe = "./$env:PROJECT_NAME"
}
"Exe is at $exe"
$params = '--unittest', '--quiet', '--unittest-log'
if ($env:PARALLEL_TESTS -eq "true") {
$params = $params += "--unittest-jobs=$env:NUMBER_OF_PROCESSORS"
}
& $exe $params
if ($LastExitCode -ne 0) { throw "Unit tests failed" }
}
catch {
throw
}
finally {
$sw.Stop()
$sw.Elapsed
Pop-Location
}
''')
script: windowsBuildCmd())
// if the powershell command fails (has nonzero exit)
// then the command above throws, we don't get our output,
// and we never create this output file.
@@ -278,43 +218,10 @@ finally {
sh "rm -fv ${bldlabel}.txt"
// execute the bld command in a redirecting shell
// to capture output
sh '''\
#!/bin/bash
set -ex
log_file=''' + "${bldlabel}.txt" + '''
exec 3>&1 1>>${log_file} 2>&1
ccache -s
source /opt/rh/devtoolset-6/enable
/usr/bin/time -p ./bin/ci/ubuntu/build-and-test.sh 2>&1
ccache -s
'''
sh redhatBuildCmd(bldlabel)
}
}
finally {
def outstr = ''
def loglink = '[console](' + env.BUILD_URL + '/console)'
def logfile = "${bldlabel}.txt"
if (fileExists(logfile)) {
outstr = readFile(logfile)
loglink = "[logfile](" + env.BUILD_URL + "/artifact/${logfile})"
}
def st = getResults(outstr, bldlabel)
def time = getTime(outstr, bldlabel)
def fail_count = getFailures(outstr, bldlabel)
outstr = null
def txtcolor =
fail_count == 0 ? 'DarkGreen' : 'Crimson'
def shortbld = bldlabel
shortbld = shortbld.replace('debug', 'dbg')
shortbld = shortbld.replace('release', 'rel')
shortbld = shortbld.replace('unity', 'un')
manager.addShortText(
"${shortbld}: ${st}, t: ${time}",
txtcolor,
'white',
'0px',
'white')
archive("${bldlabel}.txt")
if (bldtype == 'docs') {
publishHTML(
allowMissing: true,
@@ -332,9 +239,9 @@ ccache -s
if (cmake_txt != '') {
cmake_txt = " <br/>" + cmake_txt
}
def st = reportStatus(bldlabel, bldtype + cmake_txt + envs, env.BUILD_URL)
lock('rippled_dev_status') {
all_status[bldlabel] =
[fail_count == 0, bldtype + cmake_txt + envs, "${st}, t: ${time}", loglink]
all_status[bldlabel] = st
}
} //try-catch-finally
} //withEnv
@@ -343,6 +250,50 @@ ccache -s
} //builds item
} //for variants
// Also add a single build job for doing the RPM build
// on a docker node
builds['rpm'] = {
node('docker') {
def bldlabel = 'rpm'
configFileProvider (
[configFile(
fileId: 'rippled-commit-signer-public-keys.txt',
variable: 'SIGNER_PUBLIC_KEYS')])
{
def remote =
(git_fork == 'ripple') ? 'origin' : git_fork
withCredentials(
[string(
credentialsId: 'RIPPLED_RPM_ROLE_ID',
variable: 'ROLE_ID')])
{
withEnv([
'docker_image=artifactory.ops.ripple.com:6555/rippled-rpm-builder:latest',
"git_commit=${commit_id}",
"git_remote=${remote}",
"rpm_release=${env.BUILD_ID}"])
{
try {
sh "rm -fv ${bldlabel}.txt"
sh "if [ -d rpm-out ]; then rm -rf rpm-out; fi"
sh rpmBuildCmd(bldlabel)
}
finally {
def st = reportStatus(bldlabel, bldlabel, env.BUILD_URL)
lock('rippled_dev_status') {
all_status[bldlabel] = st
}
archiveArtifacts(
artifacts: 'rpm-out/*.rpm',
allowEmptyArchive: true)
}
} //withEnv
} //withCredentials
} //configFile
} //node
}
// this actually executes all the builds we just defined
// above, in parallel as slaves are available
parallel builds
@@ -352,76 +303,15 @@ finally {
// anything here should run always...
stage ('Final Status') {
node {
def start_time = new Date()
def sdf = new SimpleDateFormat('yyyyMMdd - HH:mm:ss')
def datestamp = sdf.format(start_time)
def results = """
## Jenkins Build Summary
Built from [this commit](https://github.com/ripple/rippled/commit/${commit_id})
Built at __${datestamp}__
### Test Results
Build Type | Log | Result | Status
---------- | --- | ------ | ------
"""
for ( e in all_status) {
results += e.value[1] + ' | ' + e.value[3] + ' | ' + e.value[2] + ' | ' +
(e.value[0] ? 'PASS :white_check_mark: ' : 'FAIL :red_circle: ') + '\n'
}
results += '\n'
echo 'FINAL BUILD RESULTS'
echo results
def results = makeResultText()
try {
def url_comment = ''
if (env.CHANGE_ID && env.CHANGE_ID ==~ /\d+/) {
//
// CHANGE_ID indicates we are building a PR
// find PR comments
//
def resp = httpRequest(
timeout: 10,
authentication: github_cred,
url: "${github_repo}/pulls/$CHANGE_ID")
def result = readJSON(text: resp.content)
//
// follow issue comments link
//
url_comment = result['_links']['issue']['href'] + '/comments'
}
else {
//
// if not a PR, just search comments for our commit ID
//
url_comment =
"${github_repo}/commits/${commit_id}/comments"
}
def response = httpRequest(
timeout: 10,
authentication: github_cred,
url: url_comment)
def data = readJSON(text: response.content)
def comment_id = 0
def mode = 'POST'
// see if we can find and existing comment here with
// a heading that matches ours...
for (comment in data) {
if (comment['body'] =~ /(?m)^##\s+Jenkins Build/) {
comment_id = comment['id']
echo "existing status comment ${comment_id} found"
url_comment = comment['url']
mode = 'PATCH'
break;
}
}
def res = getCommentID() //get array return b/c jenkins does not allow multiple direct return/assign
def comment_id = res[0]
def url_comment = res[1]
def mode = 'PATCH'
if (comment_id == 0) {
echo 'no existing status comment found'
mode = 'POST'
}
def body = JsonOutput.toJson([
@@ -464,6 +354,89 @@ ${log}
"""
}
def makeResultText () {
def start_time = new Date()
def sdf = new SimpleDateFormat('yyyyMMdd - HH:mm:ss')
def datestamp = sdf.format(start_time)
def results = """
## Jenkins Build Summary
Built from [this commit](https://github.com/${git_fork}/${git_repo}/commit/${commit_id})
Built at __${datestamp}__
### Test Results
Build Type | Log | Result | Status
---------- | --- | ------ | ------
"""
for ( e in all_status) {
results += e.value[1] + ' | ' + e.value[3] + ' | ' + e.value[2] + ' | ' +
(e.value[0] ? 'PASS :white_check_mark: ' : 'FAIL :red_circle: ') + '\n'
}
results += '\n'
echo 'FINAL BUILD RESULTS'
echo results
results
}
def getCommentID () {
def url_c = ''
if (env.CHANGE_ID && env.CHANGE_ID ==~ /\d+/) {
//
// CHANGE_ID indicates we are building a PR
// find PR comments
//
def resp = httpRequest(
timeout: 10,
authentication: github_cred,
url: "${github_api}/pulls/$CHANGE_ID")
def result = readJSON(text: resp.content)
//
// follow issue comments link
//
url_c = result['_links']['issue']['href'] + '/comments'
}
else {
//
// if not a PR, just search comments for our commit ID
//
url_c =
"${github_api}/commits/${commit_id}/comments"
}
def response = httpRequest(
timeout: 10,
authentication: github_cred,
url: url_c)
def data = readJSON(text: response.content)
def comment_id = 0
// see if we can find and existing comment here with
// a heading that matches ours...
for (comment in data) {
if (comment['body'] =~ /(?m)^##\s+Jenkins Build/) {
comment_id = comment['id']
echo "existing status comment ${comment_id} found"
url_c = comment['url']
break;
}
}
[comment_id, url_c]
}
def getCommitID () {
def cid = sh (
script: 'git rev-parse HEAD',
returnStdout: true)
cid = cid.trim()
echo "commit ID is ${cid}"
commit_log = sh (
script: "git show --name-status ${cid}",
returnStdout: true)
printGitInfo (cid, commit_log)
cid
}
@NonCPS
def getResults(text, label) {
// example:
@@ -477,6 +450,7 @@ def getResults(text, label) {
matcher ? matcher[0][1] + ' cases, ' + matcher[0][3] + ' failed' : 'no test results'
}
@NonCPS
def getFailures(text, label) {
// [see above for format]
def matcher =
@@ -537,4 +511,193 @@ def upDir(path) {
matcher ? matcher[0][1] : path
}
// the shell command used for building on redhat
def redhatBuildCmd(bldlabel) {
'''\
#!/bin/bash
set -ex
log_file=''' + "${bldlabel}.txt" + '''
exec 3>&1 1>>${log_file} 2>&1
ccache -s
source /opt/rh/devtoolset-6/enable
/usr/bin/time -p ./bin/ci/ubuntu/build-and-test.sh 2>&1
ccache -s
'''
}
// the powershell command used for building an RPM
def windowsBuildCmd() {
'''
# Enable streams 3-6
$WarningPreference = 'Continue'
$VerbosePreference = 'Continue'
$DebugPreference = 'Continue'
$InformationPreference = 'Continue'
Invoke-BatchFile "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
Get-ChildItem env:* | Sort-Object name
cl
cmake --version
New-Item -ItemType Directory -Force -Path "build/$env:BUILD_DIR" -ErrorAction Stop
$sw = [Diagnostics.Stopwatch]::StartNew()
try {
Push-Location "build/$env:BUILD_DIR"
if ($env:NINJA_BUILD -eq "true") {
cmake -G"Ninja" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../..
}
else {
cmake -G"Visual Studio 15 2017 Win64" -Dtarget="$env:COMPILER.$env:TARGET" -DCMAKE_VERBOSE_MAKEFILE=ON ../..
}
if ($LastExitCode -ne 0) { throw "CMake failed" }
## as of 01/2018, DO NOT USE cmake to run the actual build step. for some
## reason, cmake spawning the build under jenkins causes MSBUILD/ninja to
## get stuck at the end of the build. Perhaps cmake is spawning
## incorrectly or failing to pass certain params
if ($env:NINJA_BUILD -eq "true") {
ninja -j $env:NUMBER_OF_PROCESSORS -v
}
else {
msbuild /fl /m /nr:false /p:Configuration="$env:CONFIG_TYPE" /p:Platform=x64 /p:GenerateFullPaths=True /v:normal /nologo /clp:"ShowCommandLine;DisableConsoleColor" "$env:PROJECT_NAME.vcxproj"
}
if ($LastExitCode -ne 0) { throw "CMake build failed" }
$exe = "./$env:CONFIG_TYPE/$env:PROJECT_NAME"
if ($env:NINJA_BUILD -eq "true") {
$exe = "./$env:PROJECT_NAME"
}
"Exe is at $exe"
$params = '--unittest', '--quiet', '--unittest-log'
if ($env:PARALLEL_TESTS -eq "true") {
$params = $params += "--unittest-jobs=$env:NUMBER_OF_PROCESSORS"
}
& $exe $params
if ($LastExitCode -ne 0) { throw "Unit tests failed" }
}
catch {
throw
}
finally {
$sw.Stop()
$sw.Elapsed
Pop-Location
}
'''
}
// the shell command used for building an RPM
def rpmBuildCmd(bldlabel) {
'''\
#!/bin/bash
set -ex
log_file=''' + "${bldlabel}.txt" + '''
exec 3>&1 1>>${log_file} 2>&1
# Vault Steps
SECRET_ID=$(cat /.vault/rippled-build-role/secret-id)
export VAULT_TOKEN=$(/usr/local/ripple/ops-toolbox/vault/vault_approle_auth -r ${ROLE_ID} -s ${SECRET_ID} -t)
/usr/local/ripple/ops-toolbox/vault/vault_get_sts_token.py -r rippled-build-role
head $SIGNER_PUBLIC_KEYS
mkdir -p rpm-out
docker pull "${docker_image}"
echo "Running build container"
docker run --rm \
-v $PWD/rpm-out:/opt/rippled-rpm/out \
-v $SIGNER_PUBLIC_KEYS:/opt/rippled-rpm/public-keys.txt \
-e "GIT_COMMIT=$git_commit" \
-e "GIT_REMOTE=$git_remote" \
-e "RPM_RELEASE=$rpm_release" \
"${docker_image}"
. rpm-out/build_vars
cd rpm-out
tar xvf rippled-*.tar.gz
ls -la *.rpm
#################################
## for now we don't want the src
## and debugsource rpms for testing
## or archiving...
#################################
rm rippled-debugsource*.rpm
rm *.src.rpm
mkdir rpm-main
cp *.rpm rpm-main
cd rpm-main
cd ../..
cat > test_rpm.sh << "EOL"
#!/bin/bash
function error {
echo $1
exit 1
}
yum install -y yum-utils
rpm -i /opt/rippled-rpm/*.rpm
rc=$?; if [[ $rc != 0 ]]; then
error "error installing rpms"
fi
/opt/ripple/bin/rippled --unittest
rc=$?; if [[ $rc != 0 ]]; then
error "rippled --unittest failed"
fi
/opt/ripple/bin/validator-keys --unittest
rc=$?; if [[ $rc != 0 ]]; then
error "validator-keys --unittest failed"
fi
EOL
chmod +x test_rpm.sh
echo "Running test container"
docker run --rm \
-v $PWD/rpm-out/rpm-main:/opt/rippled-rpm \
-v $PWD:/opt/rippled --entrypoint /opt/rippled/test_rpm.sh \
centos:latest
'''
}
// post processing step after each build:
// * archives the log file
// * adds short description/status to build status
// * returns an array of result info to add to the all_build summary
def reportStatus(label, type, bldurl) {
def outstr = ''
def loglink = "[console](${bldurl}/console)"
def logfile = "${label}.txt"
if ( fileExists(logfile) ) {
archiveArtifacts( artifacts: logfile )
outstr = readFile(logfile)
loglink = "[logfile](${bldurl}/artifact/${logfile})"
}
def st = getResults(outstr, label)
def time = getTime(outstr, label)
def fail_count = getFailures(outstr, label)
outstr = null
def txtcolor =
fail_count == 0 ? 'DarkGreen' : 'Crimson'
def shortbld = label
shortbld = shortbld.replace('debug', 'dbg')
shortbld = shortbld.replace('release', 'rel')
shortbld = shortbld.replace('unity', 'un')
manager.addShortText(
"${shortbld}: ${st}, t: ${time}",
txtcolor,
'white',
'0px',
'white')
[fail_count == 0, type, "${st}, t: ${time}", loglink]
}