From c1a1cfe5500deec742d2cf17d6d2ce3afe3e868a Mon Sep 17 00:00:00 2001 From: Mike Ellery Date: Tue, 28 May 2019 12:56:11 -0700 Subject: [PATCH] Pkgbld - Make approval blocking, add slack summary message --- Builds/containers/gitlab-ci/pkgbuild.yml | 1 + .../gitlab-ci/push_to_artifactory.sh | 40 +++++++++++++++++-- Builds/containers/gitlab-ci/smoketest.sh | 38 +++++++++++------- .../containers/packaging/dpkg/build_dpkg.sh | 26 +++++++----- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/Builds/containers/gitlab-ci/pkgbuild.yml b/Builds/containers/gitlab-ci/pkgbuild.yml index 206df79d4a..c3b5ddc7a3 100644 --- a/Builds/containers/gitlab-ci/pkgbuild.yml +++ b/Builds/containers/gitlab-ci/pkgbuild.yml @@ -61,6 +61,7 @@ stages: - $IS_PRIMARY_REPO == "true" - $REQUIRE_APPROVAL == "true" when: manual + allow_failure: false .smoketest_local_template: &run_local_smoketest tags: diff --git a/Builds/containers/gitlab-ci/push_to_artifactory.sh b/Builds/containers/gitlab-ci/push_to_artifactory.sh index 8033a44047..43efb668df 100644 --- a/Builds/containers/gitlab-ci/push_to_artifactory.sh +++ b/Builds/containers/gitlab-ci/push_to_artifactory.sh @@ -22,8 +22,13 @@ DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64" for dist in stretch buster xenial bionic disco ; do DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}" done +echo "{ \"debs\": {" > "${TOPDIR}/files.info" for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do - echo "FILE --> ${deb}" | tee -a "${TOPDIR}/files.info" + # first item doesn't get a comma separator + if [ $deb != $RIPPLED_PKG ] ; then + echo "," >> "${TOPDIR}/files.info" + fi + echo "\"${deb}\"": | tee -a "${TOPDIR}/files.info" ca="${CURLARGS}" if [ "${action}" = "PUT" ] ; then url="https://${ARTIFACTORY_HOST}/artifactory/${DEB_REPO}/pool/${deb}${DEB_MATRIX}" @@ -31,9 +36,10 @@ for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do elif [ "${action}" = "GET" ] ; then url="https://${ARTIFACTORY_HOST}/artifactory/api/storage/${DEB_REPO}/pool/${deb}" fi - echo "url --> ${url}" + echo "file info request url --> ${url}" eval "curl ${ca} \"${url}\"" | jq -M "${filter}" | tee -a "${TOPDIR}/files.info" done +echo "}," >> "${TOPDIR}/files.info" # RPM @@ -44,8 +50,13 @@ RIPPLED_DEV_PKG=$(ls rippled-devel*.rpm) RIPPLED_DBG_PKG=$(ls rippled-debuginfo*.rpm) # TODO - where to upload src rpm ? RIPPLED_SRC=$(ls rippled-[0-9]*.src.rpm) +echo "\"rpms\": {" >> "${TOPDIR}/files.info" for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do - echo "FILE --> ${rpm}" | tee -a "${TOPDIR}/files.info" + # first item doesn't get a comma separator + if [ $rpm != $RIPPLED_PKG ] ; then + echo "," >> "${TOPDIR}/files.info" + fi + echo "\"${rpm}\"": | tee -a "${TOPDIR}/files.info" ca="${CURLARGS}" if [ "${action}" = "PUT" ] ; then url="https://${ARTIFACTORY_HOST}/artifactory/${RPM_REPO}/${COMPONENT}/" @@ -53,7 +64,28 @@ for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do elif [ "${action}" = "GET" ] ; then url="https://${ARTIFACTORY_HOST}/artifactory/api/storage/${RPM_REPO}/${COMPONENT}/${rpm}" fi - echo "url --> ${url}" + echo "file info request url --> ${url}" eval "curl ${ca} \"${url}\"" | jq -M "${filter}" | tee -a "${TOPDIR}/files.info" done +echo "}}" >> "${TOPDIR}/files.info" +jq '.' "${TOPDIR}/files.info" > "${TOPDIR}/files.info.tmp" +mv "${TOPDIR}/files.info.tmp" "${TOPDIR}/files.info" + +if [ ! -z "${SLACK_NOTIFY_URL}" ] && [ "${action}" = "GET" ] ; then + # extract files.info content to variable and sanitize so it can + # be interpolated into a slack text field below + finfo=$(cat ${TOPDIR}/files.info | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/\\n/g' | sed -E 's/"/\\"/g') + # try posting file info to slack. + # can add channel field to payload if the + # default channel is incorrect. Get rid of + # newlines in payload json since slack doesn't accept them + CONTENT=$(tr -d '[\n]' <> /etc/apt/sources.list - # sometimes update fails and requires a cleanup - if ! apt -y update ; then - rm -rvf /var/lib/apt/lists/* - apt-get clean - apt -y update - fi - apt-get -y install rippled + updateWithRetry + # uncomment this next line if you want to see the available package versions + # apt-cache policy rippled + apt-get -y install rippled=${dpkg_full_version} elif [ "${install_from}" = "local" ] ; then # cached pkg install - apt -y update - apt -y install libprotobuf-dev libssl-dev + updateWithRetry + apt-get -y install libprotobuf-dev libssl-dev rm -f build/dpkg/packages/rippled-dbgsym*.* dpkg --no-debsig -i build/dpkg/packages/*.deb else @@ -62,7 +71,9 @@ else echo "gpgkey=${REPO_ROOT}/${RPM_REPO}/${COMPONENT}/repodata/repomd.xml.key" >> ${REPOFILE} echo "repo_gpgcheck=1" >> ${REPOFILE} yum -y update - yum -y install rippled + # uncomment this next line if you want to see the available package versions + # yum --showduplicates list rippled + yum -y install ${rpm_version_release} elif [ "${install_from}" = "local" ] ; then # cached pkg install yum install -y yum-utils openssl-static zlib-static @@ -77,7 +88,6 @@ fi # verify installed version INSTALLED=$(/opt/ripple/bin/rippled --version | awk '{print $NF}') -. build/${pkgtype}/packages/build_vars if [ "${rippled_version}" != "${INSTALLED}" ] ; then echo "INSTALLED version ${INSTALLED} does not match ${rippled_version}" exit 1 diff --git a/Builds/containers/packaging/dpkg/build_dpkg.sh b/Builds/containers/packaging/dpkg/build_dpkg.sh index 1058a63df5..0b5b8ca034 100755 --- a/Builds/containers/packaging/dpkg/build_dpkg.sh +++ b/Builds/containers/packaging/dpkg/build_dpkg.sh @@ -7,6 +7,9 @@ source update_sources.sh #dpkg uses - as separator, so we need to change our -bN versions to tilde RIPPLED_DPKG_VERSION=$(echo "${RIPPLED_VERSION}" | sed 's!-!~!g') +# TODO - decide how to handle the trailing/release +# version here (hardcoded to 1). Does it ever need to change? +RIPPLED_DPKG_FULL_VERSION="${RIPPLED_DPKG_VERSION}-1" cd rippled if [[ -n $(git status --porcelain) ]]; then @@ -27,7 +30,7 @@ cp -pr ../debian . # release packages (?) NOWSTR=$(TZ=UTC date -R) cat << CHANGELOG > ./debian/changelog -rippled (${RIPPLED_DPKG_VERSION}-1) unstable; urgency=low +rippled (${RIPPLED_DPKG_FULL_VERSION}) unstable; urgency=low * see RELEASENOTES @@ -45,21 +48,21 @@ cd .. ls -latr # copy artifacts -cp rippled-dev_${RIPPLED_DPKG_VERSION}-1_amd64.deb ${PKG_OUTDIR} -cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.deb ${PKG_OUTDIR} -cp rippled_${RIPPLED_DPKG_VERSION}-1.dsc ${PKG_OUTDIR} +cp rippled-dev_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR} +cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR} +cp rippled_${RIPPLED_DPKG_FULL_VERSION}.dsc ${PKG_OUTDIR} # dbgsym suffix is ddeb under newer debuild, but just deb under earlier -cp rippled-dbgsym_${RIPPLED_DPKG_VERSION}-1_amd64.* ${PKG_OUTDIR} -cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.changes ${PKG_OUTDIR} -cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.build ${PKG_OUTDIR} +cp rippled-dbgsym_${RIPPLED_DPKG_FULL_VERSION}_amd64.* ${PKG_OUTDIR} +cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.changes ${PKG_OUTDIR} +cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.build ${PKG_OUTDIR} cp rippled_${RIPPLED_DPKG_VERSION}.orig.tar.gz ${PKG_OUTDIR} -cp rippled_${RIPPLED_DPKG_VERSION}-1.debian.tar.xz ${PKG_OUTDIR} +cp rippled_${RIPPLED_DPKG_FULL_VERSION}.debian.tar.xz ${PKG_OUTDIR} # buildinfo is only generated by later version of debuild -if [ -e rippled_${RIPPLED_DPKG_VERSION}-1_amd64.buildinfo ] ; then - cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.buildinfo ${PKG_OUTDIR} +if [ -e rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.buildinfo ] ; then + cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.buildinfo ${PKG_OUTDIR} fi -cat rippled_${RIPPLED_DPKG_VERSION}-1_amd64.changes +cat rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.changes # extract the text in the .changes file that appears between # Checksums-Sha256: ... # and @@ -81,4 +84,5 @@ echo "dev_sha256=${DEV_SHA256}" >> ${PKG_OUTDIR}/build_vars echo "src_sha256=${SRC_SHA256}" >> ${PKG_OUTDIR}/build_vars echo "rippled_version=${RIPPLED_VERSION}" >> ${PKG_OUTDIR}/build_vars echo "dpkg_version=${RIPPLED_DPKG_VERSION}" >> ${PKG_OUTDIR}/build_vars +echo "dpkg_full_version=${RIPPLED_DPKG_FULL_VERSION}" >> ${PKG_OUTDIR}/build_vars