Pkgbld - Make approval blocking, add slack summary message

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -61,6 +61,7 @@ stages:
       - $IS_PRIMARY_REPO == "true"
       - $REQUIRE_APPROVAL == "true"
   when: manual
+  allow_failure: false
 
 .smoketest_local_template: &run_local_smoketest
   tags:

Builds/containers/gitlab-ci/push_to_artifactory.sh

@@ -22,8 +22,13 @@ DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64"
 for dist in stretch buster xenial bionic disco ; do
     DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}"
 done
+echo "{ \"debs\": {" > "${TOPDIR}/files.info"
 for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
-    echo "FILE --> ${deb}" | tee -a "${TOPDIR}/files.info"
+    # first item doesn't get a comma separator
+    if [ $deb != $RIPPLED_PKG ] ; then
+        echo "," >> "${TOPDIR}/files.info"
+    fi
+    echo "\"${deb}\"": | tee -a "${TOPDIR}/files.info"
     ca="${CURLARGS}"
     if [ "${action}" = "PUT" ] ; then
         url="https://${ARTIFACTORY_HOST}/artifactory/${DEB_REPO}/pool/${deb}${DEB_MATRIX}"
@@ -31,9 +36,10 @@ for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
     elif [ "${action}" = "GET" ] ; then
         url="https://${ARTIFACTORY_HOST}/artifactory/api/storage/${DEB_REPO}/pool/${deb}"
     fi
-    echo "url --> ${url}"
+    echo "file info request url --> ${url}"
     eval "curl ${ca} \"${url}\"" | jq -M "${filter}" | tee -a "${TOPDIR}/files.info"
 done
+echo "}," >> "${TOPDIR}/files.info"
 
 # RPM
 
@@ -44,8 +50,13 @@ RIPPLED_DEV_PKG=$(ls rippled-devel*.rpm)
 RIPPLED_DBG_PKG=$(ls rippled-debuginfo*.rpm)
 # TODO - where to upload src rpm ?
 RIPPLED_SRC=$(ls rippled-[0-9]*.src.rpm)
+echo "\"rpms\": {" >> "${TOPDIR}/files.info"
 for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
-    echo "FILE --> ${rpm}" | tee -a "${TOPDIR}/files.info"
+    # first item doesn't get a comma separator
+    if [ $rpm != $RIPPLED_PKG ] ; then
+        echo "," >> "${TOPDIR}/files.info"
+    fi
+    echo "\"${rpm}\"": | tee -a "${TOPDIR}/files.info"
     ca="${CURLARGS}"
     if [ "${action}" = "PUT" ] ; then
         url="https://${ARTIFACTORY_HOST}/artifactory/${RPM_REPO}/${COMPONENT}/"
@@ -53,7 +64,28 @@ for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
     elif [ "${action}" = "GET" ] ; then
         url="https://${ARTIFACTORY_HOST}/artifactory/api/storage/${RPM_REPO}/${COMPONENT}/${rpm}"
     fi
-    echo "url --> ${url}"
+    echo "file info request url --> ${url}"
     eval "curl ${ca} \"${url}\"" | jq -M "${filter}" | tee -a "${TOPDIR}/files.info"
 done
+echo "}}" >> "${TOPDIR}/files.info"
+jq '.' "${TOPDIR}/files.info" > "${TOPDIR}/files.info.tmp"
+mv "${TOPDIR}/files.info.tmp" "${TOPDIR}/files.info"
+
+if [ ! -z "${SLACK_NOTIFY_URL}" ] && [ "${action}" = "GET" ] ; then
+    # extract files.info content to variable and sanitize so it can
+    # be interpolated into a slack text field below
+    finfo=$(cat ${TOPDIR}/files.info | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/\\n/g' | sed -E 's/"/\\"/g')
+    # try posting file info to slack.
+    # can add channel field to payload if the
+    # default channel is incorrect. Get rid of
+    # newlines in payload json since slack doesn't accept them
+    CONTENT=$(tr -d '[\n]' <<JSON
+       payload={
+         "username": "GitlabCI",
+         "text": "The package build for branch \`${CI_COMMIT_REF_NAME}\` is complete. File hashes are: \`\`\`${finfo}\`\`\`",
+         "icon_emoji": ":package:"}
+JSON
+)
+    curl ${SLACK_NOTIFY_URL} --data-urlencode "${CONTENT}"
+fi
 

Builds/containers/gitlab-ci/smoketest.sh

@@ -25,24 +25,33 @@ case ${ID} in
         ;;
 esac
 
+# this script provides info variables about pkg version
+. build/${pkgtype}/packages/build_vars
+
 if [ "${pkgtype}" = "dpkg" ] ; then
+    # sometimes update fails and requires a cleanup
+    updateWithRetry()
+    {
+        if ! apt-get -y update ; then
+            rm -rvf /var/lib/apt/lists/*
+            apt-get -y clean
+            apt-get -y update
+        fi
+    }
     if [ "${install_from}" = "repo" ] ; then
-        apt -y upgrade
+        apt-get -y upgrade
-        apt -y update
+        updateWithRetry
-        apt -y install apt apt-transport-https ca-certificates coreutils util-linux wget gnupg
+        apt-get -y install apt apt-transport-https ca-certificates coreutils util-linux wget gnupg
         wget -q -O - "${REPO_ROOT}/api/gpg/key/public" | apt-key add -
         echo "deb ${REPO_ROOT}/${DEB_REPO} ${DISTRO} ${COMPONENT}" >> /etc/apt/sources.list
-        # sometimes update fails and requires a cleanup
+        updateWithRetry
-        if ! apt -y update ; then
+        # uncomment this next line if you want to see the available package versions
-          rm -rvf /var/lib/apt/lists/*
+        # apt-cache policy rippled
-          apt-get clean
+        apt-get -y install rippled=${dpkg_full_version}
-          apt -y update
-        fi
-        apt-get -y install rippled
     elif [ "${install_from}" = "local" ] ; then
         # cached pkg install
-        apt -y update
+        updateWithRetry
-        apt -y install libprotobuf-dev libssl-dev
+        apt-get -y install libprotobuf-dev libssl-dev
         rm -f build/dpkg/packages/rippled-dbgsym*.*
         dpkg --no-debsig -i build/dpkg/packages/*.deb
     else
@@ -62,7 +71,9 @@ else
         echo "gpgkey=${REPO_ROOT}/${RPM_REPO}/${COMPONENT}/repodata/repomd.xml.key" >> ${REPOFILE}
         echo "repo_gpgcheck=1" >> ${REPOFILE}
         yum -y update
-        yum -y install rippled
+        # uncomment this next line if you want to see the available package versions
+        # yum --showduplicates list rippled
+        yum -y install ${rpm_version_release}
     elif [ "${install_from}" = "local" ] ; then
         # cached pkg install
         yum install -y yum-utils openssl-static zlib-static
@@ -77,7 +88,6 @@ fi
 
 # verify installed version
 INSTALLED=$(/opt/ripple/bin/rippled --version | awk '{print $NF}')
-. build/${pkgtype}/packages/build_vars
 if [ "${rippled_version}" != "${INSTALLED}" ] ; then
     echo "INSTALLED version ${INSTALLED} does not match ${rippled_version}"
     exit 1

Builds/containers/packaging/dpkg/build_dpkg.sh

@@ -7,6 +7,9 @@ source update_sources.sh
 
 #dpkg uses - as separator, so we need to change our -bN versions to tilde
 RIPPLED_DPKG_VERSION=$(echo "${RIPPLED_VERSION}" | sed 's!-!~!g')
+# TODO - decide how to handle the trailing/release
+# version here (hardcoded to 1). Does it ever need to change?
+RIPPLED_DPKG_FULL_VERSION="${RIPPLED_DPKG_VERSION}-1"
 
 cd rippled
 if [[ -n $(git status --porcelain) ]]; then
@@ -27,7 +30,7 @@ cp -pr ../debian .
 # release packages (?)
 NOWSTR=$(TZ=UTC date -R)
 cat << CHANGELOG > ./debian/changelog
-rippled (${RIPPLED_DPKG_VERSION}-1) unstable; urgency=low
+rippled (${RIPPLED_DPKG_FULL_VERSION}) unstable; urgency=low
 
   * see RELEASENOTES
 
@@ -45,21 +48,21 @@ cd ..
 ls -latr
 
 # copy artifacts
-cp rippled-dev_${RIPPLED_DPKG_VERSION}-1_amd64.deb ${PKG_OUTDIR}
+cp rippled-dev_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR}
-cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.deb ${PKG_OUTDIR}
+cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.deb ${PKG_OUTDIR}
-cp rippled_${RIPPLED_DPKG_VERSION}-1.dsc ${PKG_OUTDIR}
+cp rippled_${RIPPLED_DPKG_FULL_VERSION}.dsc ${PKG_OUTDIR}
 # dbgsym suffix is ddeb under newer debuild, but just deb under earlier
-cp rippled-dbgsym_${RIPPLED_DPKG_VERSION}-1_amd64.* ${PKG_OUTDIR}
+cp rippled-dbgsym_${RIPPLED_DPKG_FULL_VERSION}_amd64.* ${PKG_OUTDIR}
-cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.changes ${PKG_OUTDIR}
+cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.changes ${PKG_OUTDIR}
-cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.build ${PKG_OUTDIR}
+cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.build ${PKG_OUTDIR}
 cp rippled_${RIPPLED_DPKG_VERSION}.orig.tar.gz ${PKG_OUTDIR}
-cp rippled_${RIPPLED_DPKG_VERSION}-1.debian.tar.xz ${PKG_OUTDIR}
+cp rippled_${RIPPLED_DPKG_FULL_VERSION}.debian.tar.xz ${PKG_OUTDIR}
 # buildinfo is only generated by later version of debuild
-if [ -e rippled_${RIPPLED_DPKG_VERSION}-1_amd64.buildinfo ] ; then
+if [ -e rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.buildinfo ] ; then
-    cp rippled_${RIPPLED_DPKG_VERSION}-1_amd64.buildinfo ${PKG_OUTDIR}
+    cp rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.buildinfo ${PKG_OUTDIR}
 fi
 
-cat rippled_${RIPPLED_DPKG_VERSION}-1_amd64.changes
+cat rippled_${RIPPLED_DPKG_FULL_VERSION}_amd64.changes
 # extract the text in the .changes file that appears between
 #    Checksums-Sha256:  ...
 # and
@@ -81,4 +84,5 @@ echo "dev_sha256=${DEV_SHA256}" >> ${PKG_OUTDIR}/build_vars
 echo "src_sha256=${SRC_SHA256}" >> ${PKG_OUTDIR}/build_vars
 echo "rippled_version=${RIPPLED_VERSION}" >> ${PKG_OUTDIR}/build_vars
 echo "dpkg_version=${RIPPLED_DPKG_VERSION}" >> ${PKG_OUTDIR}/build_vars
+echo "dpkg_full_version=${RIPPLED_DPKG_FULL_VERSION}" >> ${PKG_OUTDIR}/build_vars