mirror of
https://github.com/XRPLF/rippled.git
synced 2025-11-28 23:15:52 +00:00
393ca87572
The automated build system only builds packages signed with a list of approved keys. This is a security measure to prevent someone who gains push access to the repository from producing potentially malicious packages that are signed by Ripple's trusted private keys. Moving this list to the new location makes it easy to add and delete new keys to the list.
623 lines
18 KiB
YAML
623 lines
18 KiB
YAML
#########################################################################
|
|
## ##
|
|
## gitlab CI defintition for rippled build containers and distro ##
|
|
## packages (rpm and dpkg). ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
# NOTE: these are sensible defaults for Ripple pipelines. These
|
|
# can be overridden by project or group variables as needed.
|
|
variables:
|
|
# these containers are built manually using the rippled
|
|
# cmake build (container targets) and tagged/pushed so they
|
|
# can be used here
|
|
RPM_CONTAINER_TAG: "2020-02-10"
|
|
RPM_CONTAINER_NAME: "rippled-rpm-builder"
|
|
RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
|
|
DPKG_CONTAINER_TAG: "2020-02-10"
|
|
DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
|
|
DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
|
|
ARTIFACTORY_HOST: "artifactory.ops.ripple.com"
|
|
ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555"
|
|
GIT_SIGN_PUBKEYS_URL: "https://gitlab.ops.ripple.com/xrpledger/rippled-packages/snippets/49/raw"
|
|
PUBLIC_REPO_ROOT: "https://repos.ripple.com/repos"
|
|
# also need to define this variable ONLY for the primary
|
|
# build/publish pipeline on the mainline repo:
|
|
# IS_PRIMARY_REPO = "true"
|
|
|
|
stages:
|
|
- build_packages
|
|
- sign_packages
|
|
- smoketest
|
|
- verify_sig
|
|
- tag_images
|
|
- push_to_test
|
|
- verify_from_test
|
|
- wait_approval_prod
|
|
- push_to_prod
|
|
- verify_from_prod
|
|
- get_final_hashes
|
|
- build_containers
|
|
|
|
.dind_template: &dind_param
|
|
before_script:
|
|
- . ./Builds/containers/gitlab-ci/docker_alpine_setup.sh
|
|
variables:
|
|
docker_driver: overlay2
|
|
image:
|
|
name: docker:latest
|
|
services:
|
|
# workaround for TLS issues - consider going back
|
|
# back to unversioned `dind` when issues are resolved
|
|
- docker:18-dind
|
|
tags:
|
|
- docker-4xlarge
|
|
|
|
.only_primary_template: &only_primary
|
|
only:
|
|
refs:
|
|
- /^(master|release|develop)$/
|
|
variables:
|
|
- $IS_PRIMARY_REPO == "true"
|
|
|
|
.smoketest_local_template: &run_local_smoketest
|
|
tags:
|
|
- xlarge
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/smoketest.sh local
|
|
|
|
.smoketest_repo_template: &run_repo_smoketest
|
|
tags:
|
|
- xlarge
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/smoketest.sh repo
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: build_packages ##
|
|
## ##
|
|
## build packages using containers from previous stage. ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
rpm_build:
|
|
stage: build_packages
|
|
<<: *dind_param
|
|
artifacts:
|
|
paths:
|
|
- build/rpm/packages/
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/build_package.sh rpm
|
|
|
|
dpkg_build:
|
|
stage: build_packages
|
|
<<: *dind_param
|
|
artifacts:
|
|
paths:
|
|
- build/dpkg/packages/
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/build_package.sh dpkg
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: sign_packages ##
|
|
## ##
|
|
## build packages using containers from previous stage. ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
rpm_sign:
|
|
stage: sign_packages
|
|
dependencies:
|
|
- rpm_build
|
|
image:
|
|
name: centos:7
|
|
<<: *only_primary
|
|
before_script:
|
|
- |
|
|
# Make sure GnuPG is installed
|
|
yum -y install gnupg rpm-sign
|
|
# checking GPG signing support
|
|
if [ -n "$GPG_KEY_B64" ]; then
|
|
echo "$GPG_KEY_B64"| base64 -d | gpg --batch --no-tty --allow-secret-key-import --import -
|
|
unset GPG_KEY_B64
|
|
export GPG_PASSPHRASE=$(echo $GPG_KEY_PASS_B64 | base64 -di)
|
|
unset GPG_KEY_PASS_B64
|
|
export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
|
|
else
|
|
echo -e "\033[0;31m****** GPG signing disabled ******\033[0m"
|
|
exit 1
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- build/rpm/packages/
|
|
script:
|
|
- ls -alh build/rpm/packages
|
|
- . ./Builds/containers/gitlab-ci/sign_package.sh rpm
|
|
|
|
dpkg_sign:
|
|
stage: sign_packages
|
|
dependencies:
|
|
- dpkg_build
|
|
image:
|
|
name: ubuntu:18.04
|
|
<<: *only_primary
|
|
before_script:
|
|
- |
|
|
# make sure we have GnuPG
|
|
apt update
|
|
apt install -y gpg dpkg-sig
|
|
# checking GPG signing support
|
|
if [ -n "$GPG_KEY_B64" ]; then
|
|
echo "$GPG_KEY_B64"| base64 -d | gpg --batch --no-tty --allow-secret-key-import --import -
|
|
unset GPG_KEY_B64
|
|
export GPG_PASSPHRASE=$(echo $GPG_KEY_PASS_B64 | base64 -di)
|
|
unset GPG_KEY_PASS_B64
|
|
export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
|
|
else
|
|
echo -e "\033[0;31m****** GPG signing disabled ******\033[0m"
|
|
exit 1
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- build/dpkg/packages/
|
|
script:
|
|
- ls -alh build/dpkg/packages
|
|
- . ./Builds/containers/gitlab-ci/sign_package.sh dpkg
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: smoketest ##
|
|
## ##
|
|
## install unsigned packages from previous step and run unit tests. ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
centos_7_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- rpm_build
|
|
- rpm_sign
|
|
image:
|
|
name: centos:7
|
|
<<: *run_local_smoketest
|
|
|
|
fedora_29_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- rpm_build
|
|
- rpm_sign
|
|
image:
|
|
name: fedora:29
|
|
<<: *run_local_smoketest
|
|
|
|
fedora_28_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- rpm_build
|
|
- rpm_sign
|
|
image:
|
|
name: fedora:28
|
|
<<: *run_local_smoketest
|
|
|
|
fedora_27_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- rpm_build
|
|
- rpm_sign
|
|
image:
|
|
name: fedora:27
|
|
<<: *run_local_smoketest
|
|
|
|
## this one is not LTS, but we
|
|
## get some extra coverage by including it
|
|
## consider dropping it when 20.04 is ready
|
|
ubuntu_19_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- dpkg_build
|
|
- dpkg_sign
|
|
image:
|
|
name: ubuntu:19.04
|
|
<<: *run_local_smoketest
|
|
|
|
ubuntu_18_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- dpkg_build
|
|
- dpkg_sign
|
|
image:
|
|
name: ubuntu:18.04
|
|
<<: *run_local_smoketest
|
|
|
|
ubuntu_16_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- dpkg_build
|
|
- dpkg_sign
|
|
image:
|
|
name: ubuntu:16.04
|
|
<<: *run_local_smoketest
|
|
|
|
debian_9_smoketest:
|
|
stage: smoketest
|
|
dependencies:
|
|
- dpkg_build
|
|
- dpkg_sign
|
|
image:
|
|
name: debian:9
|
|
<<: *run_local_smoketest
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: verify_sig ##
|
|
## ##
|
|
## use git/gpg to verify that HEAD is signed by an approved ##
|
|
## committer. The whitelist of pubkeys is manually mantained ##
|
|
## and fetched from GIT_SIGN_PUBKEYS_URL (currently a snippet ##
|
|
## link). ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
verify_head_signed:
|
|
stage: verify_sig
|
|
image:
|
|
name: ubuntu:latest
|
|
<<: *only_primary
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/verify_head_commit.sh
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: tag_images ##
|
|
## ##
|
|
## apply rippled version tag to containers from previous stage. ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
tag_bld_images:
|
|
stage: tag_images
|
|
variables:
|
|
docker_driver: overlay2
|
|
image:
|
|
name: docker:latest
|
|
services:
|
|
# workaround for TLS issues - consider going back
|
|
# back to unversioned `dind` when issues are resolved
|
|
- docker:18-dind
|
|
tags:
|
|
- docker-large
|
|
dependencies:
|
|
- rpm_sign
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/tag_docker_image.sh
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: push_to_test ##
|
|
## ##
|
|
## push packages to artifactory repositories (test) ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
push_test:
|
|
stage: push_to_test
|
|
variables:
|
|
DEB_REPO: "rippled-deb-test-mirror"
|
|
RPM_REPO: "rippled-rpm-test-mirror"
|
|
image:
|
|
name: alpine:latest
|
|
artifacts:
|
|
paths:
|
|
- files.info
|
|
dependencies:
|
|
- rpm_sign
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/push_to_artifactory.sh "PUT" "."
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: verify_from_test ##
|
|
## ##
|
|
## install/test packages from test repos. ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
centos_7_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
RPM_REPO: "rippled-rpm-test-mirror"
|
|
image:
|
|
name: centos:7
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_29_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
RPM_REPO: "rippled-rpm-test-mirror"
|
|
image:
|
|
name: fedora:29
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_28_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
RPM_REPO: "rippled-rpm-test-mirror"
|
|
image:
|
|
name: fedora:28
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_27_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
RPM_REPO: "rippled-rpm-test-mirror"
|
|
image:
|
|
name: fedora:27
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_19_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
DISTRO: "disco"
|
|
DEB_REPO: "rippled-deb-test-mirror"
|
|
image:
|
|
name: ubuntu:19.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_18_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
DISTRO: "bionic"
|
|
DEB_REPO: "rippled-deb-test-mirror"
|
|
image:
|
|
name: ubuntu:18.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_16_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
DISTRO: "xenial"
|
|
DEB_REPO: "rippled-deb-test-mirror"
|
|
image:
|
|
name: ubuntu:16.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
debian_9_verify_repo_test:
|
|
stage: verify_from_test
|
|
variables:
|
|
DISTRO: "stretch"
|
|
DEB_REPO: "rippled-deb-test-mirror"
|
|
image:
|
|
name: debian:9
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: wait_approval_prod ##
|
|
## ##
|
|
## wait for manual approval before proceeding to next stage ##
|
|
## which pushes to prod repo. ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
wait_before_push_prod:
|
|
stage: wait_approval_prod
|
|
image:
|
|
name: alpine:latest
|
|
<<: *only_primary
|
|
script:
|
|
- echo "proceeding to next stage"
|
|
when: manual
|
|
allow_failure: false
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: push_to_prod ##
|
|
## ##
|
|
## push packages to artifactory repositories (prod) ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
push_prod:
|
|
variables:
|
|
DEB_REPO: "rippled-deb"
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: alpine:latest
|
|
stage: push_to_prod
|
|
artifacts:
|
|
paths:
|
|
- files.info
|
|
dependencies:
|
|
- rpm_sign
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/push_to_artifactory.sh "PUT" "."
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: verify_from_prod ##
|
|
## ##
|
|
## install/test packages from prod repos. ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
centos_7_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: centos:7
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_29_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: fedora:29
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_28_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: fedora:28
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
fedora_27_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: fedora:27
|
|
dependencies:
|
|
- rpm_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_19_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
DISTRO: "disco"
|
|
DEB_REPO: "rippled-deb"
|
|
image:
|
|
name: ubuntu:19.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_18_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
DISTRO: "bionic"
|
|
DEB_REPO: "rippled-deb"
|
|
image:
|
|
name: ubuntu:18.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
ubuntu_16_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
DISTRO: "xenial"
|
|
DEB_REPO: "rippled-deb"
|
|
image:
|
|
name: ubuntu:16.04
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
debian_9_verify_repo_prod:
|
|
stage: verify_from_prod
|
|
variables:
|
|
DISTRO: "stretch"
|
|
DEB_REPO: "rippled-deb"
|
|
image:
|
|
name: debian:9
|
|
dependencies:
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
<<: *run_repo_smoketest
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: get_final_hashes ##
|
|
## ##
|
|
## fetch final hashes from artifactory. ##
|
|
## ONLY RUNS FOR PRIMARY BRANCHES/REPO ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
get_prod_hashes:
|
|
variables:
|
|
DEB_REPO: "rippled-deb"
|
|
RPM_REPO: "rippled-rpm"
|
|
image:
|
|
name: alpine:latest
|
|
stage: get_final_hashes
|
|
artifacts:
|
|
paths:
|
|
- files.info
|
|
dependencies:
|
|
- rpm_sign
|
|
- dpkg_sign
|
|
<<: *only_primary
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/push_to_artifactory.sh "GET" ".checksums"
|
|
|
|
#########################################################################
|
|
## ##
|
|
## stage: build_containers ##
|
|
## ##
|
|
## build containers from docker definitions. These containers are NOT ##
|
|
## used for the package build. This step is only used to ensure that ##
|
|
## the package build targets and files are still working properly. ##
|
|
## ##
|
|
#########################################################################
|
|
|
|
build_centos_container:
|
|
stage: build_containers
|
|
<<: *dind_param
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/build_container.sh rpm
|
|
allow_failure: true
|
|
|
|
build_ubuntu_container:
|
|
stage: build_containers
|
|
<<: *dind_param
|
|
script:
|
|
- . ./Builds/containers/gitlab-ci/build_container.sh dpkg
|
|
allow_failure: true
|
|
|
|
|