Add getRippledInfo to packages:

* include info gathering script in rippled linux pkgs
* modify script to output results to single markdown file
* rename info script

Builds/CMake/RippledRelease.cmake

@@ -54,6 +54,7 @@ if (is_root_project)
         Builds/containers/shared/update-rippled.sh
         Builds/containers/packaging/rpm/rippled.spec
         Builds/containers/packaging/rpm/build_rpm.sh
+        bin/getRippledInfo
     )
     exclude_from_default (rpm_container)
     add_custom_target (rpm
@@ -64,6 +65,7 @@ if (is_root_project)
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
         "$<$<BOOL:${map_user}>:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>"
         -t rippled-rpm-builder:${container_label}
+        /bin/bash -c "cp -pu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh"
       VERBATIM
       USES_TERMINAL
       COMMAND_EXPAND_LISTS
@@ -115,6 +117,7 @@ if (is_root_project)
         Builds/containers/packaging/dpkg/debian/rippled.preinst
         Builds/containers/packaging/dpkg/debian/rippled.prerm
         Builds/containers/packaging/dpkg/debian/rules
+        bin/getRippledInfo
     )
     exclude_from_default (dpkg_container)
     add_custom_target (dpkg
@@ -125,6 +128,7 @@ if (is_root_project)
         -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out
         "$<$<BOOL:${map_user}>:--volume=/etc/passwd:/etc/passwd;--volume=/etc/group:/etc/group;--user=${DOCKER_USER_ID}:${DOCKER_GROUP_ID}>"
         -t rippled-dpkg-builder:${container_label}
+        /bin/bash -c "cp -pu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh"
       VERBATIM
       USES_TERMINAL
       COMMAND_EXPAND_LISTS
@@ -177,9 +181,6 @@ if (is_root_project)
         Builds/containers/ubuntu-builder/Dockerfile
         Builds/containers/ubuntu-builder/ubuntu_setup.sh
         Builds/containers/shared/build_deps.sh
-        Builds/containers/shared/rippled.service
-        Builds/containers/shared/update_sources.sh
-        Builds/containers/shared/update-rippled.sh
     )
     exclude_from_default (ci_container)
   else ()

Builds/containers/centos-builder/Dockerfile

@@ -34,9 +34,6 @@ RUN source scl_source enable devtoolset-7 python27 && \
 # prep files for package building
 RUN mkdir -m 777 -p /opt/rippled_bld/pkg
 WORKDIR /opt/rippled_bld/pkg
-
-COPY packaging/rpm/rippled.spec ./
-COPY shared/update_sources.sh ./
 RUN mkdir -m 777 ./rpmbuild
 RUN mkdir -m 777 ./rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
 

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -11,10 +11,10 @@ variables:
   # these containers are built manually using the rippled
   # cmake build (container targets) and tagged/pushed so they 
   # can be used here
-  RPM_CONTAINER_TAG: "2020-01-08"
+  RPM_CONTAINER_TAG: "2020-02-10"
   RPM_CONTAINER_NAME: "rippled-rpm-builder"
   RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}"
-  DPKG_CONTAINER_TAG: "2020-01-08"
+  DPKG_CONTAINER_TAG: "2020-02-10"
   DPKG_CONTAINER_NAME: "rippled-dpkg-builder"
   DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}"
   ARTIFACTORY_HOST: "artifactory.ops.ripple.com"

Builds/containers/packaging/dpkg/build_dpkg.sh

@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
 set -ex
 
+# make sure pkg source files are up to date with repo
+cd /opt/rippled_bld/pkg
+cp -pru rippled/Builds/containers/packaging/dpkg/debian/. debian/
+cp -pu rippled/Builds/containers/shared/rippled.service debian/
+cp -pu rippled/Builds/containers/shared/update_sources.sh .
 source update_sources.sh
 
 # Build the dpkg

Builds/containers/packaging/dpkg/debian/rippled.install

@@ -1,6 +1,7 @@
 opt/ripple/bin/rippled
 opt/ripple/bin/validator-keys
 opt/ripple/bin/update-rippled.sh
+opt/ripple/bin/getRippledInfo
 opt/ripple/etc/rippled.cfg
 opt/ripple/etc/validators.txt
 opt/ripple/etc/update-rippled-cron

Builds/containers/packaging/dpkg/debian/rules

@@ -32,6 +32,7 @@ override_dh_auto_install:
 	cd bld && DESTDIR=../debian/tmp cmake --build . --target install -- -v
 	install -D bld/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys
 	install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh
+	install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo
 	install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron
 	install -D Builds/containers/shared/rippled-logrotate debian/tmp/etc/logrotate.d/rippled
 	rm -rf debian/tmp/opt/ripple/lib64/cmake/date

Builds/containers/packaging/rpm/build_rpm.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -ex
 
+cd /opt/rippled_bld/pkg
+cp -pu rippled/Builds/containers/packaging/rpm/rippled.spec .
+cp -pu rippled/Builds/containers/shared/update_sources.sh .
 source update_sources.sh
 
 # Build the rpm

Builds/containers/packaging/rpm/rippled.spec

@@ -51,6 +51,7 @@ install -D rippled/bld.release/validator-keys/validator-keys ${RPM_BUILD_ROOT}%{
 install -D ./rippled/Builds/containers/shared/rippled.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled.service
 install -D ./rippled/Builds/containers/packaging/rpm/50-rippled.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled.preset
 install -D ./rippled/Builds/containers/shared/update-rippled.sh ${RPM_BUILD_ROOT}%{_bindir}/update-rippled.sh
+install -D ./rippled/bin/getRippledInfo ${RPM_BUILD_ROOT}%{_bindir}/getRippledInfo
 install -D ./rippled/Builds/containers/shared/update-rippled-cron ${RPM_BUILD_ROOT}%{_prefix}/etc/update-rippled-cron
 install -D ./rippled/Builds/containers/shared/rippled-logrotate ${RPM_BUILD_ROOT}/etc/logrotate.d/rippled
 install -d $RPM_BUILD_ROOT/var/log/rippled
@@ -79,6 +80,7 @@ chown -R root:$GROUP_NAME %{_prefix}/etc/update-rippled-cron
 %{_bindir}/rippled
 /usr/local/bin/rippled
 %{_bindir}/update-rippled.sh
+%{_bindir}/getRippledInfo
 %{_prefix}/etc/update-rippled-cron
 %{_bindir}/validator-keys
 %config(noreplace) %{_prefix}/etc/rippled.cfg

Builds/containers/ubuntu-builder/Dockerfile

@@ -27,15 +27,10 @@ ENV BOOST_ROOT="/opt/local/boost/_INSTALLED_"
 ENV OPENSSL_ROOT="/opt/local/openssl"
 
 # prep files for package building
-RUN mkdir -m 777 -p /opt/rippled_bld/pkg
+RUN mkdir -m 777 -p /opt/rippled_bld/pkg/debian
+RUN update-alternatives --set gcc /usr/bin/gcc-8
 WORKDIR /opt/rippled_bld/pkg
 
-COPY packaging/dpkg/debian /opt/rippled_bld/pkg/debian/
-COPY shared/update_sources.sh ./
-COPY shared/rippled.service /opt/rippled_bld/pkg/debian/
-
 COPY packaging/dpkg/build_dpkg.sh ./
-RUN update-alternatives --set gcc /usr/bin/gcc-8
-
 CMD ./build_dpkg.sh
 

bin/getInfoRippled.sh

@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-rippled_exe=/opt/ripple/bin/rippled
-conf_file=/etc/opt/ripple/rippled.cfg
-
-while getopts ":e:c:" opt; do
-    case $opt in
-        e)
-            rippled_exe=${OPTARG}
-            ;;
-        c)
-            conf_file=${OPTARG}
-            ;;
-        \?)
-            echo "Invalid option: -$OPTARG"
-    esac
-done
-
-tmp_loc=$(mktemp -d --tmpdir ripple_info.XXXX)
-cd /tmp
-chmod 751 ripple_info.*
-cd ~
-echo ${tmp_loc}
-
-cleaned_conf=${tmp_loc}/cleaned_rippled_cfg.txt
-
-if [[ -f ${conf_file} ]]
-then
-    db=$(sed -r -e 's/\<s[a-zA-Z0-9]{28}\>/secretsecretsecretsecretmaybe/g' ${conf_file} |\
-            awk -v OUT_FILE=${cleaned_conf} '
-    BEGIN {skip=0; db_path="";print > OUT_FILE}
-    /^\[validation_seed\]/ {skip=1; next}
-    /^\[node_seed\]/ {skip=1; next}
-    /^\[validation_manifest\]/ {skip=1; next}
-    /^\[validator_token\]/ {skip=1; next}
-    /^\[.*\]/ {skip=0}
-    skip==1 {next}
-    save==1 {save=0;db_path=$0}
-    /^\[database_path\]/ {save=1}
-    {print >> OUT_FILE}
-    END {print db_path}
-    ')
-fi
-
-echo "database_path: ${db}"
-df ${db} > ${tmp_loc}/db_path_df.txt
-echo
-
-# Send output from this script to a log file
-## this captures any messages
-## or errors from the script itself
-
-log_file=${tmp_loc}/get_info.log
-exec 3>&1 1>>${log_file} 2>&1
-
-## Send all stdout files to /tmp
-
-if [[ -x ${rippled_exe} ]]
-then
-    pgrep rippled && \
-    ${rippled_exe} --conf ${conf_file} \
-    -- server_info                  > ${tmp_loc}/server_info.txt
-fi
-
-df -h                               > ${tmp_loc}/free_disk_space.txt
-cat /proc/meminfo                   > ${tmp_loc}/amount_mem.txt
-cat /proc/swaps                     > ${tmp_loc}/swap_space.txt
-ulimit -a                           > ${tmp_loc}/reported_current_limits.txt
-
-for dev_path in $(df | awk '$1 ~ /^\/dev\// {print $1}'); do
-    # strip numbers from end and remove '/dev/'
-    dev=$(basename ${dev_path%%[0-9]})
-    if [[ "$(cat /sys/block/${dev}/queue/rotational)" = 0 ]]
-    then
-        echo "${dev} : SSD" >> ${tmp_loc}/is_ssd.txt
-    else
-        echo "${dev} : NO SSD" >> ${tmp_loc}/is_ssd.txt
-    fi
-done
-
-pushd ${tmp_loc}
-tar -czvf info-package.tar.gz *.txt *.log
-popd
-
-echo "Use the following command on your local machine to download from your rippled instance: scp <remote_rippled_username>@<remote_host>:${tmp_loc}/info-package.tar.gz <path/to/local_machine/directory>"| tee /dev/fd/3
-

bin/getRippledInfo

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+
+rippled_exe=/opt/ripple/bin/rippled
+conf_file=/etc/opt/ripple/rippled.cfg
+
+while getopts ":e:c:" opt; do
+    case $opt in
+        e)
+            rippled_exe=${OPTARG}
+            ;;
+        c)
+            conf_file=${OPTARG}
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            exit -1
+    esac
+done
+
+tmp_loc=$(mktemp -d --tmpdir ripple_info.XXXXX)
+chmod 751 ${tmp_loc}
+awk_prog=${tmp_loc}/cfg.awk
+summary_out=${tmp_loc}/rippled_info.md
+printf "# rippled report info\n\n> generated at %s\n" "$(date -R)" > ${summary_out}
+
+function log_section {
+    printf "\n## %s\n" "$*" >> ${summary_out}
+
+    while read -r l; do
+        echo "    $l" >> ${summary_out}
+    done </dev/stdin
+}
+
+function join_by {
+    local IFS="$1"; shift; echo "$*";
+}
+
+if [[ -f ${conf_file} ]] ; then
+    exclude=( ips ips_fixed node_seed validation_seed validator_token )
+    cleaned_conf=${tmp_loc}/cleaned_rippled_cfg.txt
+    cat << 'EOP' >> ${awk_prog}
+    BEGIN {FS="[[:space:]]*=[[:space:]]*"; skip=0; db_path=""; print > OUT_FILE; split(exl,exa,"|")}
+    /^#/ {next}
+    save==2 && /^[[:space:]]*$/ {next}
+    /^\[.+\]$/ {
+      section=tolower(gensub(/^\[[[:space:]]*([a-zA-Z_]+)[[:space:]]*\]$/, "\\1", "g"))
+      skip = 0
+      for (i in exa) {
+        if (section == exa[i])
+          skip = 1
+      }
+      if (section == "database_path")
+        save = 1
+    }
+    skip==1 {next}
+    save==2 {save=0; db_path=$0}
+    save==1 {save=2}
+    $1 ~ /password/ {$0=$1"=<redacted>"}
+    {print >> OUT_FILE}
+    END {print db_path}
+EOP
+
+    db=$(\
+        sed -r -e 's/\<s[[:alnum:]]{28}\>/<redactedsecret>/g;s/^[[:space:]]*//;s/[[:space:]]*$//' ${conf_file} |\
+        awk -v OUT_FILE=${cleaned_conf} -v exl="$(join_by '|' "${exclude[@]}")" -f ${awk_prog})
+    rm ${awk_prog}
+    cat ${cleaned_conf} | log_section "cleaned config file"
+    rm ${cleaned_conf}
+    echo "${db}"  | log_section "database path"
+    df ${db}      | log_section "df: database"
+fi
+
+# Send output from this script to a log file
+## this captures any messages
+## or errors from the script itself
+
+log_file=${tmp_loc}/get_info.log
+exec 3>&1 1>>${log_file} 2>&1
+
+## Send all stdout files to /tmp
+
+if [[ -x ${rippled_exe} ]] ; then
+    pgrep rippled && \
+    ${rippled_exe} --conf ${conf_file} \
+    -- server_info                  | log_section "server info"
+fi
+
+cat /proc/meminfo                   | log_section "meminfo"
+cat /proc/swaps                     | log_section "swap space"
+ulimit -a                           | log_section "ulimit"
+
+if command -v lshw >/dev/null 2>&1 ; then
+    lshw    2>/dev/null             | log_section "hardware info"
+else
+    lscpu                           >  ${tmp_loc}/hw_info.txt
+    hwinfo                          >> ${tmp_loc}/hw_info.txt
+    lspci                           >> ${tmp_loc}/hw_info.txt
+    lsblk                           >> ${tmp_loc}/hw_info.txt
+    cat ${tmp_loc}/hw_info.txt | log_section "hardware info"
+    rm ${tmp_loc}/hw_info.txt
+fi
+
+if command -v iostat >/dev/null 2>&1 ; then
+    iostat -t -d -x 2 6             | log_section "iostat"
+fi
+
+df -h                               | log_section "free disk space"
+drives=($(df | awk '$1 ~ /^\/dev\// {print $1}' | xargs -n 1 basename))
+block_devs=($(ls /sys/block/))
+for d in "${drives[@]}"; do
+    for dev in "${block_devs[@]}"; do
+        #echo "D: [$d], DEV: [$dev]"
+        if [[ $d =~ $dev ]]; then
+            # this file (if exists) has 0 for SSD and 1 for HDD
+            if [[ "$(cat /sys/block/${dev}/queue/rotational 2>/dev/null)" == 0 ]] ; then
+                echo "${d} : SSD" >> ${tmp_loc}/is_ssd.txt
+            else
+                echo "${d} : NO SSD" >> ${tmp_loc}/is_ssd.txt
+            fi
+        fi
+    done
+done
+
+if [[ -f ${tmp_loc}/is_ssd.txt ]] ; then
+    cat ${tmp_loc}/is_ssd.txt | log_section "SSD"
+    rm ${tmp_loc}/is_ssd.txt
+fi
+
+cat ${log_file} | log_section "script log"
+
+cat << MSG | tee /dev/fd/3
+####################################################
+  rippled info has been gathered. Please copy the
+  contents of ${summary_out}
+  to a github gist at https://gist.github.com/
+
+  PLEASE REVIEW THIS FILE FOR ANY SENSITIVE DATA
+  BEFORE POSTING! We have tried our best to omit
+  any sensitive information from this file, but you
+  should verify before posting.
+####################################################
+MSG
+