Squashed 'src/nudb/' content from commit 00adc6a

git-subtree-dir: src/nudb
git-subtree-split: 00adc6a4f16679a376f40c967f77dfa544c179c1

scripts/blacklist.supp

@@ -0,0 +1,38 @@
+# Remember that this blacklist file is GLOBAL to all sanitizers
+# Be therefore extremely careful when considering to add a sanitizer
+# filter here instead of using a runtime suppression
+#
+# Remember also that filters here quite literally completely
+# remove instrumentation altogether, so filtering here means
+# that sanitizers such as tsan will false positive on problems
+# introduced by code filtered here.
+#
+# The main use for this file is ubsan, as it's the only sanitizer
+# without a runtime suppression facility.
+#
+# Be ESPECIALLY careful when filtering out entire source files!
+# Try if at all possible to filter only functions using fun:regex
+# Remember you must use mangled symbol names with fun:regex
+
+#### Compile time filters for ubsan ####
+
+## The well known ubsan failure in libstdc++ extant for years :)
+# Line 96:24: runtime error: load of value 4294967221, which is not a valid value for type 'std::_Ios_Fmtflags'
+fun:*_Ios_Fmtflags*
+
+# boost/any.hpp:259:16: runtime error: downcast of address 0x000004392e70 which does not point to an object of type 'any::holder<int>'
+fun:*any_cast*
+
+# boost/lexical_cast.hpp:1625:43: runtime error: downcast of address 0x7fbb4fffbce8 which does not point to an object of type 'buffer_t' (aka 'parser_buf<std::basic_streambuf<char, char_traits<char> >, char>')
+fun:*shl_input_streamable*
+
+
+
+
+#### Compile time filters for asan ####
+
+
+#### Compile time filters for msan ####
+
+
+#### Compile time filters for tsan ####

scripts/build-and-test.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+# The above bash options do the following:
+
+# -e When this option is on, if a simple command fails for any of the reasons
+#    listed in Consequences of Shell Errors or returns an exit status value >0,
+#    and is not part of the compound list following a while, until, or if
+#    keyword, and is not a part of an AND or OR list, and is not a pipeline
+#    preceded by the ! reserved word, then the shell shall immediately exit.
+# -u The shell shall write a message to standard error when it tries to expand a
+#    variable that is not set and immediately exit. An interactive shell shall
+#    not exit.
+# -x The shell shall write to standard error a trace for each command after it
+#    expands the command and before it executes it. It is unspecified
+#    whether the command that turns tracing off is traced.
+# -o pipefail
+#    Pipelines fail on the first command which fails instead of dying later on
+#    down the pipeline.
+
+shopt -s globstar
+
+################################## ENVIRONMENT #################################
+
+# If not CI, then set some defaults
+if [[ -z ${CI:-} ]]; then
+  : ${TRAVIS_BRANCH:=feature}
+  : ${CC:=gcc}
+  : ${ADDRESS_MODEL:=64}
+  : ${VARIANT:=debug}
+  # If running locally we assume we have lcov/valgrind on PATH
+else
+  export PATH=${VALGRIND_ROOT}/bin:${LCOV_ROOT}/usr/bin:${PATH}
+fi
+
+MAIN_BRANCH=0
+# For builds not triggered by a pull request TRAVIS_BRANCH is the name of the
+# branch currently being built; whereas for builds triggered by a pull request
+# it is the name of the branch targeted by the pull request (in many cases this
+# will be master).
+if [[ ${TRAVIS_BRANCH} == master || ${TRAVIS_BRANCH} == develop ]]; then
+    MAIN_BRANCH=1
+fi
+
+num_jobs=1
+if [[ $(uname) == Darwin ]]; then
+  num_jobs=$(sysctl -n hw.physicalcpu)
+elif [[ $(uname -s) == Linux ]]; then
+  # CircleCI returns 32 phys procs, but 2 virt proc
+  num_proc_units=$(nproc)
+  # Physical cores
+  num_jobs=$(lscpu -p | grep -v '^#' | sort -u -t, -k 2,4 | wc -l)
+  if ((${num_proc_units} < ${num_jobs})); then
+      num_jobs=${num_proc_units}
+  fi
+fi
+
+echo "using toolset: ${CC}"
+echo "using variant: ${VARIANT}"
+echo "using address-model: ${ADDRESS_MODEL}"
+echo "using PATH: ${PATH}"
+echo "using MAIN_BRANCH: ${MAIN_BRANCH}"
+echo "using BOOST_ROOT: ${BOOST_ROOT}"
+
+#################################### HELPERS ###################################
+
+function run_tests_with_debugger {
+  for x in bin/**/${VARIANT}/**/test-all; do
+    scripts/run-with-debugger.sh "${x}"
+  done
+}
+
+function run_tests {
+  for x in bin/**/${VARIANT}/**/test-all; do
+    ${x}
+  done
+}
+
+function run_benchmark {
+    for x in bin/**/${VARIANT}/**/bench; do
+        ${x} --inserts=10000
+    done
+}
+
+function run_tests_with_valgrind {
+  for x in bin/**/${VARIANT}/**/test-all; do
+    # TODO --max-stackframe=8388608
+    # see: https://travis-ci.org/vinniefalco/Beast/jobs/132486245
+    valgrind --error-exitcode=1 "${x}"
+  done
+}
+
+function build_bjam {
+  ${BOOST_ROOT}/bjam toolset=${CC} \
+    variant=${VARIANT} \
+    address-model=${ADDRESS_MODEL} \
+    -j${num_jobs}
+}
+
+function build_cmake {
+    mkdir -p build
+    pushd build > /dev/null
+    cmake -DVARIANT=${VARIANT} ..
+    make -j${num_jobs}
+    mkdir -p ../bin/${VARIANT}
+    find . -executable -type f -exec cp {} ../bin/${VARIANT}/. \;
+    popd > /dev/null
+}
+
+##################################### BUILD ####################################
+
+if [[ ${BUILD_SYSTEM:-} == cmake ]]; then
+    build_cmake
+else
+    build_bjam
+fi
+
+##################################### TESTS ####################################
+
+if [[ ${VARIANT} == coverage ]]; then
+  find . -name "*.gcda" | xargs rm -f
+  rm *.info -f
+  # Create baseline coverage data file
+  lcov --no-external -c -i -d . -o baseline.info > /dev/null
+
+  # Perform test
+  if [[ ${MAIN_BRANCH} == 1 ]]; then
+    run_tests_with_valgrind
+  else
+    run_tests
+  fi
+
+  # Create test coverage data file
+  lcov --no-external -c -d . -o testrun.info > /dev/null
+
+  # Combine baseline and test coverage data
+  lcov -a baseline.info -a testrun.info -o lcov-all.info > /dev/null
+
+  # Extract only include/*, and don\'t report on examples or test
+  lcov -e "lcov-all.info" "${PWD}/include/nudb/*" -o lcov.info > /dev/null
+
+  ~/.local/bin/codecov -X gcov
+  cat lcov.info | node_modules/.bin/coveralls
+
+  # Clean up these stragglers so BOOST_ROOT cache is clean
+  find ${BOOST_ROOT}/bin.v2 -name "*.gcda" | xargs rm -f
+else
+  run_tests_with_debugger
+  run_benchmark
+fi

scripts/install-boost.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Assumptions:
+# 1) BOOST_ROOT and BOOST_URL are already defined,
+# and contain valid values.
+# 2) The last namepart of BOOST_ROOT matches the
+# folder name internal to boost's .tar.gz
+# When testing you can force a boost build by clearing travis caches:
+# https://travis-ci.org/ripple/rippled/caches
+set -eu
+if [ ! -d "$BOOST_ROOT/lib" ]
+then
+  wget $BOOST_URL -O /tmp/boost.tar.gz
+  cd `dirname $BOOST_ROOT`
+  rm -fr ${BOOST_ROOT}
+  tar xzf /tmp/boost.tar.gz
+
+  params="define=_GLIBCXX_USE_CXX11_ABI=0 \
+          address-model=$ADDRESS_MODEL --with-program_options \
+          --with-system --with-coroutine --with-filesystem"
+  cd $BOOST_ROOT && \
+    ./bootstrap.sh --prefix=$BOOST_ROOT && \
+    ./b2 -d1 $params && \
+    ./b2 -d0 $params install
+else
+  echo "Using cached boost at $BOOST_ROOT"
+fi
+

scripts/install-dependencies.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+# The above bash options do the following:
+
+# -e When this option is on, if a simple command fails for any of the reasons
+#    listed in Consequences of Shell Errors or returns an exit status value >0,
+#    and is not part of the compound list following a while, until, or if
+#    keyword, and is not a part of an AND or OR list, and is not a pipeline
+#    preceded by the ! reserved word, then the shell shall immediately exit.
+# -u The shell shall write a message to standard error when it tries to expand a
+#    variable that is not set and immediately exit. An interactive shell shall
+#    not exit.
+# -x The shell shall write to standard error a trace for each command after it
+#    expands the command and before it executes it. It is unspecified
+#    whether the command that turns tracing off is traced.
+# -o pipefail
+#    Pipelines fail on the first command which fails instead of dying later on
+#    down the pipeline.
+
+HERE=${PWD}
+
+# Override gcc version to $GCC_VER.
+# Put an appropriate symlink at the front of the path.
+mkdir -v ${HOME}/bin
+for g in gcc g++ gcov gcc-ar gcc-nm gcc-ranlib
+do
+  test -x $( type -p ${g}-${GCC_VER} )
+  ln -sv $(type -p ${g}-${GCC_VER}) $HOME/bin/${g}
+done
+
+if [[ -n ${CLANG_VER:-} ]]; then
+    # There are cases where the directory exists, but the exe is not available.
+    # Use this workaround for now.
+    if [[ ! -x llvm-${LLVM_VERSION}/bin/llvm-config ]] && [[ -d llvm-${LLVM_VERSION} ]]; then
+        rm -fr llvm-${LLVM_VERSION}
+    fi
+    if [[ ! -d llvm-${LLVM_VERSION} ]]; then
+        mkdir llvm-${LLVM_VERSION}
+        LLVM_URL="http://llvm.org/releases/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-14.04.tar.xz"
+        wget -O - ${LLVM_URL} | tar -Jxvf - --strip 1 -C llvm-${LLVM_VERSION}
+    fi
+    llvm-${LLVM_VERSION}/bin/llvm-config --version;
+    export LLVM_CONFIG="llvm-${LLVM_VERSION}/bin/llvm-config";
+fi
+
+# There are cases where the directory exists, but the exe is not available.
+# Use this workaround for now.
+if [[ ! -x cmake/bin/cmake && -d cmake ]]; then
+    rm -fr cmake
+fi
+if [[ ! -d cmake && ${BUILD_SYSTEM:-} == cmake ]]; then
+  CMAKE_URL="http://www.cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz"
+  mkdir cmake && wget --no-check-certificate -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C cmake
+fi
+
+# NOTE, changed from PWD -> HOME
+export PATH=${HOME}/bin:${PATH}
+
+# What versions are we ACTUALLY running?
+if [ -x $HOME/bin/g++ ]; then
+    ${HOME}/bin/g++ -v
+fi
+if [ -x ${HOME}/bin/clang ]; then
+    ${HOME}/bin/clang -v
+fi
+# Avoid `spurious errors` caused by ~/.npm permission issues
+# Does it already exist? Who owns? What permissions?
+ls -lah ~/.npm || mkdir ~/.npm
+# Make sure we own it
+chown -Rc ${USER} ~/.npm
+# We use this so we can filter the subtrees from our coverage report
+pip install --user https://github.com/codecov/codecov-python/archive/master.zip
+
+bash scripts/install-boost.sh
+bash scripts/install-valgrind.sh
+
+# Install lcov
+# Download the archive
+wget http://downloads.sourceforge.net/ltp/lcov-1.12.tar.gz
+# Extract to ~/lcov-1.12
+tar xfvz lcov-1.12.tar.gz -C ${HOME}
+# Set install path
+mkdir -p ${LCOV_ROOT}
+cd ${HOME}/lcov-1.12 && make install PREFIX=${LCOV_ROOT}
+
+# Install coveralls reporter
+cd ${HERE}
+mkdir -p node_modules
+npm install coveralls

scripts/install-valgrind.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Assumptions:
+# 1) VALGRIND_ROOT is already defined, and contains a valid values
+set -eu
+if [ ! -d "$VALGRIND_ROOT/bin" ]
+then
+  # These are specified in the addons/apt section of .travis.yml
+  # sudo apt-get install subversion automake autotools-dev libc6-dbg
+  export PATH=$PATH:$VALGRIND_ROOT/bin
+  svn co svn://svn.valgrind.org/valgrind/trunk valgrind-co
+  cd valgrind-co
+  ./autogen.sh
+  ./configure --prefix=$VALGRIND_ROOT
+  make
+  make install
+  # test it
+  valgrind ls -l
+else
+  echo "Using cached valgrind at $VALGRIND_ROOT"
+fi

scripts/run-with-debugger.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -eu
+
+if [[ $(uname) == "Darwin" ]]; then
+  # -o runs after loading the binary
+  # -k runs after any crash
+  # We use a ghetto appromixation of --return-child-result, exiting with
+  # 1 on a crash
+  lldb --batch \
+       -o 'run' \
+       -k 'thread backtrace all' \
+       -k 'script import os; os._exit(1)' \
+       $@
+else
+  gdb --silent \
+      --batch \
+      --return-child-result \
+      -ex="set print thread-events off" \
+      -ex=run \
+      -ex="thread apply all bt full" \
+      --args $@
+fi

scripts/run-with-gdb.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -u
+set -e
+gdb --silent \
+    --batch \
+    --return-child-result \
+    -ex="set print thread-events off" \
+    -ex=run \
+    -ex="thread apply all bt full" \
+    --args $@