Back to clang-16

Before `XRPLF/ci` images, we did not have a `dependencies:` job for clang-16, so `instrumentation:` had to build its own dependencies. Now we have clang-16 Conan dependencies built in a separate job that can be used.

.clang-format

@@ -44,6 +44,7 @@ DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
 ForEachMacros:   [ Q_FOREACH,  BOOST_FOREACH ]
+IncludeBlocks:   Regroup
 IncludeCategories:
   - Regex:           '^<(test)/'
     Priority:        0
@@ -53,11 +54,16 @@ IncludeCategories:
     Priority:        2
   - Regex:           '^<(boost)/'
     Priority:        3
-  - Regex:           '.*'
+  - Regex:           '^.*/'
     Priority:        4
+  - Regex:           '^.*\.h'
+    Priority:        5
+  - Regex:           '.*'
+    Priority:        6
 IncludeIsMainRegex: '$'
 IndentCaseLabels: true
 IndentFunctionDeclarationAfterType: false
+IndentRequiresClause: true
 IndentWidth:     4
 IndentWrappedFunctionNames: false
 KeepEmptyLinesAtTheStartOfBlocks: false
@@ -73,6 +79,7 @@ PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
 ReflowComments:  true
+RequiresClausePosition: OwnLine
 SortIncludes:    true
 SpaceAfterCStyleCast: false
 SpaceBeforeAssignmentOperators: true
@@ -87,3 +94,4 @@ SpacesInSquareBrackets: false
 Standard:        Cpp11
 TabWidth:        8
 UseTab:          Never
+QualifierAlignment: Right

.codecov.yml

@@ -7,13 +7,13 @@ comment:
   show_carryforward_flags: false
 
 coverage:
-  range: "60..80"
+  range: "70..85"
   precision: 1
   round: nearest
   status:
     project:
       default:
-        target: 60%
+        target: 75%
         threshold: 2%
     patch:
       default:

.git-blame-ignore-revs

@@ -10,3 +10,4 @@ e2384885f5f630c8f0ffe4bf21a169b433a16858
 b9d007813378ad0ff45660dc07285b823c7e9855
 fe9a5365b8a52d4acc42eb27369247e6f238a4f9
 9a93577314e6a8d4b4a8368cc9d2b15a5d8303e8
+552377c76f55b403a1c876df873a23d780fcc81c

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Allow anyone to review any change by default.
+*
+
+# Require the rpc-reviewers team to review changes to the rpc code.
+include/xrpl/protocol/ @xrplf/rpc-reviewers
+src/libxrpl/protocol/ @xrplf/rpc-reviewers
+src/xrpld/rpc/ @xrplf/rpc-reviewers
+src/xrpld/app/misc/ @xrplf/rpc-reviewers

.github/actions/build/action.yml

@@ -20,6 +20,8 @@ runs:
           ${{ inputs.generator && format('-G "{0}"', inputs.generator) || '' }} \
           -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
           -DCMAKE_BUILD_TYPE=${{ inputs.configuration }} \
+          -Dtests=TRUE \
+          -Dxrpld=TRUE \
           ${{ inputs.cmake-args }} \
           ..
     - name: build

.github/actions/dependencies/action.yml

@@ -6,35 +6,33 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: unlock Conan
-      shell: bash
-      run: conan remove --locks
     - name: export custom recipes
       shell: bash
       run: |
-        conan config set general.revisions_enabled=1
-        conan export external/snappy snappy/1.1.10@
-        conan export external/rocksdb rocksdb/6.29.5@
-        conan export external/soci soci/4.0.3@
+        conan export --version 1.1.10 external/snappy
+        conan export --version 9.7.3 external/rocksdb
+        conan export --version 4.0.3 external/soci
+        conan export --version 6.30.1 external/protobuf
     - name: add Ripple Conan remote
+      if: env.CONAN_URL != ''
       shell: bash
       run: |
-        conan remote list
-        conan remote remove ripple || true
-        # Do not quote the URL. An empty string will be accepted (with
-        # a non-fatal warning), but a missing argument will not.
-        conan remote add ripple ${{ env.CONAN_URL }} --insert 0
+        if conan remote list | grep -q "ripple"; then
+            conan remote remove ripple
+            echo "Removed conan remote ripple"
+        fi
+        conan remote add --index 0 ripple "${CONAN_URL}"
+        echo "Added conan remote ripple at ${CONAN_URL}"
+
     - name: try to authenticate to Ripple Conan remote
+      if: env.CONAN_URL != '' && env.CONAN_LOGIN_USERNAME_RIPPLE != '' && env.CONAN_PASSWORD_RIPPLE != ''
       id: remote
       shell: bash
       run: |
-        # `conan user` implicitly uses the environment variables
-        # CONAN_LOGIN_USERNAME_<REMOTE> and CONAN_PASSWORD_<REMOTE>.
-        # https://docs.conan.io/1/reference/commands/misc/user.html#using-environment-variables
-        # https://docs.conan.io/1/reference/env_vars.html#conan-login-username-conan-login-username-remote-name
-        # https://docs.conan.io/1/reference/env_vars.html#conan-password-conan-password-remote-name
-        echo outcome=$(conan user --remote ripple --password >&2 \
-          && echo success || echo failure) | tee ${GITHUB_OUTPUT}
+        echo "Authenticating to ripple remote..."
+        conan remote auth ripple --force
+        conan remote list-users
+
     - name: list missing binaries
       id: binaries
       shell: bash
@@ -50,9 +48,7 @@ runs:
         conan install \
           --output-folder . \
           --build missing \
-          --settings build_type=${{ inputs.configuration }} \
+          --options:host "&:tests=True" \
+          --options:host "&:xrpld=True" \
+          --settings:all build_type=${{ inputs.configuration }} \
           ..
-    - name: upload dependencies to remote
-      if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success')
-      shell: bash
-      run: conan upload --remote ripple '*' --all --parallel --confirm

.github/workflows/clang-format.yml

@@ -1,59 +1,64 @@
 name: clang-format
 
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
 
 jobs:
   check:
-    runs-on: ubuntu-20.04
-    env:
-      CLANG_VERSION: 10
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
+    runs-on: ubuntu-24.04
+    container: ghcr.io/xrplf/ci/tools-rippled-clang-format
     steps:
-    - uses: actions/checkout@v4
-    - name: Install clang-format
-      run: |
-        codename=$( lsb_release --codename --short )
-        sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null <<EOF
-        deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
-        deb-src http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
-        EOF
-        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install clang-format-${CLANG_VERSION}
-    - name: Format sources
-      run: find include src -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -exec clang-format-${CLANG_VERSION} -i {} +
-    - name: Check for differences
-      id: assert
-      run: |
-        set -o pipefail
-        git diff --exit-code | tee "clang-format.patch"
-    - name: Upload patch
-      if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        name: clang-format.patch
-        if-no-files-found: ignore
-        path: clang-format.patch
-    - name: What happened?
-      if: failure() && steps.assert.outcome == 'failure'
-      env:
-        PREAMBLE: |
-          If you are reading this, you are looking at a failed Github Actions
-          job.  That means you pushed one or more files that did not conform
-          to the formatting specified in .clang-format. That may be because
-          you neglected to run 'git clang-format' or 'clang-format' before
-          committing, or that your version of clang-format has an
-          incompatibility with the one on this
-          machine, which is:
-        SUGGESTION: |
+      # For jobs running in containers, $GITHUB_WORKSPACE and ${{ github.workspace }} might not be the
+      # same directory. The actions/checkout step is *supposed* to checkout into $GITHUB_WORKSPACE and
+      # then add it to safe.directory (see instructions at https://github.com/actions/checkout)
+      # but that's apparently not happening for some container images. We can't be sure what is actually
+      # happening, so let's pre-emptively add both directories to safe.directory. There's a
+      # Github issue opened in 2022 and not resolved in 2025 https://github.com/actions/runner/issues/2058 ¯\_(ツ)_/¯
+      - run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          git config --global --add safe.directory ${{ github.workspace }}
+      - uses: actions/checkout@v4
+      - name: Format first-party sources
+        run: |
+          clang-format --version
+          find include src tests -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' -o -name '*.ipp' \) -exec clang-format -i {} +
+      - name: Check for differences
+        id: assert
+        shell: bash
+        run: |
+          set -o pipefail
+          git diff --exit-code | tee "clang-format.patch"
+      - name: Upload patch
+        if: failure() && steps.assert.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: clang-format.patch
+          if-no-files-found: ignore
+          path: clang-format.patch
+      - name: What happened?
+        if: failure() && steps.assert.outcome == 'failure'
+        env:
+          PREAMBLE: |
+            If you are reading this, you are looking at a failed Github Actions
+            job.  That means you pushed one or more files that did not conform
+            to the formatting specified in .clang-format. That may be because
+            you neglected to run 'git clang-format' or 'clang-format' before
+            committing, or that your version of clang-format has an
+            incompatibility with the one on this
+            machine, which is:
+          SUGGESTION: |
 
-          To fix it, you can do one of two things:
-          1. Download and apply the patch generated as an artifact of this
-             job to your repo, commit, and push.
-          2. Run 'git-clang-format --extensions c,cpp,h,cxx,ipp develop'
-             in your repo, commit, and push.
-      run: |
-        echo "${PREAMBLE}"
-        clang-format-${CLANG_VERSION} --version
-        echo "${SUGGESTION}"
-        exit 1
+            To fix it, you can do one of two things:
+            1. Download and apply the patch generated as an artifact of this
+               job to your repo, commit, and push.
+            2. Run 'git-clang-format --extensions cpp,h,hpp,ipp develop'
+               in your repo, commit, and push.
+        run: |
+          echo "${PREAMBLE}"
+          clang-format --version
+          echo "${SUGGESTION}"
+          exit 1

.github/workflows/doxygen.yml

@@ -4,16 +4,17 @@ on:
   push:
     branches:
       - develop
+      - doxygen
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  job:
+  documentation:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-    container: rippleci/rippled-build-ubuntu:aaf5e3e
+    container: ghcr.io/xrplf/rippled-build-ubuntu:aaf5e3e
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/levelization.yml

@@ -1,49 +1,53 @@
 name: levelization
 
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
 
 jobs:
   check:
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
     runs-on: ubuntu-latest
     env:
       CLANG_VERSION: 10
     steps:
-    - uses: actions/checkout@v4
-    - name: Check levelization
-      run: Builds/levelization/levelization.sh
-    - name: Check for differences
-      id: assert
-      run: |
-        set -o pipefail
-        git diff --exit-code | tee "levelization.patch"
-    - name: Upload patch
-      if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        name: levelization.patch
-        if-no-files-found: ignore
-        path: levelization.patch
-    - name: What happened?
-      if: failure() && steps.assert.outcome == 'failure'
-      env:
-        MESSAGE: |
-          If you are reading this, you are looking at a failed Github
-          Actions job. That means you changed the dependency relationships
-          between the modules in rippled. That may be an improvement or a
-          regression. This check doesn't judge.
+      - uses: actions/checkout@v4
+      - name: Check levelization
+        run: Builds/levelization/levelization.sh
+      - name: Check for differences
+        id: assert
+        run: |
+          set -o pipefail
+          git diff --exit-code | tee "levelization.patch"
+      - name: Upload patch
+        if: failure() && steps.assert.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: levelization.patch
+          if-no-files-found: ignore
+          path: levelization.patch
+      - name: What happened?
+        if: failure() && steps.assert.outcome == 'failure'
+        env:
+          MESSAGE: |
+            If you are reading this, you are looking at a failed Github
+            Actions job. That means you changed the dependency relationships
+            between the modules in rippled. That may be an improvement or a
+            regression. This check doesn't judge.
 
-          A rule of thumb, though, is that if your changes caused
-          something to be removed from loops.txt, that's probably an
-          improvement. If something was added, it's probably a regression.
+            A rule of thumb, though, is that if your changes caused
+            something to be removed from loops.txt, that's probably an
+            improvement. If something was added, it's probably a regression.
 
-          To fix it, you can do one of two things:
-          1. Download and apply the patch generated as an artifact of this
-             job to your repo, commit, and push.
-          2. Run './Builds/levelization/levelization.sh' in your repo,
-             commit, and push.
+            To fix it, you can do one of two things:
+            1. Download and apply the patch generated as an artifact of this
+               job to your repo, commit, and push.
+            2. Run './Builds/levelization/levelization.sh' in your repo,
+               commit, and push.
 
-          See Builds/levelization/README.md for more info.
-      run: |
-        echo "${MESSAGE}"
-        exit 1
+            See Builds/levelization/README.md for more info.
+        run: |
+          echo "${MESSAGE}"
+          exit 1

.github/workflows/libxrpl.yml

@@ -0,0 +1,91 @@
+name: Check libXRPL compatibility with Clio
+env:
+  CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
+  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+on:
+  pull_request:
+    paths:
+      - 'src/libxrpl/protocol/BuildInfo.cpp'
+      - '.github/workflows/libxrpl.yml'
+    types: [opened, reopened, synchronize, ready_for_review]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
+    name: Publish libXRPL
+    outputs:
+      outcome: ${{ steps.upload.outputs.outcome }}
+      version: ${{ steps.version.outputs.version }}
+      channel: ${{ steps.channel.outputs.channel }}
+    runs-on: [self-hosted, heavy]
+    container: ghcr.io/xrplf/rippled-build-ubuntu:aaf5e3e
+    steps:
+      - name: Wait for essential checks to succeed
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          running-workflow-name: wait-for-check-regexp
+          check-regexp: '(dependencies|test).*linux.*' # Ignore windows and mac tests but make sure linux passes
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 10
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Generate channel
+        id: channel
+        shell: bash
+        run: |
+          echo channel="clio/pr_${{ github.event.pull_request.number }}" | tee ${GITHUB_OUTPUT}
+      - name: Export new package
+        shell: bash
+        run: |
+          conan export . ${{ steps.channel.outputs.channel }}
+      - name: Add Ripple Conan remote
+        shell: bash
+        run: |
+          conan remote list
+          conan remote remove ripple || true
+          # Do not quote the URL. An empty string will be accepted (with a non-fatal warning), but a missing argument will not.
+          conan remote add ripple ${{ env.CONAN_URL }} --insert 0
+      - name: Parse new version
+        id: version
+        shell: bash
+        run: |
+          echo version="$(cat src/libxrpl/protocol/BuildInfo.cpp | grep "versionString =" \
+            | awk -F '"' '{print $2}')" | tee ${GITHUB_OUTPUT}
+      - name: Try to authenticate to Ripple Conan remote
+        id: remote
+        shell: bash
+        run: |
+          # `conan user` implicitly uses the environment variables CONAN_LOGIN_USERNAME_<REMOTE> and CONAN_PASSWORD_<REMOTE>.
+          # https://docs.conan.io/1/reference/commands/misc/user.html#using-environment-variables
+          # https://docs.conan.io/1/reference/env_vars.html#conan-login-username-conan-login-username-remote-name
+          # https://docs.conan.io/1/reference/env_vars.html#conan-password-conan-password-remote-name
+          echo outcome=$(conan user --remote ripple --password >&2 \
+            && echo success || echo failure) | tee ${GITHUB_OUTPUT}
+      - name: Upload new package
+        id: upload
+        if: (steps.remote.outputs.outcome == 'success')
+        shell: bash
+        run: |
+          echo "conan upload version ${{ steps.version.outputs.version }} on channel ${{ steps.channel.outputs.channel }}"
+          echo outcome=$(conan upload xrpl/${{ steps.version.outputs.version }}@${{ steps.channel.outputs.channel }} --remote ripple --confirm >&2 \
+            && echo success || echo failure) | tee ${GITHUB_OUTPUT}
+  notify_clio:
+    name: Notify Clio
+    runs-on: ubuntu-latest
+    needs: publish
+    env:
+      GH_TOKEN: ${{ secrets.CLIO_NOTIFY_TOKEN }}
+    steps:
+      - name: Notify Clio about new version
+        if: (needs.publish.outputs.outcome == 'success')
+        shell: bash
+        run: |
+          gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
+          /repos/xrplf/clio/dispatches -f "event_type=check_libxrpl" \
+          -F "client_payload[version]=${{ needs.publish.outputs.version }}@${{ needs.publish.outputs.channel }}" \
+          -F "client_payload[pr]=${{ github.event.pull_request.number }}"

.github/workflows/macos.yml

@@ -1,9 +1,10 @@
 name: macos
 on:
   pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
   push:
     # If the branches list is ever changed, be sure to change it on all
-    # build/test jobs (nix, macos, windows)
+    # build/test jobs (nix, macos, windows, instrumentation)
     branches:
       # Always build the package branches
       - develop
@@ -14,10 +15,23 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
+# This part of Conan configuration is specific to this workflow only; we do not want
+# to pollute conan/profiles directory with settings which might not work for others
+env:
+  #CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
+  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_GLOBAL_CONF: |
+    core.download:parallel={{os.cpu_count()}}
+    core.upload:parallel={{os.cpu_count()}}
+    tools.build:jobs={{ (os.cpu_count() * 4/5) | int }}
+    tools.build:verbosity=verbose
+    tools.compilation:verbosity=verbose
 
 jobs:
 
   test:
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
     strategy:
       matrix:
         platform:
@@ -26,21 +40,38 @@ jobs:
           - Ninja
         configuration:
           - Release
-    runs-on: [self-hosted, macOS]
+    runs-on: [self-hosted, macOS, mac-runner-m1]
     env:
       # The `build` action requires these variables.
       build_dir: .build
       NUM_PROCESSORS: 12
     steps:
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - name: install Conan
         run: |
-          brew install conan@1
-          echo '/opt/homebrew/opt/conan@1/bin' >> $GITHUB_PATH
+          brew install conan
       - name: install Ninja
         if: matrix.generator == 'Ninja'
         run: brew install ninja
+      - name: install python
+        run: |
+          if which python > /dev/null 2>&1; then
+              echo "Python executable exists"
+          else
+              brew install python@3.13
+              ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python
+          fi
+      - name: install cmake
+        run: |
+          if which cmake > /dev/null 2>&1; then
+              echo "cmake executable exists"
+          else
+              brew install cmake
+          fi
+      - name: install nproc
+        run: |
+          brew install coreutils
       - name: check environment
         run: |
           env | sort
@@ -48,17 +79,36 @@ jobs:
           python --version
           conan --version
           cmake --version
+          nproc --version
+          echo -n "nproc returns: "
+          nproc
+          system_profiler SPHardwareDataType
+          sysctl -n hw.logicalcpu
+          clang --version
       - name: configure Conan
         run : |
-          conan profile new default --detect || true
-          conan profile update settings.compiler.cppstd=20 default
-          conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default
+          echo "${CONAN_GLOBAL_CONF}" >> $(conan config home)/global.conf
+          conan config install conan/profiles/ -tf $(conan config home)/profiles/
+          conan profile show
+      - name: export custom recipes
+        shell: bash
+        run: |
+          conan export --version 1.1.10 external/snappy
+          conan export --version 9.7.3 external/rocksdb
+          conan export --version 4.0.3 external/soci
+          conan export --version 6.30.1 external/protobuf
+      - name: add Ripple Conan remote
+        if: env.CONAN_URL != ''
+        shell: bash
+        run: |
+          if conan remote list | grep -q "ripple"; then
+              conan remote remove ripple
+              echo "Removed conan remote ripple"
+          fi
+          conan remote add --index 0 ripple "${CONAN_URL}"
+          echo "Added conan remote ripple at ${CONAN_URL}"
       - name: build dependencies
         uses: ./.github/actions/dependencies
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-          CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-          CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
         with:
           configuration: ${{ matrix.configuration }}
       - name: build
@@ -66,6 +116,12 @@ jobs:
         with:
           generator: ${{ matrix.generator }}
           configuration: ${{ matrix.configuration }}
+          cmake-args: "-Dassert=TRUE -Dwerr=TRUE ${{ matrix.cmake-args }}"
       - name: test
         run: |
-          ${build_dir}/rippled --unittest
+          n=$(nproc)
+          echo "Using $n test jobs"
+
+          cd ${build_dir}
+          ./rippled --unittest --unittest-jobs $n
+          ctest -j $n --output-on-failure

.github/workflows/missing-commits.yml

@@ -0,0 +1,60 @@
+name: missing-commits
+
+on:
+  push:
+    branches:
+      # Only check that the branches are up to date when updating the
+      # relevant branches.
+      - develop
+      - release
+
+jobs:
+  up_to_date:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Check for missing commits
+      id: commits
+      env:
+        SUGGESTION: |
+
+          If you are reading this, then the commits indicated above are
+          missing from "develop" and/or "release". Do a reverse-merge
+          as soon as possible. See CONTRIBUTING.md for instructions.
+      run: |
+        set -o pipefail
+        # Branches ordered by how "canonical" they are. Every commit in
+        # one branch should be in all the branches behind it
+        order=( master release develop )
+        branches=()
+        for branch in "${order[@]}"
+        do
+          # Check that the branches exist so that this job will work on
+          # forked repos, which don't necessarily have master and
+          # release branches.
+          if git ls-remote --exit-code --heads origin \
+            refs/heads/${branch} > /dev/null
+          then
+            branches+=( origin/${branch} )
+          fi
+        done
+
+        prior=()
+        for branch in "${branches[@]}"
+        do
+          if [[ ${#prior[@]} -ne 0 ]]
+          then
+            echo "Checking ${prior[@]} for commits missing from ${branch}"
+            git log --oneline --no-merges "${prior[@]}" \
+              ^$branch | tee -a "missing-commits.txt"
+            echo
+          fi
+          prior+=( "${branch}" )
+        done
+        if [[ $( cat missing-commits.txt | wc -l ) -ne 0 ]]
+        then
+          echo "${SUGGESTION}"
+          exit 1
+        fi

.github/workflows/nix.yml

@@ -1,6 +1,7 @@
 name: nix
 on:
   pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
   push:
     # If the branches list is ever changed, be sure to change it on all
     # build/test jobs (nix, macos, windows)
@@ -10,14 +11,27 @@ on:
       - release
       - master
       # Branches that opt-in to running
-      - 'ci/**'
+      - "ci/**"
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-# This workflow has two job matrixes.
-# They can be considered phases because the second matrix ("test")
-# depends on the first ("dependencies").
+# This part of Conan configuration is specific to this workflow only; we do not want
+# to pollute conan/profiles directory with settings which might not work for others
+env:
+  #CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
+  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_GLOBAL_CONF: |
+    core.download:parallel={{ os.cpu_count() }}
+    core.upload:parallel={{ os.cpu_count() }}
+    tools.build:jobs={{ (os.cpu_count() * 4/5) | int }}
+    tools.build:verbosity=verbose
+    tools.compilation:verbosity=verbose
+
+# This workflow has multiple job matrixes.
+# They can be considered phases because most of the matrices ("test",
+# "coverage", "conan", ) depend on the first ("dependencies").
 #
 # The first phase has a job in the matrix for each combination of
 # variables that affects dependency ABI:
@@ -30,13 +44,16 @@ concurrency:
 # to hold the binaries if they are built locally.
 # We must use the "{upload,download}-artifact" actions instead.
 #
-# The second phase has a job in the matrix for each test configuration.
-# It installs dependency binaries from the cache, whichever was used,
-# and builds and tests rippled.
+# The remaining phases have a job in the matrix for each test
+# configuration. They install dependency binaries from the cache,
+# whichever was used, and build and test rippled.
+#
+# "instrumentation" is independent, but is included here because it also
+# builds on linux in the same "on:" conditions.
 
 jobs:
-
   dependencies:
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
     strategy:
       fail-fast: false
       matrix:
@@ -50,57 +67,48 @@ jobs:
           - Release
         include:
           - compiler: gcc
-            profile:
-              version: 11
-              cc: /usr/bin/gcc
-              cxx: /usr/bin/g++
+            compiler_version: 12
+            distro: ubuntu
+            codename: jammy
           - compiler: clang
-            profile:
-              version: 14
-              cc: /usr/bin/clang-14
-              cxx: /usr/bin/clang++-14
+            compiler_version: 16
+            distro: debian
+            codename: bookworm
     runs-on: [self-hosted, heavy]
-    container: rippleci/rippled-build-ubuntu:aaf5e3e
+    container: ghcr.io/xrplf/ci/${{ matrix.distro }}-${{ matrix.codename }}:${{ matrix.compiler }}-${{ matrix.compiler_version }}
     env:
       build_dir: .build
     steps:
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - name: check environment
         run: |
           echo ${PATH} | tr ':' '\n'
+          lsb_release -a || true
+          ${{ matrix.compiler }}-${{ matrix.compiler_version }} --version
           conan --version
           cmake --version
           env | sort
       - name: configure Conan
         run: |
-          conan profile new default --detect
-          conan profile update settings.compiler.cppstd=20 default
-          conan profile update settings.compiler=${{ matrix.compiler }} default
-          conan profile update settings.compiler.version=${{ matrix.profile.version }} default
-          conan profile update settings.compiler.libcxx=libstdc++11 default
-          conan profile update env.CC=${{ matrix.profile.cc }} default
-          conan profile update env.CXX=${{ matrix.profile.cxx }} default
-          conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default
+          echo "${CONAN_GLOBAL_CONF}" >> $(conan config home)/global.conf
+          conan config install conan/profiles/ -tf $(conan config home)/profiles/
+          conan profile show
       - name: archive profile
         # Create this archive before dependencies are added to the local cache.
-        run: tar -czf conan.tar -C ~/.conan .
+        run: tar -czf conan.tar.gz -C ${CONAN_HOME} .
       - name: build dependencies
         uses: ./.github/actions/dependencies
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-          CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-          CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+
         with:
           configuration: ${{ matrix.configuration }}
       - name: upload archive
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
-          path: conan.tar
+          path: conan.tar.gz
           if-no-files-found: error
 
-
   test:
     strategy:
       fail-fast: false
@@ -113,23 +121,32 @@ jobs:
         configuration:
           - Debug
           - Release
+        include:
+          - compiler: gcc
+            compiler_version: 12
+            distro: ubuntu
+            codename: jammy
+          - compiler: clang
+            compiler_version: 16
+            distro: debian
+            codename: bookworm
         cmake-args:
           -
           - "-Dunity=ON"
     needs: dependencies
     runs-on: [self-hosted, heavy]
-    container: rippleci/rippled-build-ubuntu:aaf5e3e
+    container: ghcr.io/xrplf/ci/${{ matrix.distro }}-${{ matrix.codename }}:${{ matrix.compiler }}-${{ matrix.compiler_version }}
     env:
       build_dir: .build
     steps:
       - name: download cache
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
         with:
           name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
       - name: extract cache
         run: |
-          mkdir -p ~/.conan
-          tar -xzf conan.tar -C ~/.conan
+          mkdir -p ${CONAN_HOME}
+          tar -xzf conan.tar.gz -C ${CONAN_HOME}
       - name: check environment
         run: |
           env | sort
@@ -137,11 +154,9 @@ jobs:
           conan --version
           cmake --version
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - name: dependencies
         uses: ./.github/actions/dependencies
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
         with:
           configuration: ${{ matrix.configuration }}
       - name: build
@@ -149,11 +164,73 @@ jobs:
         with:
           generator: Ninja
           configuration: ${{ matrix.configuration }}
-          cmake-args: ${{ matrix.cmake-args }}
+          cmake-args: "-Dassert=TRUE -Dwerr=TRUE ${{ matrix.cmake-args }}"
+      - name: check linking
+        run: |
+          cd ${build_dir}
+          ldd ./rippled
+          if [ "$(ldd ./rippled | grep -E '(libstdc\+\+|libgcc)' | wc -l)" -eq 0 ]; then
+            echo 'The binary is statically linked.'
+          else
+            echo 'The binary is dynamically linked.'
+            exit 1
+          fi
       - name: test
         run: |
-          ${build_dir}/rippled --unittest --unittest-jobs $(nproc)
+          cd ${build_dir}
+          ./rippled --unittest --unittest-jobs $(nproc)
+          ctest -j $(nproc) --output-on-failure
 
+  reference-fee-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux
+        compiler:
+          - gcc
+        configuration:
+          - Debug
+        cmake-args:
+          - "-DUNIT_TEST_REFERENCE_FEE=200"
+          - "-DUNIT_TEST_REFERENCE_FEE=1000"
+    needs: dependencies
+    runs-on: [self-hosted, heavy]
+    container: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12
+    env:
+      build_dir: .build
+    steps:
+      - name: download cache
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
+      - name: extract cache
+        run: |
+          mkdir -p ${CONAN_HOME}
+          tar -xzf conan.tar.gz -C ${CONAN_HOME}
+      - name: check environment
+        run: |
+          env | sort
+          echo ${PATH} | tr ':' '\n'
+          conan --version
+          cmake --version
+      - name: checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ matrix.configuration }}
+      - name: build
+        uses: ./.github/actions/build
+        with:
+          generator: Ninja
+          configuration: ${{ matrix.configuration }}
+          cmake-args: "-Dassert=TRUE -Dwerr=TRUE ${{ matrix.cmake-args }}"
+      - name: test
+        run: |
+          cd ${build_dir}
+          ./rippled --unittest --unittest-jobs $(nproc)
+          ctest -j $(nproc) --output-on-failure
 
   coverage:
     strategy:
@@ -167,20 +244,18 @@ jobs:
           - Debug
     needs: dependencies
     runs-on: [self-hosted, heavy]
-    container: rippleci/rippled-build-ubuntu:aaf5e3e
+    container: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12
     env:
       build_dir: .build
     steps:
       - name: download cache
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
         with:
           name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
       - name: extract cache
         run: |
-          mkdir -p ~/.conan
-          tar -xzf conan.tar -C ~/.conan
-      - name: install gcovr
-        run: pip install "gcovr>=7,<8"
+          mkdir -p ${CONAN_HOME}
+          tar -xzf conan.tar.gz -C ${CONAN_HOME}
       - name: check environment
         run: |
           echo ${PATH} | tr ':' '\n'
@@ -188,13 +263,11 @@ jobs:
           cmake --version
           gcovr --version
           env | sort
-          ls ~/.conan
+          ls ${CONAN_HOME}
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - name: dependencies
         uses: ./.github/actions/dependencies
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
         with:
           configuration: ${{ matrix.configuration }}
       - name: build
@@ -203,6 +276,8 @@ jobs:
           generator: Ninja
           configuration: ${{ matrix.configuration }}
           cmake-args: >-
+            -Dassert=TRUE
+            -Dwerr=TRUE
             -Dcoverage=ON
             -Dcoverage_format=xml
             -DCODE_COVERAGE_VERBOSE=ON
@@ -214,7 +289,7 @@ jobs:
         run: |
           mv "${build_dir}/coverage.xml" ./
       - name: archive coverage report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: coverage.xml
           path: coverage.xml
@@ -222,7 +297,7 @@ jobs:
       - name: upload coverage report
         uses: wandalen/wretry.action@v1.4.10
         with:
-          action: codecov/codecov-action@v4.3.0
+          action: codecov/codecov-action@v4.5.0
           with: |
             files: coverage.xml
             fail_ci_if_error: true
@@ -232,3 +307,108 @@ jobs:
             token: ${{ secrets.CODECOV_TOKEN }}
           attempt_limit: 5
           attempt_delay: 210000 # in milliseconds
+
+  conan:
+    needs: dependencies
+    runs-on: [self-hosted, heavy]
+    container:
+      image: ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12
+    env:
+      build_dir: .build
+      platform: linux
+      compiler: gcc
+      compiler_version: 12
+      configuration: Release
+    steps:
+      - name: download cache
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: ${{ env.platform }}-${{ env.compiler }}-${{ env.configuration }}
+      - name: extract cache
+        run: |
+          mkdir -p ${CONAN_HOME}
+          tar -xzf conan.tar.gz -C ${CONAN_HOME}
+      - name: check environment
+        run: |
+          env | sort
+          echo ${PATH} | tr ':' '\n'
+          conan --version
+          cmake --version
+      - name: checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ env.configuration }}
+      - name: export
+        run: |
+          conan export . --version head
+      - name: build
+        run: |
+          cd tests/conan
+          mkdir ${build_dir} && cd ${build_dir}
+          conan install .. \
+            --settings:all build_type=${configuration} \
+            --output-folder . \
+            --build missing
+          cmake .. \
+            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=./build/${configuration}/generators/conan_toolchain.cmake \
+            -DCMAKE_BUILD_TYPE=${configuration}
+          cmake --build .
+          ./example | grep '^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+'
+
+  instrumentation-build:
+    needs: dependencies
+    runs-on: [self-hosted, heavy]
+    container: ghcr.io/xrplf/ci/debian-bookworm:clang-16
+    env:
+      build_dir: .build
+    steps:
+        - name: download cache
+          uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+          with:
+            name: linux-clang-Debug
+        - name: extract cache
+          run: |
+            mkdir -p ${CONAN_HOME}
+            tar -xzf conan.tar.gz -C ${CONAN_HOME}
+        - name: check environment
+          run: |
+            echo ${PATH} | tr ':' '\n'
+            conan --version
+            cmake --version
+            env | sort
+            ls ${CONAN_HOME}
+        - name: checkout
+          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        - name: dependencies
+          uses: ./.github/actions/dependencies
+          with:
+            configuration: Debug
+        - name: prepare environment
+          run: |
+            mkdir -p ${build_dir}
+            echo "SOURCE_DIR=$(pwd)" >> $GITHUB_ENV
+            echo "BUILD_DIR=$(pwd)/${build_dir}" >> $GITHUB_ENV
+        - name: build with instrumentation
+          run: |
+            cd ${BUILD_DIR}
+            cmake -S ${SOURCE_DIR} -B ${BUILD_DIR} \
+              -Dvoidstar=ON \
+              -Dtests=ON \
+              -Dxrpld=ON \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DSECP256K1_BUILD_BENCHMARK=OFF \
+              -DSECP256K1_BUILD_TESTS=OFF \
+              -DSECP256K1_BUILD_EXHAUSTIVE_TESTS=OFF \
+              -DCMAKE_TOOLCHAIN_FILE=${BUILD_DIR}/build/generators/conan_toolchain.cmake
+            cmake --build .  --parallel $(nproc)
+        - name: verify instrumentation enabled
+          run: |
+            cd ${BUILD_DIR}
+            ./rippled --version | grep libvoidstar
+        - name: run unit tests
+          run: |
+            cd ${BUILD_DIR}
+            ./rippled -u --unittest-jobs $(( $(nproc)/4 ))
+            ctest -j $(nproc) --output-on-failure

.github/workflows/windows.yml

@@ -2,9 +2,10 @@ name: windows
 
 on:
   pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
   push:
     # If the branches list is ever changed, be sure to change it on all
-    # build/test jobs (nix, macos, windows)
+    # build/test jobs (nix, macos, windows, instrumentation)
     branches:
       # Always build the package branches
       - develop
@@ -17,32 +18,47 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
+# This part of Conan configuration is specific to this workflow only; we do not want
+# to pollute conan/profiles directory with settings which might not work for others
+env:
+  #CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
+  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_GLOBAL_CONF: |
+    core.download:parallel={{os.cpu_count()}}
+    core.upload:parallel={{os.cpu_count()}}
+    tools.build:jobs=24
+    tools.build:verbosity=verbose
+    tools.compilation:verbosity=verbose
 
 jobs:
 
   test:
+    if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true || contains(github.event.pull_request.labels.*.name, 'DraftRunCI') }}
     strategy:
       fail-fast: false
       matrix:
-        generator:
-          - Visual Studio 16 2019
+        version:
+          - generator: Visual Studio 17 2022
+            runs-on: windows-2022
         configuration:
-          - Release
-          # Github hosted runners tend to hang when running Debug unit tests.
-          # Instead of trying to work around it, disable the Debug job until
-          # something beefier (i.e. a heavy self-hosted runner) becomes
-          # available.
-          # - Debug
-    runs-on: windows-2019
+          - type: Release
+            tests: true
+          - type: Debug
+            # Skip running unit tests on debug builds, because they
+            # take an unreasonable amount of time
+            tests: false
+            runtime: d
+    runs-on: ${{ matrix.version.runs-on }}
     env:
       build_dir: .build
     steps:
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - name: choose Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
         with:
-          python-version: 3.9
+          python-version: 3.13
       - name: learn Python cache directory
         id: pip-cache
         shell: bash
@@ -50,12 +66,12 @@ jobs:
           python -m pip install --upgrade pip
           echo "dir=$(pip cache dir)" | tee ${GITHUB_OUTPUT}
       - name: restore Python cache directory
-        uses: actions/cache@v4
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
         with:
             path: ${{ steps.pip-cache.outputs.dir }}
             key: ${{ runner.os }}-${{ hashFiles('.github/workflows/windows.yml') }}
       - name: install Conan
-        run: pip install wheel 'conan<2'
+        run: pip install wheel conan
       - name: check environment
         run: |
           dir env:
@@ -66,26 +82,42 @@ jobs:
       - name: configure Conan
         shell: bash
         run: |
-          conan profile new default --detect
-          conan profile update settings.compiler.cppstd=20 default
-          conan profile update settings.compiler.runtime=MT${{ matrix.configuration == 'Debug' && 'd' || '' }} default
+          echo "${CONAN_GLOBAL_CONF}" >> $(conan config home)/global.conf
+          conan config install conan/profiles/ -tf $(conan config home)/profiles/
+          conan profile show
+      - name: export custom recipes
+        shell: bash
+        run: |
+          conan export --version 1.1.10 external/snappy
+          conan export --version 9.7.3 external/rocksdb
+          conan export --version 4.0.3 external/soci
+          conan export --version 6.30.1 external/protobuf
+      - name: add Ripple Conan remote
+        if: env.CONAN_URL != ''
+        shell: bash
+        run: |
+          if conan remote list | grep -q "ripple"; then
+              conan remote remove ripple
+              echo "Removed conan remote ripple"
+          fi
+          conan remote add --index 0 ripple "${CONAN_URL}"
+          echo "Added conan remote ripple at ${CONAN_URL}"
       - name: build dependencies
         uses: ./.github/actions/dependencies
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-          CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-          CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
         with:
-          configuration: ${{ matrix.configuration }}
+          configuration: ${{ matrix.configuration.type }}
       - name: build
         uses: ./.github/actions/build
         with:
-          generator: '${{ matrix.generator }}'
-          configuration: ${{ matrix.configuration }}
+          generator: '${{ matrix.version.generator }}'
+          configuration: ${{ matrix.configuration.type }}
           # Hard code for now. Move to the matrix if varied options are needed
-          cmake-args: '-Dassert=ON -Dreporting=OFF -Dunity=ON'
+          cmake-args: '-Dassert=TRUE -Dwerr=TRUE -Dreporting=OFF -Dunity=ON'
           cmake-target: install
       - name: test
         shell: bash
+        if: ${{ matrix.configuration.tests }}
         run: |
-          ${build_dir}/${{ matrix.configuration }}/rippled --unittest --unittest-jobs $(nproc)
+          cd ${build_dir}/${{ matrix.configuration.type }}
+          ./rippled --unittest --unittest-jobs $(nproc)
+          ctest -j $(nproc) --output-on-failure

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # .pre-commit-config.yaml
 repos:
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v10.0.1
+  rev: v18.1.8
   hooks:
   - id: clang-format

API-CHANGELOG.md

@@ -8,47 +8,138 @@ The API version controls the API behavior you see. This includes what properties
 
 For a log of breaking changes, see the **API Version [number]** headings. In general, breaking changes are associated with a particular API Version number. For non-breaking changes, scroll to the **XRP Ledger version [x.y.z]** headings. Non-breaking changes are associated with a particular XRP Ledger (`rippled`) release.
 
+## API Version 2
+
+API version 2 is available in `rippled` version 2.0.0 and later. To use this API, clients specify `"api_version" : 2` in each request.
+
+#### Removed methods
+
+In API version 2, the following deprecated methods are no longer available: (https://github.com/XRPLF/rippled/pull/4759)
+
+- `tx_history` - Instead, use other methods such as `account_tx` or `ledger` with the `transactions` field set to `true`.
+- `ledger_header` - Instead, use the `ledger` method.
+
+#### Modifications to JSON transaction element in V2
+
+In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions. (https://github.com/XRPLF/rippled/pull/4775)
+This helps to unify the JSON serialization format of transactions. (https://github.com/XRPLF/clio/issues/722, https://github.com/XRPLF/rippled/issues/4727)
+
+- JSON transaction element is named `tx_json`
+- Binary transaction element is named `tx_blob`
+- JSON transaction metadata element is named `meta`
+- Binary transaction metadata element is named `meta_blob`
+
+Additionally, these elements are now consistently available next to `tx_json` (i.e. sibling elements), where possible:
+
+- `hash` - Transaction ID. This data was stored inside transaction output in API version 1, but in API version 2 is a sibling element.
+- `ledger_index` - Ledger index (only set on validated ledgers)
+- `ledger_hash` - Ledger hash (only set on closed or validated ledgers)
+- `close_time_iso` - Ledger close time expressed in ISO 8601 time format (only set on validated ledgers)
+- `validated` - Bool element set to `true` if the transaction is in a validated ledger, otherwise `false`
+
+This change affects the following methods:
+
+- `tx` - Transaction data moved into element `tx_json` (was inline inside `result`) or, if binary output was requested, moved from `tx` to `tx_blob`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `account_tx` - Renamed transaction element from `tx` to `tx_json`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `transaction_entry` - Renamed transaction metadata element from `metadata` to `meta`. Changed location of `hash` and added new elements
+- `subscribe` - Renamed transaction element from `transaction` to `tx_json`. Changed location of `hash` and added new elements
+- `sign`, `sign_for`, `submit` and `submit_multisigned` - Changed location of `hash` element.
+
+#### Modification to `Payment` transaction JSON schema
+
+When reading Payments, the `Amount` field should generally **not** be used. Instead, use [delivered_amount](https://xrpl.org/partial-payments.html#the-delivered_amount-field) to see the amount that the Payment delivered. To clarify its meaning, the `Amount` field is being renamed to `DeliverMax`. (https://github.com/XRPLF/rippled/pull/4733)
+
+- In `Payment` transaction type, JSON RPC field `Amount` is renamed to `DeliverMax`. To enable smooth client transition, `Amount` is still handled, as described below: (https://github.com/XRPLF/rippled/pull/4733)
+  - On JSON RPC input (e.g. `submit_multisigned` etc. methods), `Amount` is recognized as an alias to `DeliverMax` for both API version 1 and version 2 clients.
+  - On JSON RPC input, submitting both `Amount` and `DeliverMax` fields is allowed _only_ if they are identical; otherwise such input is rejected with `rpcINVALID_PARAMS` error.
+  - On JSON RPC output (e.g. `subscribe`, `account_tx` etc. methods), `DeliverMax` is present in both API version 1 and version 2.
+  - On JSON RPC output, `Amount` is only present in API version 1 and _not_ in version 2.
+
+#### Modifications to account_info response
+
+- `signer_lists` is returned in the root of the response. (In API version 1, it was nested under `account_data`.) (https://github.com/XRPLF/rippled/pull/3770)
+- When using an invalid `signer_lists` value, the API now returns an "invalidParams" error. (https://github.com/XRPLF/rippled/pull/4585)
+  - (`signer_lists` must be a boolean. In API version 1, strings were accepted and may return a normal response - i.e. as if `signer_lists` were `true`.)
+
+#### Modifications to [account_tx](https://xrpl.org/account_tx.html#account_tx) response
+
+- Using `ledger_index_min`, `ledger_index_max`, and `ledger_index` returns `invalidParams` because if you use `ledger_index_min` or `ledger_index_max`, then it does not make sense to also specify `ledger_index`. In API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4571)
+  - The same applies for `ledger_index_min`, `ledger_index_max`, and `ledger_hash`. (https://github.com/XRPLF/rippled/issues/4545#issuecomment-1565065579)
+- Using a `ledger_index_min` or `ledger_index_max` beyond the range of ledgers that the server has:
+  - returns `lgrIdxMalformed` in API version 2. Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/issues/4288)
+- Attempting to use a non-boolean value (such as a string) for the `binary` or `forward` parameters returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
+
+#### Modifications to [noripple_check](https://xrpl.org/noripple_check.html#noripple_check) response
+
+- Attempting to use a non-boolean value (such as a string) for the `transactions` parameter returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
+
 ## API Version 1
 
-This version is supported by all `rippled` versions. At time of writing, it is the default API version, used when no `api_version` is specified. When a new API version is introduced, the command line interface will default to the latest API version. The command line is intended for ad-hoc usage by humans, not programs or automated scripts. The command line is not meant for use in production code.
+This version is supported by all `rippled` versions. For WebSocket and HTTP JSON-RPC requests, it is currently the default API version used when no `api_version` is specified.
 
-### Idiosyncrasies
+The [commandline](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/request-formatting/#commandline-format) always uses the latest API version. The command line is intended for ad-hoc usage by humans, not programs or automated scripts. The command line is not meant for use in production code.
 
-#### V1 account_info response
+### Inconsistency: server_info - network_id
 
-In [the response to the `account_info` command](https://xrpl.org/account_info.html#response-format), there is `account_data` - which is supposed to be an `AccountRoot` object - and `signer_lists` is returned in this object. However, the docs say that `signer_lists` should be at the root level of the reponse.
+The `network_id` field was added in the `server_info` response in version 1.5.0 (2019), but it is not returned in [reporting mode](https://xrpl.org/rippled-server-modes.html#reporting-mode). However, use of reporting mode is now discouraged, in favor of using [Clio](https://github.com/XRPLF/clio) instead.
 
-It makes sense for `signer_lists` to be at the root level because signer lists are not part of the AccountRoot object. (First reported in [xrpl-dev-portal#938](https://github.com/XRPLF/xrpl-dev-portal/issues/938).)
+## XRP Ledger server version 2.5.0
 
-In `api_version: 2`, the `signer_lists` field [will be moved](#modifications-to-account_info-response-in-v2) to the root level of the account_info response. (https://github.com/XRPLF/rippled/pull/3770)
+As of 2025-04-04, version 2.5.0 is in development. You can use a pre-release version by building from source or [using the `nightly` package](https://xrpl.org/docs/infrastructure/installation/install-rippled-on-ubuntu).
 
-#### server_info - network_id
+### Additions and bugfixes in 2.5.0
 
-The `network_id` field was added in the `server_info` response in version 1.5.0 (2019), but it is not returned in [reporting mode](https://xrpl.org/rippled-server-modes.html#reporting-mode).
+- `channel_authorize`: If `signing_support` is not enabled in the config, the RPC is disabled.
+
+## XRP Ledger server version 2.4.0
+
+[Version 2.4.0](https://github.com/XRPLF/rippled/releases/tag/2.4.0) was released on March 4, 2025.
+
+### Additions and bugfixes in 2.4.0
+
+- `ledger_entry`: `state` is added an alias for `ripple_state`.
+- `ledger_entry`: Enables case-insensitive filtering by canonical name in addition to case-sensitive filtering by RPC name.
+- `validators`: Added new field `validator_list_threshold` in response.
+- `simulate`: A new RPC that executes a [dry run of a transaction submission](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0069d-simulate#2-rpc-simulate)
+- Signing methods autofill fees better and properly handle transactions that don't have a base fee, and will also autofill the `NetworkID` field.
+
+## XRP Ledger server version 2.3.0
+
+[Version 2.3.0](https://github.com/XRPLF/rippled/releases/tag/2.3.0) was released on Nov 25, 2024.
+
+### Breaking changes in 2.3.0
+
+- `book_changes`: If the requested ledger version is not available on this node, a `ledgerNotFound` error is returned and the node does not attempt to acquire the ledger from the p2p network (as with other non-admin RPCs).
+
+Admins can still attempt to retrieve old ledgers with the `ledger_request` RPC.
+
+### Additions and bugfixes in 2.3.0
+
+- `book_changes`: Returns a `validated` field in its response, which was missing in prior versions.
+
+## XRP Ledger server version 2.2.0
+
+[Version 2.2.0](https://github.com/XRPLF/rippled/releases/tag/2.2.0) was released on Jun 5, 2024. The following additions are non-breaking (because they are purely additive):
+
+- The `feature` method now has a non-admin mode for users. (It was previously only available to admin connections.) The method returns an updated list of amendments, including their names and other information. ([#4781](https://github.com/XRPLF/rippled/pull/4781))
 
 ## XRP Ledger server version 2.0.0
 
-### Additions in 2.2
-
-Additions are intended to be non-breaking (because they are purely additive).
-
-- `feature`: A non-admin mode that uses the same formatting as admin RPC, but hides potentially-sensitive data.
-
-### Additions in 2.0
-
-Additions are intended to be non-breaking (because they are purely additive).
+[Version 2.0.0](https://github.com/XRPLF/rippled/releases/tag/2.0.0) was released on Jan 9, 2024. The following additions are non-breaking (because they are purely additive):
 
 - `server_definitions`: A new RPC that generates a `definitions.json`-like output that can be used in XRPL libraries.
 - In `Payment` transactions, `DeliverMax` has been added. This is a replacement for the `Amount` field, which should not be used. Typically, the `delivered_amount` (in transaction metadata) should be used. To ease the transition, `DeliverMax` is present regardless of API version, since adding a field is non-breaking.
 - API version 2 has been moved from beta to supported, meaning that it is generally available (regardless of the `beta_rpc_api` setting).
 
+## XRP Ledger server version 2.2.0
+
+The following is a non-breaking addition to the API.
+
+- The `feature` method now has a non-admin mode for users. (It was previously only available to admin connections.) The method returns an updated list of amendments, including their names and other information. ([#4781](https://github.com/XRPLF/rippled/pull/4781))
+
 ## XRP Ledger server version 1.12.0
 
-[Version 1.12.0](https://github.com/XRPLF/rippled/releases/tag/1.12.0) was released on Sep 6, 2023.
-
-### Additions in 1.12
-
-Additions are intended to be non-breaking (because they are purely additive).
+[Version 1.12.0](https://github.com/XRPLF/rippled/releases/tag/1.12.0) was released on Sep 6, 2023. The following additions are non-breaking (because they are purely additive).
 
 - `server_info`: Added `ports`, an array which advertises the RPC and WebSocket ports. This information is also included in the `/crawl` endpoint (which calls `server_info` internally). `grpc` and `peer` ports are also included. (https://github.com/XRPLF/rippled/pull/4427)
   - `ports` contains objects, each containing a `port` for the listening port (a number string), and a `protocol` array listing the supported protocols on that port.
@@ -125,71 +216,6 @@ was released on Mar 14, 2023.
   removed from the [ledger subscription stream](https://xrpl.org/subscribe.html#ledger-stream), because it will no longer
   have any meaning.
 
-## API Version 2
-
-API version 2 is introduced in `rippled` version 2.0. Users can request it explicitly by specifying `"api_version" : 2`.
-
-#### Removed methods
-
-In API version 2, the following deprecated methods are no longer available: (https://github.com/XRPLF/rippled/pull/4759)
-
-- `tx_history` - Instead, use other methods such as `account_tx` or `ledger` with the `transactions` field set to `true`.
-- `ledger_header` - Instead, use the `ledger` method.
-
-#### Modifications to JSON transaction element in V2
-
-In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions. (https://github.com/XRPLF/rippled/pull/4775)
-This helps to unify the JSON serialization format of transactions. (https://github.com/XRPLF/clio/issues/722, https://github.com/XRPLF/rippled/issues/4727)
-
-- JSON transaction element is named `tx_json`
-- Binary transaction element is named `tx_blob`
-- JSON transaction metadata element is named `meta`
-- Binary transaction metadata element is named `meta_blob`
-
-Additionally, these elements are now consistently available next to `tx_json` (i.e. sibling elements), where possible:
-
-- `hash` - Transaction ID. This data was stored inside transaction output in API version 1, but in API version 2 is a sibling element.
-- `ledger_index` - Ledger index (only set on validated ledgers)
-- `ledger_hash` - Ledger hash (only set on closed or validated ledgers)
-- `close_time_iso` - Ledger close time expressed in ISO 8601 time format (only set on validated ledgers)
-- `validated` - Bool element set to `true` if the transaction is in a validated ledger, otherwise `false`
-
-This change affects the following methods:
-
-- `tx` - Transaction data moved into element `tx_json` (was inline inside `result`) or, if binary output was requested, moved from `tx` to `tx_blob`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
-- `account_tx` - Renamed transaction element from `tx` to `tx_json`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
-- `transaction_entry` - Renamed transaction metadata element from `metadata` to `meta`. Changed location of `hash` and added new elements
-- `subscribe` - Renamed transaction element from `transaction` to `tx_json`. Changed location of `hash` and added new elements
-- `sign`, `sign_for`, `submit` and `submit_multisigned` - Changed location of `hash` element.
-
-#### Modification to `Payment` transaction JSON schema
-
-When reading Payments, the `Amount` field should generally **not** be used. Instead, use [delivered_amount](https://xrpl.org/partial-payments.html#the-delivered_amount-field) to see the amount that the Payment delivered. To clarify its meaning, the `Amount` field is being renamed to `DeliverMax`. (https://github.com/XRPLF/rippled/pull/4733)
-
-- In `Payment` transaction type, JSON RPC field `Amount` is renamed to `DeliverMax`. To enable smooth client transition, `Amount` is still handled, as described below: (https://github.com/XRPLF/rippled/pull/4733)
-  - On JSON RPC input (e.g. `submit_multisigned` etc. methods), `Amount` is recognized as an alias to `DeliverMax` for both API version 1 and version 2 clients.
-  - On JSON RPC input, submitting both `Amount` and `DeliverMax` fields is allowed _only_ if they are identical; otherwise such input is rejected with `rpcINVALID_PARAMS` error.
-  - On JSON RPC output (e.g. `subscribe`, `account_tx` etc. methods), `DeliverMax` is present in both API version 1 and version 2.
-  - On JSON RPC output, `Amount` is only present in API version 1 and _not_ in version 2.
-
-#### Modifications to account_info response
-
-- `signer_lists` is returned in the root of the response. In API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770)
-- When using an invalid `signer_lists` value, the API now returns an "invalidParams" error. (https://github.com/XRPLF/rippled/pull/4585)
-  - (`signer_lists` must be a boolean. In API version 1, strings were accepted and may return a normal response - i.e. as if `signer_lists` were `true`.)
-
-#### Modifications to [account_tx](https://xrpl.org/account_tx.html#account_tx) response
-
-- Using `ledger_index_min`, `ledger_index_max`, and `ledger_index` returns `invalidParams` because if you use `ledger_index_min` or `ledger_index_max`, then it does not make sense to also specify `ledger_index`. In API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4571)
-  - The same applies for `ledger_index_min`, `ledger_index_max`, and `ledger_hash`. (https://github.com/XRPLF/rippled/issues/4545#issuecomment-1565065579)
-- Using a `ledger_index_min` or `ledger_index_max` beyond the range of ledgers that the server has:
-  - returns `lgrIdxMalformed` in API version 2. Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/issues/4288)
-- Attempting to use a non-boolean value (such as a string) for the `binary` or `forward` parameters returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
-
-#### Modifications to [noripple_check](https://xrpl.org/noripple_check.html#noripple_check) response
-
-- Attempting to use a non-boolean value (such as a string) for the `transactions` parameter returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
-
 # Unit tests for API changes
 
 The following information is useful to developers contributing to this project:

BUILD.md

@@ -98,6 +98,12 @@ Update the compiler settings:
    conan profile update settings.compiler.cppstd=20 default
    ```
 
+Configure Conan (1.x only) to use recipe revisions:
+
+   ```
+   conan config set general.revisions_enabled=1
+   ```
+
 **Linux** developers will commonly have a default Conan [profile][] that compiles
 with GCC and links with libstdc++.
 If you are linking with libstdc++ (see profile setting `compiler.libcxx`),
@@ -161,8 +167,6 @@ It does not explicitly link the C++ standard library,
 which allows you to statically link it with GCC, if you want.
 
    ```
-   # Conan 1.x
-   conan export external/snappy snappy/1.1.10@
    # Conan 2.x
    conan export --version 1.1.10 external/snappy
    ```
@@ -171,22 +175,26 @@ Export our [Conan recipe for RocksDB](./external/rocksdb).
 It does not override paths to dependencies when building with Visual Studio.
 
    ```
-   # Conan 1.x
-   conan export external/rocksdb rocksdb/6.29.5@
    # Conan 2.x
-   conan export --version 6.29.5 external/rocksdb
+   conan export --version 9.7.3 external/rocksdb
    ```
 
 Export our [Conan recipe for SOCI](./external/soci).
 It patches their CMake to correctly import its dependencies.
 
    ```
-   # Conan 1.x
-   conan export external/soci soci/4.0.3@
    # Conan 2.x
    conan export --version 4.0.3 external/soci
    ```
 
+Export our [Conan recipe for Protobuf](./external/rocksdb).
+It fixes compilation errors with clang-16 and Visual Studio.
+
+   ```
+   # Conan 2.x
+   conan export --version 6.30.1 external/protobuf
+   ```
+
 ### Build and Test
 
 1. Create a build directory and move into it.
@@ -205,13 +213,15 @@ It patches their CMake to correctly import its dependencies.
    the `install-folder` or `-if` option to every `conan install` command
    in the next step.
 
-2. Generate CMake files for every configuration you want to build. 
+2. Use conan to generate CMake files for every configuration you want to build:
 
     ```
     conan install .. --output-folder . --build missing --settings build_type=Release
     conan install .. --output-folder . --build missing --settings build_type=Debug
     ```
 
+    To build Debug, in the next step, be sure to set `-DCMAKE_BUILD_TYPE=Debug`
+
     For a single-configuration generator, e.g. `Unix Makefiles` or `Ninja`,
     you only need to run this command once.
     For a multi-configuration generator, e.g. `Visual Studio`, you may want to
@@ -241,23 +251,26 @@ It patches their CMake to correctly import its dependencies.
 
     Single-config generators:
 
+    Pass the CMake variable [`CMAKE_BUILD_TYPE`][build_type]
+    and make sure it matches the one of the `build_type` settings
+    you chose in the previous step.
+
+    For example, to build Debug, in the next command, replace "Release" with "Debug"
+
     ```
-    cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
+    cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release -Dxrpld=ON -Dtests=ON ..
     ```
 
-    Pass the CMake variable [`CMAKE_BUILD_TYPE`][build_type]
-    and make sure it matches the `build_type` setting you chose in the previous
-    step.
 
     Multi-config generators:
 
     ```
-    cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
+    cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -Dxrpld=ON -Dtests=ON  ..
     ```
 
     **Note:** You can pass build options for `rippled` in this step.
 
-4. Build `rippled`.
+5. Build `rippled`.
 
    For a single-configuration generator, it will build whatever configuration
    you passed for `CMAKE_BUILD_TYPE`. For a multi-configuration generator,
@@ -266,7 +279,7 @@ It patches their CMake to correctly import its dependencies.
    Single-config generators:
 
    ```
-   cmake --build .
+   cmake --build . -j $(nproc)
    ```
 
    Multi-config generators:
@@ -276,7 +289,7 @@ It patches their CMake to correctly import its dependencies.
    cmake --build . --config Debug
    ```
 
-5. Test rippled.
+6. Test rippled.
 
    Single-config generators:
 
@@ -343,7 +356,7 @@ Example use with some cmake variables set:
 ```
 cd .build
 conan install .. --output-folder . --build missing --settings build_type=Debug
-cmake -DCMAKE_BUILD_TYPE=Debug -Dcoverage=ON -Dcoverage_test_parallelism=2 -Dcoverage_format=html-details -Dcoverage_extra_args="--json coverage.json" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
+cmake -DCMAKE_BUILD_TYPE=Debug -Dcoverage=ON -Dxrpld=ON -Dtests=ON -Dcoverage_test_parallelism=2 -Dcoverage_format=html-details -Dcoverage_extra_args="--json coverage.json" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
 cmake --build . --target coverage
 ```
 
@@ -359,11 +372,11 @@ stored inside the build directory, as either of:
 | Option | Default Value | Description |
 | --- | ---| ---|
 | `assert` | OFF | Enable assertions.
-| `reporting` | OFF | Build the reporting mode feature. |
 | `coverage` | OFF | Prepare the coverage report. |
-| `tests` | ON | Build tests. |
-| `unity` | ON | Configure a unity build. |
 | `san` | N/A | Enable a sanitizer with Clang. Choices are `thread` and `address`. |
+| `tests` | OFF | Build tests. |
+| `unity` | ON | Configure a unity build. |
+| `xrpld` | OFF | Build the xrpld (`rippled`) application, and not just the libxrpl library. |
 
 [Unity builds][5] may be faster for the first build
 (at the cost of much more memory) since they concatenate sources into fewer
@@ -386,6 +399,23 @@ After any updates or changes to dependencies, you may need to do the following:
 4. Re-run [conan install](#build-and-test).
 
 
+### 'protobuf/port_def.inc' file not found
+
+If `cmake --build .` results in an error due to a missing a protobuf file, then you might have generated CMake files for a different `build_type` than the `CMAKE_BUILD_TYPE` you passed to conan.
+
+```
+/rippled/.build/pb-xrpl.libpb/xrpl/proto/ripple.pb.h:10:10: fatal error: 'google/protobuf/port_def.inc' file not found
+   10 | #include <google/protobuf/port_def.inc>
+      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+1 error generated.
+```
+
+For example, if you want to build Debug:
+
+1. For conan install, pass `--settings build_type=Debug`
+2. For cmake, pass `-DCMAKE_BUILD_TYPE=Debug`
+
+
 ### no std::result_of
 
 If your compiler version is recent enough to have removed `std::result_of` as

Builds/levelization/levelization.sh

@@ -13,12 +13,15 @@ then
   git clean -ix
 fi
 
+# Ensure all sorting is ASCII-order consistently across platforms.
+export LANG=C
+
 rm -rfv results
 mkdir results
 includes="$( pwd )/results/rawincludes.txt"
 pushd ../..
 echo Raw includes:
-grep -r '#include.*/.*\.h' include src | \
+grep -r '^[ ]*#include.*/.*\.h' include src | \
     grep -v boost | tee ${includes}
 popd
 pushd results

Builds/levelization/results/loops.txt

@@ -4,9 +4,6 @@ Loop: test.jtx test.toplevel
 Loop: test.jtx test.unit_test
   test.unit_test == test.jtx
 
-Loop: xrpl.basics xrpl.json
-  xrpl.json ~= xrpl.basics
-
 Loop: xrpld.app xrpld.core
   xrpld.app > xrpld.core
 
@@ -16,14 +13,11 @@ Loop: xrpld.app xrpld.ledger
 Loop: xrpld.app xrpld.net
   xrpld.app > xrpld.net
 
-Loop: xrpld.app xrpld.nodestore
-  xrpld.app > xrpld.nodestore
-
 Loop: xrpld.app xrpld.overlay
-  xrpld.overlay ~= xrpld.app
+  xrpld.overlay > xrpld.app
 
 Loop: xrpld.app xrpld.peerfinder
-  xrpld.app > xrpld.peerfinder
+  xrpld.peerfinder ~= xrpld.app
 
 Loop: xrpld.app xrpld.rpc
   xrpld.rpc > xrpld.app
@@ -35,13 +29,10 @@ Loop: xrpld.core xrpld.net
   xrpld.net > xrpld.core
 
 Loop: xrpld.core xrpld.perflog
-  xrpld.perflog ~= xrpld.core
+  xrpld.perflog == xrpld.core
 
 Loop: xrpld.net xrpld.rpc
-  xrpld.rpc > xrpld.net
-
-Loop: xrpld.nodestore xrpld.overlay
-  xrpld.overlay ~= xrpld.nodestore
+  xrpld.rpc ~= xrpld.net
 
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay

Builds/levelization/results/ordering.txt

@@ -1,5 +1,4 @@
 libxrpl.basics > xrpl.basics
-libxrpl.basics > xrpl.protocol
 libxrpl.crypto > xrpl.basics
 libxrpl.json > xrpl.basics
 libxrpl.json > xrpl.json
@@ -7,6 +6,7 @@ libxrpl.protocol > xrpl.basics
 libxrpl.protocol > xrpl.json
 libxrpl.protocol > xrpl.protocol
 libxrpl.resource > xrpl.basics
+libxrpl.resource > xrpl.json
 libxrpl.resource > xrpl.resource
 libxrpl.server > xrpl.basics
 libxrpl.server > xrpl.json
@@ -20,6 +20,7 @@ test.app > xrpl.basics
 test.app > xrpld.app
 test.app > xrpld.core
 test.app > xrpld.ledger
+test.app > xrpld.nodestore
 test.app > xrpld.overlay
 test.app > xrpld.rpc
 test.app > xrpl.json
@@ -42,6 +43,7 @@ test.consensus > xrpl.basics
 test.consensus > xrpld.app
 test.consensus > xrpld.consensus
 test.consensus > xrpld.ledger
+test.consensus > xrpl.json
 test.core > test.jtx
 test.core > test.toplevel
 test.core > test.unit_test
@@ -58,7 +60,6 @@ test.json > test.jtx
 test.json > xrpl.json
 test.jtx > xrpl.basics
 test.jtx > xrpld.app
-test.jtx > xrpld.consensus
 test.jtx > xrpld.core
 test.jtx > xrpld.ledger
 test.jtx > xrpld.net
@@ -74,20 +75,15 @@ test.ledger > xrpld.app
 test.ledger > xrpld.core
 test.ledger > xrpld.ledger
 test.ledger > xrpl.protocol
-test.net > test.jtx
-test.net > test.toplevel
-test.net > test.unit_test
-test.net > xrpld.net
 test.nodestore > test.jtx
 test.nodestore > test.toplevel
 test.nodestore > test.unit_test
 test.nodestore > xrpl.basics
-test.nodestore > xrpld.app
 test.nodestore > xrpld.core
 test.nodestore > xrpld.nodestore
 test.nodestore > xrpld.unity
-test.nodestore > xrpl.protocol
 test.overlay > test.jtx
+test.overlay > test.toplevel
 test.overlay > test.unit_test
 test.overlay > xrpl.basics
 test.overlay > xrpld.app
@@ -109,13 +105,11 @@ test.resource > test.unit_test
 test.resource > xrpl.basics
 test.resource > xrpl.resource
 test.rpc > test.jtx
-test.rpc > test.nodestore
 test.rpc > test.toplevel
 test.rpc > xrpl.basics
 test.rpc > xrpld.app
 test.rpc > xrpld.core
 test.rpc > xrpld.net
-test.rpc > xrpld.nodestore
 test.rpc > xrpld.overlay
 test.rpc > xrpld.rpc
 test.rpc > xrpl.json
@@ -138,6 +132,8 @@ test.shamap > xrpl.protocol
 test.toplevel > test.csf
 test.toplevel > xrpl.json
 test.unit_test > xrpl.basics
+tests.libxrpl > xrpl.basics
+xrpl.json > xrpl.basics
 xrpl.protocol > xrpl.basics
 xrpl.protocol > xrpl.json
 xrpl.resource > xrpl.basics
@@ -150,6 +146,7 @@ xrpld.app > test.unit_test
 xrpld.app > xrpl.basics
 xrpld.app > xrpld.conditions
 xrpld.app > xrpld.consensus
+xrpld.app > xrpld.nodestore
 xrpld.app > xrpld.perflog
 xrpld.app > xrpl.json
 xrpld.app > xrpl.protocol
@@ -163,7 +160,6 @@ xrpld.core > xrpl.basics
 xrpld.core > xrpl.json
 xrpld.core > xrpl.protocol
 xrpld.ledger > xrpl.basics
-xrpld.ledger > xrpld.core
 xrpld.ledger > xrpl.json
 xrpld.ledger > xrpl.protocol
 xrpld.net > xrpl.basics
@@ -178,6 +174,7 @@ xrpld.nodestore > xrpl.protocol
 xrpld.overlay > xrpl.basics
 xrpld.overlay > xrpld.core
 xrpld.overlay > xrpld.peerfinder
+xrpld.overlay > xrpld.perflog
 xrpld.overlay > xrpl.json
 xrpld.overlay > xrpl.protocol
 xrpld.overlay > xrpl.resource
@@ -186,14 +183,11 @@ xrpld.peerfinder > xrpl.basics
 xrpld.peerfinder > xrpld.core
 xrpld.peerfinder > xrpl.protocol
 xrpld.perflog > xrpl.basics
-xrpld.perflog > xrpld.nodestore
 xrpld.perflog > xrpl.json
-xrpld.perflog > xrpl.protocol
 xrpld.rpc > xrpl.basics
 xrpld.rpc > xrpld.core
 xrpld.rpc > xrpld.ledger
 xrpld.rpc > xrpld.nodestore
-xrpld.rpc > xrpld.shamap
 xrpld.rpc > xrpl.json
 xrpld.rpc > xrpl.protocol
 xrpld.rpc > xrpl.resource

CMakeLists.txt

@@ -16,16 +16,36 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
+    # GCC-specific fixes
+    add_compile_options(-Wno-unknown-pragmas -Wno-subobject-linkage)
+    # -Wno-subobject-linkage can be removed when we upgrade GCC version to at least 13.3
+elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    # Clang-specific fixes
+    add_compile_options(-Wno-unknown-warning-option) # Ignore unknown warning options
+elseif(MSVC)
+    # MSVC-specific fixes
+    add_compile_options(/wd4068) # Ignore unknown pragmas
+endif()
+
 # make GIT_COMMIT_HASH define available to all sources
 find_package(Git)
 if(Git_FOUND)
-    execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir=${CMAKE_CURRENT_SOURCE_DIR}/.git describe --always --abbrev=40
+    execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir=${CMAKE_CURRENT_SOURCE_DIR}/.git rev-parse HEAD
         OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE gch)
     if(gch)
         set(GIT_COMMIT_HASH "${gch}")
         message(STATUS gch: ${GIT_COMMIT_HASH})
         add_definitions(-DGIT_COMMIT_HASH="${GIT_COMMIT_HASH}")
     endif()
+
+    execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir=${CMAKE_CURRENT_SOURCE_DIR}/.git rev-parse --abbrev-ref HEAD
+        OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE gb)
+    if(gb)
+        set(GIT_BRANCH "${gb}")
+        message(STATUS gb: ${GIT_BRANCH})
+        add_definitions(-DGIT_BRANCH="${GIT_BRANCH}")
+    endif()
 endif() #git
 
 if(thread_safety_analysis)
@@ -70,9 +90,15 @@ set_target_properties(OpenSSL::SSL PROPERTIES
   INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2
 )
 set(SECP256K1_INSTALL TRUE)
+set(SECP256K1_BUILD_BENCHMARK FALSE)
+set(SECP256K1_BUILD_TESTS FALSE)
+set(SECP256K1_BUILD_EXHAUSTIVE_TESTS FALSE)
+set(SECP256K1_BUILD_CTIME_TESTS FALSE)
+set(SECP256K1_BUILD_EXAMPLES FALSE)
 add_subdirectory(external/secp256k1)
 add_library(secp256k1::secp256k1 ALIAS secp256k1)
 add_subdirectory(external/ed25519-donna)
+add_subdirectory(external/antithesis-sdk)
 find_package(gRPC REQUIRED)
 find_package(lz4 REQUIRED)
 # Target names with :: are not allowed in a generator expression.
@@ -115,15 +141,6 @@ else()
 endif()
 target_link_libraries(ripple_libs INTERFACE ${nudb})
 
-if(reporting)
-  find_package(cassandra-cpp-driver REQUIRED)
-  find_package(PostgreSQL REQUIRED)
-  target_link_libraries(ripple_libs INTERFACE
-    cassandra-cpp-driver::cassandra-cpp-driver
-    PostgreSQL::PostgreSQL
-  )
-endif()
-
 if(coverage)
   include(RippledCov)
 endif()
@@ -132,3 +149,8 @@ set(PROJECT_EXPORT_SET RippleExports)
 include(RippledCore)
 include(RippledInstall)
 include(RippledValidatorKeys)
+
+if(tests)
+  include(CTest)
+  add_subdirectory(src/tests/libxrpl)
+endif()

CONTRIBUTING.md

@@ -5,15 +5,12 @@ XRPL.
 # Contributing
 
 We assume you are familiar with the general practice of [making
-contributions on GitHub][1]. This file includes only special
+contributions on GitHub][contrib]. This file includes only special
 instructions specific to this project.
 
 
 ## Before you start
 
-In general, contributions should be developed in your personal
-[fork](https://github.com/XRPLF/rippled/fork).
-
 The following branches exist in the main project repository:
 
 - `develop`: The latest set of unreleased features, and the most common
@@ -26,9 +23,20 @@ The tip of each branch must be signed. In order for GitHub to sign a
 squashed commit that it builds from your pull request, GitHub must know
 your verifying key. Please set up [signature verification][signing].
 
-[rippled]: https://github.com/XRPLF/rippled
-[signing]:
-    https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification
+In general, external contributions should be developed in your personal
+[fork][forking]. Contributions from developers with write permissions
+should be done in [the main repository][rippled] in a branch with
+a permitted prefix. Permitted prefixes are:
+* XLS-[a-zA-Z0-9]+/.+
+  * e.g. XLS-0033d/mpt-clarify-STEitherAmount
+* [GitHub username]/.+
+  * e.g. JoelKatz/fix-rpc-webhook-queue
+* [Organization name]/.+
+  * e.g. ripple/antithesis
+
+Regardless of where the branch is created, please open a *draft* pull
+request as soon as possible after pushing the branch to Github, to
+increase visibility, and ease feedback during the development process.
 
 
 ## Major contributions
@@ -49,6 +57,7 @@ author delegates that responsibility to others.
 
 
 ## Before making a pull request
+(Or marking a draft pull request as ready.)
 
 Changes that alter transaction processing must be guarded by an
 [Amendment](https://xrpl.org/amendments.html).
@@ -57,43 +66,141 @@ Amendment.
 
 Ensure that your code compiles according to the build instructions in
 [`BUILD.md`](./BUILD.md).
-If you create new source files, they must go under `src/ripple`.
-You will need to add them to one of the
-[source lists](./Builds/CMake/RippledCore.cmake) in CMake.
 
 Please write tests for your code.
-If you create new test source files, they must go under `src/test`.
-You will need to add them to one of the
-[source lists](./Builds/CMake/RippledCore.cmake) in CMake.
 If your test can be run offline, in under 60 seconds, then it can be an
 automatic test run by `rippled --unittest`.
 Otherwise, it must be a manual test.
 
+If you create new source files, they must be organized as follows:
+* If the files are in any of the `libxrpl` modules, the headers (`.h`) must go
+  under `include/xrpl`, and source (`.cpp`) files must go under
+  `src/libxrpl`.
+* All other non-test files must go under `src/xrpld`.
+* All test source files must go under `src/test`.
+
 The source must be formatted according to the style guide below.
 
 Header includes must be [levelized](./Builds/levelization).
 
+Changes should be usually squashed down into a single commit.
+Some larger or more complicated change sets make more sense,
+and are easier to review if organized into multiple logical commits.
+Either way, all commits should fit the following criteria:
+* Changes should be presented in a single commit or a logical
+  sequence of commits.
+  Specifically, chronological commits that simply
+  reflect the history of how the author implemented
+  the change, "warts and all", are not useful to
+  reviewers.
+* Every commit should have a [good message](#good-commit-messages).
+  to explain a specific aspects of the change.
+* Every commit should be signed.
+* Every commit should be well-formed (builds successfully,
+  unit tests passing), as this helps to resolve merge
+  conflicts, and makes it easier to use `git bisect`
+  to find bugs.
+
+### Good commit messages
+
+Refer to
+["How to Write a Git Commit Message"](https://cbea.ms/git-commit/)
+for general rules on writing a good commit message.
+
+tl;dr
+> 1. Separate subject from body with a blank line.
+> 2. Limit the subject line to 50 characters.
+>    * [...]shoot for 50 characters, but consider 72 the hard limit.
+> 3. Capitalize the subject line.
+> 4. Do not end the subject line with a period.
+> 5. Use the imperative mood in the subject line.
+>    * A properly formed Git commit subject line should always be able
+>      to complete the following sentence: "If applied, this commit will
+>      _your subject line here_".
+> 6. Wrap the body at 72 characters.
+> 7. Use the body to explain what and why vs. how.
+
+In addition to those guidelines, please add one of the following
+prefixes to the subject line if appropriate.
+* `fix:` - The primary purpose is to fix an existing bug.
+* `perf:` - The primary purpose is performance improvements.
+* `refactor:` - The changes refactor code without affecting
+  functionality.
+* `test:` - The changes _only_ affect unit tests.
+* `docs:` - The changes _only_ affect documentation. This can
+  include code comments in addition to `.md` files like this one.
+* `build:` - The changes _only_ affect the build process,
+  including CMake and/or Conan settings.
+* `chore:` - Other tasks that don't affect the binary, but don't fit
+  any of the other cases. e.g. formatting, git settings, updating
+  Github Actions jobs.
+
+Whenever possible, when updating commits after the PR is open, please
+add the PR number to the end of the subject line. e.g. `test: Add
+unit tests for Feature X (#1234)`.
 
 ## Pull requests
 
 In general, pull requests use `develop` as the base branch.
+The exceptions are
+* Fixes and improvements to a release candidate use `release` as the
+  base.
+* Hotfixes use `master` as the base.
 
-(Hotfixes are an exception.)
+If your changes are not quite ready, but you want to make it easily available
+for preliminary examination or review, you can create a "Draft" pull request.
+While a pull request is marked as a "Draft", you can rebase or reorganize the
+commits in the pull request as desired.
 
-Changes to pull requests must be added as new commits.
-Once code reviewers have started looking at your code, please avoid
-force-pushing a branch in a pull request.
+Github pull requests are created as "Ready" by default, or you can mark
+a "Draft" pull request as "Ready".
+Once a pull request is marked as "Ready",
+any changes must be added as new commits. Do not
+force-push to a branch in a pull request under review.
+(This includes rebasing your branch onto the updated base branch.
+Use a merge operation, instead or hit the "Update branch" button
+at the bottom of the Github PR page.)
 This preserves the ability for reviewers to filter changes since their last
 review.
 
-A pull request must obtain **approvals from at least two reviewers** before it
-can be considered for merge by a Maintainer.
+A pull request must obtain **approvals from at least two reviewers**
+before it can be considered for merge by a Maintainer.
 Maintainers retain discretion to require more approvals if they feel the
 credibility of the existing approvals is insufficient.
 
-Pull requests must be merged by [squash-and-merge][2]
+Pull requests must be merged by [squash-and-merge][squash]
 to preserve a linear history for the `develop` branch.
 
+### "Ready to merge"
+
+A pull request should only have the "Ready to merge" label added when it
+meets a few criteria:
+
+1. It must have two approving reviews [as described
+   above](#pull-requests). (Exception: PRs that are deemed "trivial"
+   only need one approval.)
+2. All CI checks must be complete and passed. (One-off failures may
+   be acceptable if they are related to a known issue.)
+3. The PR must have a [good commit message](#good-commit-messages).
+   * If the PR started with a good commit message, and it doesn't
+     need to be updated, the author can indicate that in a comment.
+   * Any contributor, preferably the author, can leave a comment
+     suggesting a commit message.
+   * If the author squashes and rebases the code in preparation for
+     merge, they should also ensure the commit message(s) are updated
+     as well.
+4. The PR branch must be up to date with the base branch (usually
+   `develop`). This is usually accomplished by merging the base branch
+   into the feature branch, but if the other criteria are met, the
+   changes can be squashed and rebased on top of the base branch.
+5. Finally, and most importantly, the author of the PR must
+   positively indicate that the PR is ready to merge. That can be
+   accomplished by adding the "Ready to merge" label if their role
+   allows, or by leaving a comment to the effect that the PR is ready to
+   merge.
+
+Once the "Ready to merge" label is added, a maintainer may merge the PR
+at any time, so don't use it lightly.
 
 # Style guide
 
@@ -104,7 +211,7 @@ coherent rather than a set of _thou shalt not_ commandments.
 
 ## Formatting
 
-All code must conform to `clang-format` version 10,
+All code must conform to `clang-format` version 18,
 according to the settings in [`.clang-format`](./.clang-format),
 unless the result would be unreasonably difficult to read or maintain.
 To demarcate lines that should be left as-is, surround them with comments like
@@ -135,6 +242,68 @@ pip3 install pre-commit
 pre-commit install
 ```
 
+## Contracts and instrumentation
+
+We are using [Antithesis](https://antithesis.com/) for continuous fuzzing,
+and keep a copy of [Antithesis C++ SDK](https://github.com/antithesishq/antithesis-sdk-cpp/)
+in `external/antithesis-sdk`. One of the aims of fuzzing is to identify bugs
+by finding external conditions which cause contracts violations inside `rippled`.
+The contracts are expressed as `XRPL_ASSERT` or `UNREACHABLE` (defined in
+`include/xrpl/beast/utility/instrumentation.h`), which are effectively (outside
+of Antithesis) wrappers for `assert(...)` with added name. The purpose of name
+is to provide contracts with stable identity which does not rely on line numbers.
+
+When `rippled` is built with the Antithesis instrumentation enabled
+(using `voidstar` CMake option) and ran on the Antithesis platform, the
+contracts become
+[test properties](https://antithesis.com/docs/using_antithesis/properties.html);
+otherwise they are just like a regular `assert`.
+To learn more about Antithesis, see
+[How Antithesis Works](https://antithesis.com/docs/introduction/how_antithesis_works.html)
+and [C++ SDK](https://antithesis.com/docs/using_antithesis/sdk/cpp/overview.html#)
+
+We continue to use the old style `assert` or `assert(false)` in certain
+locations, where the reporting of contract violations on the Antithesis
+platform is either not possible or not useful.
+
+For this reason:
+* The locations where `assert` or `assert(false)` contracts should continue to be used:
+  * `constexpr` functions
+  * unit tests i.e. files under `src/test`
+  * unit tests-related modules (files under `beast/test` and `beast/unit_test`)
+* Outside of the listed locations, do not use `assert`; use `XRPL_ASSERT` instead,
+  giving it unique name, with the short description of the contract.
+* Outside of the listed locations, do not use `assert(false)`; use
+  `UNREACHABLE` instead, giving it unique name, with the description of the
+  condition being violated
+* The contract name should start with a full name (including scope) of the
+  function, optionally a named lambda, followed by a colon ` : ` and a brief
+  (typically at most five words) description. `UNREACHABLE` contracts
+  can use slightly longer descriptions. If there are multiple overloads of the
+  function, use common sense to balance both brevity and unambiguity of the
+  function name. NOTE: the purpose of name is to provide stable means of
+  unique identification of every contract; for this reason try to avoid elements
+  which can change in some obvious refactors or when reinforcing the condition.
+* Contract description typically (except for `UNREACHABLE`) should describe the
+  _expected_ condition, as in "I assert that _expected_ is true".
+* Contract description for `UNREACHABLE` should describe the _unexpected_
+  situation which caused the line to have been reached.
+* Example good name for an
+  `UNREACHABLE` macro `"Json::operator==(Value, Value) : invalid type"`; example
+  good name for an `XRPL_ASSERT` macro `"Json::Value::asCString : valid type"`.
+* Example **bad** name
+  `"RFC1751::insert(char* s, int x, int start, int length) : length is greater than or equal zero"`
+  (missing namespace, unnecessary full function signature, description too verbose).
+  Good name: `"ripple::RFC1751::insert : minimum length"`.
+* In **few** well-justified cases a non-standard name can be used, in which case a
+  comment should be placed to explain the rationale (example in `contract.cpp`)
+* Do **not** rename a contract without a good reason (e.g. the name no longer
+  reflects the location or the condition being checked)
+* Do not use `std::unreachable`
+* Do not put contracts where they can be violated by an external condition
+  (e.g. timing, data payload before mandatory validation etc.) as this creates
+  bogus bug reports (and causes crashes of Debug builds)
+
 ## Unit Tests
 To execute all unit tests:
 
@@ -207,17 +376,22 @@ existing maintainer without a vote.
 
 ## Current Maintainers
 
-Maintainers are users with admin access to the repo. Maintainers do not typically approve or deny pull requests.
+Maintainers are users with maintain or admin access to the repo.
 
+* [bthomee](https://github.com/bthomee) (Ripple)
 * [intelliot](https://github.com/intelliot) (Ripple)
 * [JoelKatz](https://github.com/JoelKatz) (Ripple)
 * [nixer89](https://github.com/nixer89) (XRP Ledger Foundation)
+* [RichardAH](https://github.com/RichardAH) (XRP Ledger Foundation)
 * [Silkjaer](https://github.com/Silkjaer) (XRP Ledger Foundation)
 * [WietseWind](https://github.com/WietseWind) (XRPL Labs + XRP Ledger Foundation)
+* [ximinez](https://github.com/ximinez) (Ripple)
+
 
 ## Current Code Reviewers
 
-Code Reviewers are developers who have the ability to review and approve source code changes.
+Code Reviewers are developers who have the ability to review, approve, and
+in some cases merge source code changes.
 
 * [HowardHinnant](https://github.com/HowardHinnant) (Ripple)
 * [scottschurr](https://github.com/scottschurr) (Ripple)
@@ -241,6 +415,607 @@ Code Reviewers are developers who have the ability to review and approve source
 * [RichardAH](https://github.com/RichardAH) (XRPL Labs + XRP Ledger Foundation)
 * [dangell7](https://github.com/dangell7) (XRPL Labs)
 
+Developers not on this list are able and encouraged to submit feedback
+on pending code changes (open pull requests).
 
-[1]: https://docs.github.com/en/get-started/quickstart/contributing-to-projects
-[2]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits
+## Instructions for maintainers
+
+These instructions assume you have your git upstream remotes configured
+to avoid accidental pushes to the main repo, and a remote group
+specifying both of them. e.g.
+```
+$ git remote -v | grep upstream
+upstream        https://github.com/XRPLF/rippled.git (fetch)
+upstream        https://github.com/XRPLF/rippled.git (push)
+upstream-push   git@github.com:XRPLF/rippled.git (fetch)
+upstream-push   git@github.com:XRPLF/rippled.git (push)
+
+$ git config remotes.upstreams
+upstream upstream-push
+```
+
+You can use the [setup-upstreams] script to set this up.
+
+It also assumes you have a default gpg signing key set up in git. e.g.
+```
+$ git config user.signingkey
+968479A1AFF927E37D1A566BB5690EEEBB952194
+# (This is github's key. Use your own.)
+```
+
+### When and how to merge pull requests
+
+The maintainer should double-check that the PR has met all the
+necessary criteria, and can request additional information from the
+owner, or additional reviews, and can always feel free to remove the
+"Ready to merge" label if appropriate. The maintainer has final say on
+whether a PR gets merged, and are encouraged to communicate and issues
+or concerns to other maintainers.
+
+#### Most pull requests: "Squash and merge"
+
+Most pull requests don't need special handling, and can simply be
+merged using the "Squash and merge" button on the Github UI. Update
+the suggested commit message, or modify it as needed.
+
+#### Slightly more complicated pull requests
+
+Some pull requests need to be pushed to `develop` as more than one
+commit. A PR author may *request* to merge as separate commits. They
+must *justify* why separate commits are needed, and *specify* how they
+would like the commits to be merged. If you disagree with the author,
+discuss it with them directly.
+
+If the process is reasonable, follow it. The simplest option is to do a
+fast forward only merge (`--ff-only`) on the command line and push to
+`develop`.
+
+Some examples of when separate commits are worthwhile are:
+1. PRs where source files are reorganized in multiple steps.
+2. PRs where the commits are mostly independent and *could* be separate
+   PRs, but are pulled together into one PR under a commit theme or
+   issue.
+3. PRs that are complicated enough that `git bisect` would not be much
+   help if it determined this PR introduced a problem.
+
+Either way, check that:
+* The commits are based on the current tip of `develop`.
+* The commits are clean: No merge commits (except when reverse
+  merging), no "[FOLD]" or "fixup!" messages.
+* All commits are signed. If the commits are not signed by the author, use
+  `git commit --amend -S` to sign them yourself.
+* At least one (but preferably all) of the commits has the PR number
+  in the commit message.
+
+The "Create a merge commit" and "Rebase and merge" options should be
+disabled in the Github UI, but if you ever find them available **Do not
+use them!**
+
+### Releases
+
+All releases, including release candidates and betas, are handled
+differently from typical PRs. Most importantly, never use
+the Github UI to merge a release.
+
+Rippled uses a linear workflow model that can be summarized as:
+
+1. In between releases, developers work against the `develop` branch.
+2. Periodically, a maintainer will build and tag a beta version from
+   `develop`, which is pushed to `release`.
+   * Betas are usually released every two to three weeks, though that
+     schedule can vary depending on progress, availability, and other
+     factors.
+3. When the changes in `develop` are considered stable and mature enough
+   to be ready to release, a release candidate (RC) is built and tagged
+   from `develop`, and merged to `release`.
+   * Further development for that release (primarily fixes) then
+     continues against `release`, while other development continues on
+     `develop`. Effectively, `release` is forked from `develop`. Changes
+     to `release` must be reverse merged to `develop`.
+4. When the candidate has passed testing and is ready for release, the
+   final release is merged to `master`.
+5. If any issues are found post-release, a hotfix / point release may be
+   created, which is merged to `master`, and then reverse merged to
+   `develop`.
+
+#### Betas, and the first release candidate
+
+##### Preparing the `develop` branch
+
+1. Optimally, the `develop` branch will be ready to go, with all
+   relevant PRs already merged.
+2. If there are any PRs pending, merge them **BEFORE** preparing the beta.
+   1. If only one or two PRs need to be merged, merge those PRs [as
+      normal](#when-and-how-to-merge-pull-requests), updating the second
+      one, and waiting for CI to finish in between.
+   2. If there are several pending PRs, do not use the Github UI,
+      because the delays waiting for CI in between each merge will be
+      unnecessarily onerous. (Incidentally, this process can also be
+      used to merge if the Github UI has issues.) Merge each PR branch
+      directly to a `release-next` on your local machine and create a single
+      PR, then push your branch to `develop`.
+      1. Squash the changes from each PR, one commit each (unless more
+         are needed), being sure to sign each commit and update the
+         commit message to include the PR number. You may be able to use
+         a fast-forward merge for the first PR.
+      2. Push your branch.
+      3. Continue to [Making the release](#making-the-release) to update
+         the version number, etc.
+
+      The workflow may look something like:
+```
+git fetch --multiple upstreams user1 user2 user3 [...]
+git checkout -B release-next --no-track upstream/develop
+
+# Only do an ff-only merge if prbranch1 is either already
+# squashed, or needs to be merged with separate commits,
+# and has no merge commits.
+# Use -S on the ff-only merge if prbranch1 isn't signed.
+git merge [-S] --ff-only user1/prbranch1
+
+git merge --squash user2/prbranch2
+git commit -S # Use the commit message provided on the PR
+
+git merge --squash user3/prbranch3
+git commit -S # Use the commit message provided on the PR
+
+[...]
+
+# Make sure the commits look right
+git log --show-signature "upstream/develop..HEAD"
+
+git push --set-upstream origin
+
+# Continue to "Making the release" to update the version number, so
+# everything can be done in one PR.
+```
+
+You can also use the [squash-branches] script.
+
+You may also need to manually close the open PRs after the changes are
+merged to `develop`. Be sure to include the commit ID.
+
+##### Making the release
+
+This includes, betas, and the first release candidate (RC).
+
+1. If you didn't create one [preparing the `develop`
+   branch](#preparing-the-develop-branch), Ensure there is no old
+   `release-next` branch hanging around.  Then make a `release-next`
+   branch that only changes the version number. e.g.
+```
+git fetch upstreams
+
+git checkout --no-track -B release-next upstream/develop
+
+v="A.B.C-bD"
+build=$( find -name BuildInfo.cpp )
+sed 's/\(^.*versionString =\).*$/\1 "'${v}'"/' ${build} > version.cpp && mv -vi version.cpp ${build}
+
+git diff
+
+git add ${build}
+
+git commit -S -m "Set version to ${v}"
+
+# You could use your "origin" repo, but some CI tests work better on upstream.
+git push upstream-push
+git fetch upstreams
+git branch --set-upstream-to=upstream/release-next
+```
+   You can also use the [update-version] script.
+2. Create a Pull Request for `release-next` with **`develop`** as
+   the base branch.
+   1. Use the title "[TRIVIAL] Set version to X.X.X-bX".
+   2. Instead of the default description template, use the following:
+```
+## High Level Overview of Change
+
+This PR only changes the version number. It will be merged as
+soon as Github CI actions successfully complete.
+```
+3. Wait for CI to successfully complete, and get someone to approve
+   the PR. (It is safe to ignore known CI issues.)
+4. Push the updated `develop` branch using your `release-next`
+   branch. **Do not use the Github UI. It's important to preserve
+   commit IDs.**
+```
+git push upstream-push release-next:develop
+```
+5. In the unlikely event that the push fails because someone has merged
+   something else in the meantime, rebase your branch onto the updated
+   `develop` branch, push again, and go back to step 3.
+6. Ensure that your PR against `develop` is closed. Github should do it
+   automatically.
+7. Once this is done, forward progress on `develop` can continue
+   (other PRs may be merged).
+8. Now create a Pull Request for `release-next` with **`release`** as
+   the base branch.  Instead of the default template, reuse and update
+   the message from the previous release. Include the following verbiage
+   somewhere in the description:
+```
+The base branch is `release`. [All releases (including
+betas)](https://github.com/XRPLF/rippled/blob/develop/CONTRIBUTING.md#before-you-start)
+go in `release`. This PR branch will be pushed directly to `release` (not
+squashed or rebased, and not using the GitHub UI).
+```
+7. Sign-offs for the three platforms (Linux, Mac, Windows) usually occur
+   offline, but at least one approval will be needed on the PR.
+   * If issues are discovered during testing, simply abandon the
+     release.  It's easy to start a new release, it should be easy to
+     abandon one. **DO NOT REUSE THE VERSION NUMBER.** e.g. If you
+     abandon 2.4.0-b1, the next attempt will be 2.4.0-b2.
+8. Once everything is ready to go, push to `release`.
+```
+git fetch upstreams
+
+# Just to be safe, do a dry run first:
+git push --dry-run upstream-push release-next:release
+
+# If everything looks right, push the branch
+git push upstream-push release-next:release
+
+# Check that all of the branches are updated
+git fetch upstreams
+git log -1 --oneline
+# The output should look like:
+# 0123456789 (HEAD -> upstream/release-next, upstream/release,
+#            upstream/develop) Set version to 2.4.0-b1
+# Note that upstream/develop may not be on this commit, but
+# upstream/release must be.
+# Other branches, including some from upstream-push, may also be
+# present.
+```
+9. Tag the release, too.
+```
+git tag <version number>
+git push upstream-push <version number>
+```
+10. Delete the `release-next` branch on the repo. Use the Github UI or:
+```
+git push --delete upstream-push release-next
+```
+11. Finally [create a new release on
+    Github](https://github.com/XRPLF/rippled/releases).
+
+#### Release candidates after the first
+
+Once the first release candidate is [merged into
+release](#making-the-release), then `release` and `develop` *are allowed
+to diverge*.
+
+If a bug or issue is discovered in a version that has a release
+candidate being tested, any fix and new version will need to be applied
+against `release`, then reverse-merged to `develop`. This helps keep git
+history as linear as possible.
+
+A `release-next` branch will be created from `release`, and any further
+work for that release must be based on `release-next`.  Specifically,
+PRs must use `release-next` as the base, and those PRs will be merged
+directly to `release-next` when approved. Changes should be restricted
+to bug fixes, but other changes may be necessary from time to time.
+
+1. Open any PRs for the pending release using `release-next` as the base,
+   so they can be merged directly in to it. Unlike `develop`, though,
+   `release-next` can be thrown away and recreated if necessary.
+2. Once a new release candidate is ready, create a version commit as in
+   step 1 [above](#making-the-release) on `release-next`. You can use
+   the [update-version] script for this, too.
+3. Jump to step 8 ("Now create a Pull Request for `release-next` with
+   **`release`** as the base") from the process
+   [above](#making-the-release) to merge `release-next` into `release`.
+
+##### Follow up: reverse merge
+
+Once the RC is merged and tagged, it needs to be reverse merged into
+`develop` as soon as possible.
+
+1. Create a branch, based on `upstream/develop`.
+   The branch name is not important, but could include "mergeNNNrcN".
+   E.g. For release A.B.C-rcD, use `mergeABCrcD`.
+```
+git fetch upstreams
+
+git checkout --no-track -b mergeABCrcD upstream/develop
+```
+2. Merge `release` into your branch.
+```
+# I like the "--edit --log --verbose" parameters, but they are
+# not required.
+git merge upstream/release
+```
+3. `BuildInfo.cpp` will have a conflict with the version number.
+   Resolve it with the version from `develop` - the higher version.
+4. Push your branch to your repo (or `upstream` if you have permission),
+   and open a normal PR against `develop`. The "High level overview" can
+   simply indicate that this is a merge of the RC. The "Context" should
+   summarize the changes from the RC. Include the following text
+   prominently:
+```
+This PR must be merged manually using a push. Do not use the Github UI.
+```
+5. Depending on the complexity of the changes, and/or merge conflicts,
+   the PR may need a thorough review, or just a sign-off that the
+   merge was done correctly.
+6. If `develop` is updated before this PR is merged, do not merge
+   `develop` back into your branch. Instead rebase preserving merges,
+   or do the merge again. (See also the `rerere` git config setting.)
+```
+git rebase --rebase-merges upstream/develop
+# OR
+git reset --hard upstream/develop
+git merge upstream/release
+```
+7. When the PR is ready, push it to `develop`.
+```
+git fetch upstreams
+
+# Make sure the commits look right
+git log --show-signature "upstream/develop^..HEAD"
+
+git push upstream-push mergeABCrcD:develop
+
+git fetch upstreams
+```
+Development on `develop` can proceed as normal.
+
+
+#### Final releases
+
+A final release is any release that is not a beta or RC, such as 2.2.0.
+
+Only code that has already been tested and vetted across all three
+platforms should be included in a final release. Most of the time, that
+means that the commit immediately preceding the commit setting the
+version number will be an RC. Occasionally, there may be last-minute bug
+fixes included as well. If so, those bug fixes must have been tested
+internally as if they were RCs (at minimum, ensuring unit tests pass,
+and the app starts, syncs, and stops cleanly across all three
+platforms.)
+
+*If in doubt, make an RC first.*
+
+The process for building a final release is very similar to [the process
+for building a beta](#making-the-release), except the code will be
+moving from `release` to `master` instead of from `develop` to
+`release`, and both branches will be pushed at the same time.
+
+1. Ensure there is no old `master-next` branch hanging around.
+   Then make a `master-next` branch that only changes the version
+   number. As above, or using the
+   [update-version] script.
+2. Create a Pull Request for `master-next` with **`master`** as
+   the base branch.  Instead of the default template, reuse and update
+   the message from the previous final release. Include the following verbiage
+   somewhere in the description:
+```
+The base branch is `master`. This PR branch will be pushed directly to
+`release` and `master` (not squashed or rebased, and not using the
+GitHub UI).
+```
+7. Sign-offs for the three platforms (Linux, Mac, Windows) usually occur
+   offline, but at least one approval will be needed on the PR.
+   * If issues are discovered during testing, close the PR, delete
+     `master-next`, and move development back to `release`, [issuing
+     more RCs as necessary](#release-candidates-after-the-first)
+8. Once everything is ready to go, push to `release` and `master`.
+```
+git fetch upstreams
+
+# Just to be safe, do dry runs first:
+git push --dry-run upstream-push master-next:release
+git push --dry-run upstream-push master-next:master
+
+# If everything looks right, push the branch
+git push upstream-push master-next:release
+git push upstream-push master-next:master
+
+# Check that all of the branches are updated
+git fetch upstreams
+git log -1 --oneline
+# The output should look like:
+# 0123456789 (HEAD -> upstream/master-next, upstream/master,
+#            upstream/release) Set version to A.B.0
+# Note that both upstream/release and upstream/master must be on this
+# commit.
+# Other branches, including some from upstream-push, may also be
+# present.
+```
+9. Tag the release, too.
+```
+git tag <version number>
+git push upstream-push <version number>
+```
+10. Delete the `master-next` branch on the repo. Use the Github UI or:
+```
+git push --delete upstream-push master-next
+```
+11. [Create a new release on
+    Github](https://github.com/XRPLF/rippled/releases). Be sure that
+    "Set as the latest release" is checked.
+12. Finally [reverse merge the release into `develop`](#follow-up-reverse-merge).
+
+#### Special cases: point releases, hotfixes, etc.
+
+On occassion, a bug or issue is discovered in a version that already
+had a final release. Most of the time, development will have started
+on the next version, and will usually have changes in `develop`
+and often in `release`.
+
+Because git history is kept as linear as possible, any fix and new
+version will need to be applied against `master`.
+
+The process for building a hotfix release is very similar to [the
+process for building release candidates after the
+first](#release-candidates-after-the-first) and [for building a final
+release](#final-releases), except the changes will be done against
+`master` instead of `release`.
+
+If there is only a single issue for the hotfix, the work can be done in
+any branch. When it's ready to merge, jump to step 3 using your branch
+instead of `master-next`.
+
+1. Create a `master-next` branch from `master`.
+```
+git checkout --no-track -b master-next upstream/master
+git push upstream-push
+git fetch upstreams
+```
+2. Open any PRs for the pending hotfix using `master-next` as the base,
+   so they can be merged directly in to it. Unlike `develop`, though,
+   `master-next` can be thrown away and recreated if necessary.
+3. Once the hotfix is ready, create a version commit using the same
+   steps as above, or use the
+   [update-version] script.
+4. Create a Pull Request for `master-next` with **`master`** as
+   the base branch.  Instead of the default template, reuse and update
+   the message from the previous final release. Include the following verbiage
+   somewhere in the description:
+```
+The base branch is `master`. This PR branch will be pushed directly to
+`master` (not squashed or rebased, and not using the GitHub UI).
+```
+7. Sign-offs for the three platforms (Linux, Mac, Windows) usually occur
+   offline, but at least one approval will be needed on the PR.
+   * If issues are discovered during testing, update `master-next` as
+     needed, but ensure that the changes are properly squashed, and the
+     version setting commit remains last
+8. Once everything is ready to go, push to `master` **only**.
+```
+git fetch upstreams
+
+# Just to be safe, do a dry run first:
+git push --dry-run upstream-push master-next:master
+
+# If everything looks right, push the branch
+git push upstream-push master-next:master
+
+# Check that all of the branches are updated
+git fetch upstreams
+git log -1 --oneline
+# The output should look like:
+# 0123456789 (HEAD -> upstream/master-next, upstream/master) Set version
+#            to 2.4.1
+# Note that upstream/master must be on this commit. upstream/release and
+# upstream/develop should not.
+# Other branches, including some from upstream-push, may also be
+# present.
+```
+9. Tag the release, too.
+```
+git tag <version number>
+git push upstream-push <version number>
+```
+9. Delete the `master-next` branch on the repo.
+```
+git push --delete upstream-push master-next
+```
+10. [Create a new release on
+    Github](https://github.com/XRPLF/rippled/releases). Be sure that
+    "Set as the latest release" is checked.
+
+Once the hotfix is released, it needs to be reverse merged into
+`develop` as soon as possible. It may also need to be merged into
+`release` if a release candidate is under development.
+
+1. Create a branch in your own repo, based on `upstream/develop`.
+   The branch name is not important, but could include "mergeNNN".
+   E.g. For release 2.2.3, use `merge223`.
+```
+git fetch upstreams
+
+git checkout --no-track -b merge223 upstream/develop
+```
+2. Merge master into your branch.
+```
+# I like the "--edit --log --verbose" parameters, but they are
+# not required.
+git merge upstream/master
+```
+3. `BuildInfo.cpp` will have a conflict with the version number.
+   Resolve it with the version from `develop` - the higher version.
+4. Push your branch to your repo, and open a normal PR against
+   `develop`. The "High level overview" can simply indicate that this
+   is a merge of the hotfix version. The "Context" should summarize
+   the changes from the hotfix. Include the following text
+   prominently:
+```
+This PR must be merged manually using a --ff-only merge. Do not use the Github UI.
+```
+5. Depending on the complexity of the hotfix, and/or merge conflicts,
+   the PR may need a thorough review, or just a sign-off that the
+   merge was done correctly.
+6. If `develop` is updated before this PR is merged, do not merge
+   `develop` back into your branch. Instead rebase preserving merges,
+   or do the merge again. (See also the `rerere` git config setting.)
+```
+git rebase --rebase-merges upstream/develop
+# OR
+git reset --hard upstream/develop
+git merge upstream/master
+```
+7. When the PR is ready, push it to `develop`.
+```
+git fetch upstreams
+
+# Make sure the commits look right
+git log --show-signature "upstream/develop..HEAD"
+
+git push upstream-push HEAD:develop
+```
+Development on `develop` can proceed as normal. It is recommended to
+create a beta (or RC) immediately to ensure that everything worked as
+expected.
+
+##### An even rarer scenario: A hotfix on an old release
+
+Historically, once a final release is tagged and packages are released,
+versions older than the latest final release are no longer supported.
+However, there is a possibility that a very high severity bug may occur
+in a non-amendment blocked version that is still being run by
+a significant fraction of users, which would necessitate a hotfix / point
+release to that version as well as any later versions.
+
+This scenario would follow the same basic procedure as above,
+except that *none* of `develop`, `release`, or `master`
+would be touched during the release process.
+
+In this example, consider if version 2.1.1 needed to be patched.
+
+1. Create two branches in the main (`upstream`) repo.
+```
+git fetch upstreams
+
+# Create a base branch off the tag
+git checkout --no-track -b master-2.1.2 2.1.1
+git push upstream-push
+
+# Create a working branch
+git checkout --no-track -b master212-next master-2.1.2
+git push upstream-push
+
+git fetch upstreams
+```
+2. Work continues as above, except using `master-2.1.2`as
+   the base branch for any merging, packaging, etc.
+3. After the release is tagged and packages are built, you could
+   potentially delete both branches, e.g. `master-2.1.2` and
+   `master212-next`. However, it may be useful to keep `master-2.1.2`
+   around indefinitely for reference.
+4. Assuming that a hotfix is also released for the latest
+   version in parallel with this one, or if the issue is
+   already fixed in the latest version, do no do any
+   reverse merges. However, if it is not, it probably makes
+   sense to reverse merge `master-2.1.2` into `master`,
+   release a hotfix for _that_ version, then reverse merge
+   from `master` to `develop`. (Please don't do this unless absolutely
+   necessary.)
+
+[contrib]: https://docs.github.com/en/get-started/quickstart/contributing-to-projects
+[squash]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits
+[forking]: https://github.com/XRPLF/rippled/fork
+[rippled]: https://github.com/XRPLF/rippled
+[signing]: https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification
+[setup-upstreams]: ./bin/git/setup-upstreams.sh
+[squash-branches]: ./bin/git/squash-branches.sh
+[update-version]: ./bin/git/update-version.sh

README.md

@@ -1,3 +1,5 @@
+[![codecov](https://codecov.io/gh/XRPLF/rippled/graph/badge.svg?token=WyFr5ajq3O)](https://codecov.io/gh/XRPLF/rippled)
+
 # The XRP Ledger
 
 The [XRP Ledger](https://xrpl.org/) is a decentralized cryptographic ledger powered by a network of peer-to-peer nodes. The XRP Ledger uses a novel Byzantine Fault Tolerant consensus algorithm to settle and record transactions in a secure distributed database without a central operator.
@@ -8,7 +10,7 @@ The [XRP Ledger](https://xrpl.org/) is a decentralized cryptographic ledger powe
 ## rippled
 The server software that powers the XRP Ledger is called `rippled` and is available in this repository under the permissive [ISC open-source license](LICENSE.md). The `rippled` server software is written primarily in C++ and runs on a variety of platforms. The `rippled` server software can run in several modes depending on its [configuration](https://xrpl.org/rippled-server-modes.html).
 
-If you are interested in running an **API Server** (including a **Full History Server**) or a **Reporting Mode** server, take a look at [Clio](https://github.com/XRPLF/clio). rippled Reporting Mode is expected to be replaced by Clio.
+If you are interested in running an **API Server** (including a **Full History Server**), take a look at [Clio](https://github.com/XRPLF/clio). (rippled Reporting Mode has been replaced by Clio.)
 
 ### Build from Source
 

SECURITY.md

@@ -83,7 +83,7 @@ To report a qualifying bug, please send a detailed report to:
 |Long Key ID  | `0xCD49A0AFC57929BE`                                |
 |Fingerprint  | `24E6 3B02 37E0 FA9C 5E96 8974 CD49 A0AF C579 29BE` |
 
-The full PGP key for this address, which is also available on several key servers (e.g. on [keys.gnupg.net](https://keys.gnupg.net)), is: 
+The full PGP key for this address, which is also available on several key servers (e.g. on [keyserver.ubuntu.com](https://keyserver.ubuntu.com)), is: 
 ```
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 mQINBFUwGHYBEAC0wpGpBPkd8W1UdQjg9+cEFzeIEJRaoZoeuJD8mofwI5Ejnjdt

bin/ci/README.md

@@ -1,24 +0,0 @@
-In this directory are two scripts, `build.sh` and `test.sh` used for building
-and testing rippled.
-
-(For now, they assume Bash and Linux. Once I get Windows containers for
-testing, I'll try them there, but if Bash is not available, then they will
-soon be joined by PowerShell scripts `build.ps` and `test.ps`.)
-
-We don't want these scripts to require arcane invocations that can only be
-pieced together from within a CI configuration. We want something that humans
-can easily invoke, read, and understand, for when we eventually have to test
-and debug them interactively. That means:
-
-(1) They should work with no arguments.
-(2) They should document their arguments.
-(3) They should expand short arguments into long arguments.
-
-While we want to provide options for common use cases, we don't need to offer
-the kitchen sink. We can rightfully expect users with esoteric, complicated
-needs to write their own scripts.
-
-To make argument-handling easy for us, the implementers, we can just take all
-arguments from environment variables. They have the nice advantage that every
-command-line uses named arguments. For the benefit of us and our users, we
-document those variables at the top of each script.

bin/ci/build.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-set -o xtrace
-set -o errexit
-
-# The build system. Either 'Unix Makefiles' or 'Ninja'.
-GENERATOR=${GENERATOR:-Unix Makefiles}
-# The compiler. Either 'gcc' or 'clang'.
-COMPILER=${COMPILER:-gcc}
-# The build type. Either 'Debug' or 'Release'.
-BUILD_TYPE=${BUILD_TYPE:-Debug}
-# Additional arguments to CMake.
-# We use the `-` substitution here instead of `:-` so that callers can erase
-# the default by setting `$CMAKE_ARGS` to the empty string.
-CMAKE_ARGS=${CMAKE_ARGS-'-Dwerr=ON'}
-
-# https://gitlab.kitware.com/cmake/cmake/issues/18865
-CMAKE_ARGS="-DBoost_NO_BOOST_CMAKE=ON ${CMAKE_ARGS}"
-
-if [[ ${COMPILER} == 'gcc' ]]; then
-  export CC='gcc'
-  export CXX='g++'
-elif [[ ${COMPILER} == 'clang' ]]; then
-  export CC='clang'
-  export CXX='clang++'
-fi
-
-mkdir build
-cd build
-cmake -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_ARGS} ..
-cmake --build . -- -j $(nproc)

bin/ci/test.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-
-set -o xtrace
-set -o errexit
-
-# Set to 'true' to run the known "manual" tests in rippled.
-MANUAL_TESTS=${MANUAL_TESTS:-false}
-# The maximum number of concurrent tests.
-CONCURRENT_TESTS=${CONCURRENT_TESTS:-$(nproc)}
-# The path to rippled.
-RIPPLED=${RIPPLED:-build/rippled}
-# Additional arguments to rippled.
-RIPPLED_ARGS=${RIPPLED_ARGS:-}
-
-function join_by { local IFS="$1"; shift; echo "$*"; }
-
-declare -a manual_tests=(
-  'beast.chrono.abstract_clock'
-  'beast.unit_test.print'
-  'ripple.NodeStore.Timing'
-  'ripple.app.Flow_manual'
-  'ripple.app.NoRippleCheckLimits'
-  'ripple.app.PayStrandAllPairs'
-  'ripple.consensus.ByzantineFailureSim'
-  'ripple.consensus.DistributedValidators'
-  'ripple.consensus.ScaleFreeSim'
-  'ripple.tx.CrossingLimits'
-  'ripple.tx.FindOversizeCross'
-  'ripple.tx.Offer_manual'
-  'ripple.tx.OversizeMeta'
-  'ripple.tx.PlumpBook'
-)
-
-if [[ ${MANUAL_TESTS} == 'true' ]]; then
-  RIPPLED_ARGS+=" --unittest=$(join_by , "${manual_tests[@]}")"
-else
-  RIPPLED_ARGS+=" --unittest --quiet --unittest-log"
-fi
-RIPPLED_ARGS+=" --unittest-jobs ${CONCURRENT_TESTS}"
-
-${RIPPLED} ${RIPPLED_ARGS}

bin/ci/ubuntu/build-and-test.sh

@@ -1,274 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-function version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
-
-__dirname=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-echo "using CC: ${CC}"
-"${CC}" --version
-export CC
-
-COMPNAME=$(basename $CC)
-echo "using CXX: ${CXX:-notset}"
-if [[ $CXX ]]; then
-   "${CXX}" --version
-   export CXX
-fi
-: ${BUILD_TYPE:=Debug}
-echo "BUILD TYPE: ${BUILD_TYPE}"
-
-: ${TARGET:=install}
-echo "BUILD TARGET: ${TARGET}"
-
-JOBS=${NUM_PROCESSORS:-2}
-if [[ ${TRAVIS:-false} != "true" ]]; then
-    JOBS=$((JOBS+1))
-fi
-
-if [[ ! -z "${CMAKE_EXE:-}" ]] ; then
-    export PATH="$(dirname ${CMAKE_EXE}):$PATH"
-fi
-
-if [ -x /usr/bin/time ] ; then
-    : ${TIME:="Duration: %E"}
-    export TIME
-    time=/usr/bin/time
-else
-    time=
-fi
-
-echo "Building rippled"
-: ${CMAKE_EXTRA_ARGS:=""}
-if [[ ${NINJA_BUILD:-} == true ]]; then
-    CMAKE_EXTRA_ARGS+=" -G Ninja"
-fi
-
-coverage=false
-if [[ "${TARGET}" == "coverage" ]] ; then
-    echo "coverage option detected."
-    coverage=true
-fi
-
-cmake --version
-CMAKE_VER=$(cmake --version | cut -d " " -f 3 | head -1)
-
-#
-# allow explicit setting of the name of the build
-# dir, otherwise default to the compiler.build_type
-#
-: "${BUILD_DIR:=${COMPNAME}.${BUILD_TYPE}}"
-BUILDARGS="--target ${TARGET}"
-BUILDTOOLARGS=""
-if version_ge $CMAKE_VER "3.12.0" ; then
-    BUILDARGS+=" --parallel"
-fi
-
-if [[ ${NINJA_BUILD:-} == false ]]; then
-    if version_ge $CMAKE_VER "3.12.0" ; then
-        BUILDARGS+=" ${JOBS}"
-    else
-        BUILDTOOLARGS+=" -j ${JOBS}"
-    fi
-fi
-
-if [[ ${VERBOSE_BUILD:-} == true ]]; then
-    CMAKE_EXTRA_ARGS+=" -DCMAKE_VERBOSE_MAKEFILE=ON"
-    if version_ge $CMAKE_VER "3.14.0" ; then
-        BUILDARGS+=" --verbose"
-    else
-        if [[ ${NINJA_BUILD:-} == false ]]; then
-            BUILDTOOLARGS+=" verbose=1"
-        else
-            BUILDTOOLARGS+=" -v"
-        fi
-    fi
-fi
-
-if [[ ${USE_CCACHE:-} == true ]]; then
-    echo "using ccache with basedir [${CCACHE_BASEDIR:-}]"
-    CMAKE_EXTRA_ARGS+=" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
-fi
-if [ -d "build/${BUILD_DIR}" ]; then
-    rm -rf "build/${BUILD_DIR}"
-fi
-
-mkdir -p "build/${BUILD_DIR}"
-pushd "build/${BUILD_DIR}"
-
-# cleanup possible artifacts
-rm -fv CMakeFiles/CMakeOutput.log CMakeFiles/CMakeError.log
-# Clean up NIH directories which should be git repos, but aren't
-for nih_path in ${NIH_CACHE_ROOT}/*/*/*/src ${NIH_CACHE_ROOT}/*/*/src
-do
-  for dir in lz4 snappy rocksdb
-  do
-    if [ -e ${nih_path}/${dir} -a \! -e ${nih_path}/${dir}/.git ]
-    then
-      ls -la ${nih_path}/${dir}*
-      rm -rfv ${nih_path}/${dir}*
-    fi
-  done
-done
-
-# generate
-${time} cmake ../.. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_EXTRA_ARGS}
-# Display the cmake output, to help with debugging if something fails
-for file in CMakeOutput.log CMakeError.log
-do
-  if [ -f CMakeFiles/${file} ]
-  then
-    ls -l CMakeFiles/${file}
-    cat CMakeFiles/${file}
-  fi
-done
-# build
-export DESTDIR=$(pwd)/_INSTALLED_
-
-${time} eval cmake --build . ${BUILDARGS} -- ${BUILDTOOLARGS}
-
-if [[ ${TARGET} == "docs" ]]; then
-    ## mimic the standard test output for docs build
-    ## to make controlling processes like jenkins happy
-    if [ -f docs/html/index.html ]; then
-        echo "1 case, 1 test total, 0 failures"
-    else
-        echo "1 case, 1 test total, 1 failures"
-    fi
-    exit
-fi
-popd
-
-if [[ "${TARGET}" == "validator-keys" ]] ; then
-    export APP_PATH="$PWD/build/${BUILD_DIR}/validator-keys/validator-keys"
-else
-    export APP_PATH="$PWD/build/${BUILD_DIR}/rippled"
-fi
-echo "using APP_PATH: ${APP_PATH}"
-
-# See what we've actually built
-ldd ${APP_PATH}
-
-: ${APP_ARGS:=}
-
-if [[ "${TARGET}" == "validator-keys" ]] ; then
-    APP_ARGS="--unittest"
-else
-    function join_by { local IFS="$1"; shift; echo "$*"; }
-
-    # This is a list of manual tests
-    # in rippled that we want to run
-    # ORDER matters here...sorted in approximately
-    # descending execution time (longest running tests at top)
-    declare -a manual_tests=(
-        'ripple.ripple_data.reduce_relay_simulate'
-        'ripple.tx.Offer_manual'
-        'ripple.tx.CrossingLimits'
-        'ripple.tx.PlumpBook'
-        'ripple.app.Flow_manual'
-        'ripple.tx.OversizeMeta'
-        'ripple.consensus.DistributedValidators'
-        'ripple.app.NoRippleCheckLimits'
-        'ripple.ripple_data.compression'
-        'ripple.NodeStore.Timing'
-        'ripple.consensus.ByzantineFailureSim'
-        'beast.chrono.abstract_clock'
-        'beast.unit_test.print'
-    )
-    if [[ ${TRAVIS:-false} != "true" ]]; then
-        # these two tests cause travis CI to run out of memory.
-        # TODO: investigate possible workarounds.
-        manual_tests=(
-            'ripple.consensus.ScaleFreeSim'
-            'ripple.tx.FindOversizeCross'
-            "${manual_tests[@]}"
-        )
-    fi
-
-    if [[ ${MANUAL_TESTS:-} == true ]]; then
-        APP_ARGS+=" --unittest=$(join_by , "${manual_tests[@]}")"
-    else
-        APP_ARGS+=" --unittest --quiet --unittest-log"
-    fi
-    if [[ ${coverage} == false && ${PARALLEL_TESTS:-} == true ]]; then
-        APP_ARGS+=" --unittest-jobs ${JOBS}"
-    fi
-
-    if [[ ${IPV6_TESTS:-} == true ]]; then
-        APP_ARGS+=" --unittest-ipv6"
-    fi
-fi
-
-if [[ ${coverage} == true && $CC =~ ^gcc ]]; then
-    # Push the results (lcov.info) to codecov
-    codecov -X gcov # don't even try and look for .gcov files ;)
-    find . -name "*.gcda" | xargs rm -f
-fi
-
-if [[ ${SKIP_TESTS:-} == true ]]; then
-    echo "skipping tests."
-    exit
-fi
-
-ulimit -a
-corepat=$(cat /proc/sys/kernel/core_pattern)
-if [[ ${corepat} =~ ^[:space:]*\| ]] ; then
-    echo "WARNING: core pattern is piping - can't search for core files"
-    look_core=false
-else
-    look_core=true
-    coredir=$(dirname ${corepat})
-fi
-if [[ ${look_core} == true ]]; then
-    before=$(ls -A1 ${coredir})
-fi
-
-set +e
-echo "Running tests for ${APP_PATH}"
-if [[ ${MANUAL_TESTS:-} == true && ${PARALLEL_TESTS:-} != true ]]; then
-    for t in "${manual_tests[@]}" ; do
-        ${APP_PATH} --unittest=${t}
-        TEST_STAT=$?
-        if [[ $TEST_STAT -ne 0 ]] ; then
-            break
-        fi
-    done
-else
-    ${APP_PATH} ${APP_ARGS}
-    TEST_STAT=$?
-fi
-set -e
-
-if [[ ${look_core} == true ]]; then
-    after=$(ls -A1 ${coredir})
-    oIFS="${IFS}"
-    IFS=$'\n\r'
-    found_core=false
-    for l in $(diff -w --suppress-common-lines <(echo "$before") <(echo "$after")) ; do
-        if [[ "$l" =~ ^[[:space:]]*\>[[:space:]]*(.+)$ ]] ; then
-            corefile="${BASH_REMATCH[1]}"
-            echo "FOUND core dump file at '${coredir}/${corefile}'"
-            gdb_output=$(/bin/mktemp /tmp/gdb_output_XXXXXXXXXX.txt)
-            found_core=true
-            gdb \
-                -ex "set height 0" \
-                -ex "set logging file ${gdb_output}" \
-                -ex "set logging on" \
-                -ex "print 'ripple::BuildInfo::versionString'" \
-                -ex "thread apply all backtrace full" \
-                -ex "info inferiors" \
-                -ex quit \
-                "$APP_PATH" \
-                "${coredir}/${corefile}" &> /dev/null
-
-            echo -e "CORE INFO: \n\n $(cat ${gdb_output}) \n\n)"
-        fi
-    done
-    IFS="${oIFS}"
-fi
-
-if [[ ${found_core} == true ]]; then
-    exit -1
-else
-    exit $TEST_STAT
-fi
-

bin/ci/ubuntu/build-in-docker.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# run our build script in a docker container
-# using travis-ci hosts
-set -eux
-
-function join_by { local IFS="$1"; shift; echo "$*"; }
-
-set +x
-echo "VERBOSE_BUILD=true" > /tmp/co.env
-matchers=(
-   'TRAVIS.*' 'CI' 'CC' 'CXX'
-   'BUILD_TYPE' 'TARGET' 'MAX_TIME'
-   'CODECOV.+' 'CMAKE.*' '.+_TESTS'
-   '.+_OPTIONS' 'NINJA.*' 'NUM_.+'
-   'NIH_.+' 'BOOST.*' '.*CCACHE.*')
-
-matchstring=$(join_by '|' "${matchers[@]}")
-echo "MATCHSTRING IS:: $matchstring"
-env | grep -E "^(${matchstring})=" >> /tmp/co.env
-set -x
-# need to eliminate TRAVIS_CMD...don't want to pass it to the container
-cat /tmp/co.env | grep -v TRAVIS_CMD > /tmp/co.env.2
-mv /tmp/co.env.2 /tmp/co.env
-cat /tmp/co.env
-mkdir -p -m 0777 ${TRAVIS_BUILD_DIR}/cores
-echo "${TRAVIS_BUILD_DIR}/cores/%e.%p" | sudo tee /proc/sys/kernel/core_pattern
-docker run \
-    -t --env-file /tmp/co.env \
-    -v ${TRAVIS_HOME}:${TRAVIS_HOME} \
-    -w ${TRAVIS_BUILD_DIR} \
-    --cap-add SYS_PTRACE \
-    --ulimit "core=-1" \
-    $DOCKER_IMAGE \
-    /bin/bash -c 'if [[ $CC =~ ([[:alpha:]]+)-([[:digit:].]+) ]] ; then sudo update-alternatives --set ${BASH_REMATCH[1]} /usr/bin/$CC; fi; bin/ci/ubuntu/build-and-test.sh'
-
-

bin/ci/ubuntu/travis-cache-start.sh

@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-# some cached files create churn, so save them here for
-# later restoration before packing the cache
-set -eux
-clean_cache="travis_clean_cache"
-if [[ ! ( "${TRAVIS_JOB_NAME}" =~ "windows" || \
-    "${TRAVIS_JOB_NAME}" =~ "prereq-keep" ) ]] && \
-    ( [[ "${TRAVIS_COMMIT_MESSAGE}" =~ "${clean_cache}" ]] || \
-        ( [[ -v TRAVIS_PULL_REQUEST_SHA && \
-            "${TRAVIS_PULL_REQUEST_SHA}" != "" ]] && \
-          git log -1 "${TRAVIS_PULL_REQUEST_SHA}" | grep -cq "${clean_cache}" -
-        )
-    )
-then
-    find ${TRAVIS_HOME}/_cache -maxdepth 2 -type d
-    rm -rf ${TRAVIS_HOME}/_cache
-    mkdir -p ${TRAVIS_HOME}/_cache
-fi
-
-pushd ${TRAVIS_HOME}
-if [ -f cache_ignore.tar ] ; then
-    rm -f cache_ignore.tar
-fi
-
-if [ -d _cache/nih_c ] ; then
-    find _cache/nih_c -name "build.ninja" | tar rf cache_ignore.tar --files-from -
-    find _cache/nih_c -name ".ninja_deps" | tar rf cache_ignore.tar --files-from -
-    find _cache/nih_c -name ".ninja_log" | tar rf cache_ignore.tar --files-from -
-    find _cache/nih_c -name "*.log" | tar rf cache_ignore.tar --files-from -
-    find _cache/nih_c -name "*.tlog" | tar rf cache_ignore.tar --files-from -
-    # show .a files in the cache, for sanity checking
-    find _cache/nih_c -name "*.a" -ls
-fi
-
-if [ -d _cache/ccache ] ; then
-    find _cache/ccache -name "stats" | tar rf cache_ignore.tar --files-from -
-fi
-
-if [ -f cache_ignore.tar ] ; then
-    tar -tf cache_ignore.tar
-fi
-popd
-
-

bin/git/setup-upstreams.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+if [[ $# -ne 1 || "$1" == "--help" || "$1" == "-h" ]]
+then
+  name=$( basename $0 )
+  cat <<- USAGE
+  Usage: $name <username>
+  
+  Where <username> is the Github username of the upstream repo. e.g. XRPLF
+USAGE
+  exit 0
+fi
+
+# Create upstream remotes based on origin
+shift
+user="$1"
+# Get the origin URL. Expect it be an SSH-style URL
+origin=$( git remote get-url origin )
+if [[ "${origin}" == "" ]]
+then
+  echo Invalid origin remote >&2
+  exit 1
+fi
+# echo "Origin: ${origin}"
+# Parse the origin
+ifs_orig="${IFS}"
+IFS=':' read remote originpath <<< "${origin}"
+# echo "Remote: ${remote}, Originpath: ${originpath}"
+IFS='@' read sshuser server <<< "${remote}"
+# echo "SSHUser: ${sshuser}, Server: ${server}"
+IFS='/' read originuser repo <<< "${originpath}"
+# echo "Originuser: ${originuser}, Repo: ${repo}"
+if [[ "${sshuser}" == "" || "${server}" == "" || "${originuser}" == ""
+  || "${repo}" == "" ]]
+then
+  echo "Can't parse origin URL: ${origin}" >&2
+  exit 1
+fi
+upstream="https://${server}/${user}/${repo}"
+upstreampush="${remote}:${user}/${repo}"
+upstreamgroup="upstream upstream-push"
+current=$( git remote get-url upstream 2>/dev/null )
+currentpush=$( git remote get-url upstream-push 2>/dev/null )
+currentgroup=$( git config remotes.upstreams )
+if [[ "${current}" == "${upstream}" ]]
+then
+  echo "Upstream already set up correctly. Skip"
+elif [[ -n "${current}" && "${current}" != "${upstream}" &&
+  "${current}" != "${upstreampush}" ]]
+then
+  echo "Upstream already set up as: ${current}. Skip"
+else
+  if [[ "${current}" == "${upstreampush}" ]]
+  then
+    echo "Upstream set to dangerous push URL. Update."
+    _run git remote rename upstream upstream-push || \
+    _run git remote remove upstream
+    currentpush=$( git remote get-url upstream-push 2>/dev/null )
+  fi
+  _run git remote add upstream "${upstream}"
+fi
+
+if [[ "${currentpush}" == "${upstreampush}" ]]
+then
+  echo "upstream-push already set up correctly. Skip"
+elif [[ -n "${currentpush}" && "${currentpush}" != "${upstreampush}" ]]
+then
+  echo "upstream-push already set up as: ${currentpush}. Skip"
+else
+  _run git remote add upstream-push "${upstreampush}"
+fi
+
+if [[ "${currentgroup}" == "${upstreamgroup}" ]]
+then
+  echo "Upstreams group already set up correctly. Skip"
+elif [[ -n "${currentgroup}" && "${currentgroup}" != "${upstreamgroup}" ]]
+then
+  echo "Upstreams group already set up as: ${currentgroup}. Skip"
+else
+  _run git config --add remotes.upstreams "${upstreamgroup}"
+fi
+
+_run git fetch --jobs=$(nproc) upstreams
+
+exit 0
+

bin/git/squash-branches.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+if [[ $# -lt 3 || "$1" == "--help" || "$1" = "-h" ]]
+then
+  name=$( basename $0 )
+  cat <<- USAGE
+  Usage: $name workbranch base/branch user/branch [user/branch [...]]
+  
+  * workbranch will be created locally from base/branch
+  * base/branch and user/branch may be specified as user:branch to allow
+    easy copying from Github PRs
+  * Remotes for each user must already be set up
+USAGE
+exit 0
+fi
+
+work="$1"
+shift
+
+branches=( $( echo "${@}" | sed "s/:/\//" ) )
+base="${branches[0]}"
+unset branches[0]
+
+set -e
+
+users=()
+for b in "${branches[@]}"
+do
+  users+=( $( echo $b | cut -d/ -f1 ) )
+done
+
+users=( $( printf '%s\n' "${users[@]}" | sort -u ) )
+
+git fetch --multiple upstreams "${users[@]}"
+git checkout -B "$work" --no-track "$base"
+
+for b in "${branches[@]}"
+do
+  git merge --squash "${b}"
+  git commit -S # Use the commit message provided on the PR
+done
+
+# Make sure the commits look right
+git log --show-signature "$base..HEAD"
+
+parts=( $( echo $base | sed "s/\// /" ) )
+repo="${parts[0]}"
+b="${parts[1]}"
+push=$repo
+if [[ "$push" == "upstream" ]]
+then
+  push="upstream-push"
+fi
+if [[ "$repo" == "upstream" ]]
+then
+  repo="upstreams"
+fi
+cat << PUSH
+
+-------------------------------------------------------------------
+This script will not push. Verify everything is correct, then push
+to your repo, and create a PR if necessary. Once the PR is approved,
+run:
+
+git push $push HEAD:$b
+git fetch $repo
+-------------------------------------------------------------------
+PUSH
+

bin/git/update-version.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+if [[ $# -ne 3 || "$1" == "--help" || "$1" = "-h" ]]
+then
+  name=$( basename $0 )
+  cat <<- USAGE
+  Usage: $name workbranch base/branch version
+
+  * workbranch will be created locally from base/branch. If it exists,
+    it will be reused, so make sure you don't overwrite any work.
+  * base/branch may be specified as user:branch to allow easy copying
+    from Github PRs.
+USAGE
+exit 0
+fi
+
+work="$1"
+shift
+
+base=$( echo "$1" | sed "s/:/\//" )
+shift
+
+version=$1
+shift
+
+set -e
+
+git fetch upstreams
+
+git checkout -B "${work}" --no-track "${base}"
+
+push=$( git rev-parse --abbrev-ref --symbolic-full-name '@{push}' \
+              2>/dev/null ) || true
+if [[ "${push}" != "" ]]
+then
+  echo "Warning: ${push} may already exist."
+fi
+
+build=$( find -name BuildInfo.cpp )
+sed 's/\(^.*versionString =\).*$/\1 "'${version}'"/' ${build} > version.cpp && \
+diff "${build}" version.cpp && exit 1 || \
+mv -vi version.cpp ${build}
+
+git diff
+
+git add ${build}
+
+git commit -S -m "Set version to ${version}"
+
+git log --oneline --first-parent ${base}^..
+
+cat << PUSH
+
+-------------------------------------------------------------------
+This script will not push. Verify everything is correct, then push
+to your repo, and create a PR as described in CONTRIBUTING.md.
+-------------------------------------------------------------------
+PUSH

cfg/initdb.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-# Execute this script with a running Postgres server on the current host.
-# It should work with the most generic installation of Postgres,
-# and is necessary for rippled to store data in Postgres.
-
-# usage: sudo -u postgres ./initdb.sh
-psql -c "CREATE USER rippled"
-psql -c "CREATE DATABASE rippled WITH OWNER = rippled"
-

cfg/rippled-example.cfg

@@ -13,7 +13,7 @@
 #
 #   4. HTTPS Client
 #
-#   5. Reporting Mode
+#   5. <vacated>
 #
 #   6. Database
 #
@@ -410,13 +410,17 @@
 #   starter list is included in the code and used if no other hostnames are
 #   available.
 #
-#   One address or domain name per line is allowed. A port may must be
-#   specified after adding a space to the address.  The ordering of entries
-#   does not generally matter.
+#   One address or domain name per line is allowed. A port may be specified
+#   after adding a space to the address. If a port is not specified, the default
+#   port of 2459 will be used. Many servers still use the legacy port of 51235.
+#   To connect to such servers, you must specify the port number. The ordering
+#   of entries does not generally matter.
 #
 #   The default list of entries is:
 #     - r.ripple.com 51235
 #     - sahyadri.isrdc.in 51235
+#     - hubs.xrpkuwait.com 51235
+#     - hub.xrpl-commons.org 51235
 #
 #   Examples:
 #
@@ -883,119 +887,6 @@
 #
 #-------------------------------------------------------------------------------
 #
-# 5. Reporting Mode
-#
-#------------
-#
-#   rippled has an optional operating mode called Reporting Mode. In Reporting
-#   Mode, rippled does not connect to the peer to peer network. Instead, rippled
-#   will continuously extract data from one or more rippled servers that are
-#   connected to the peer to peer network (referred to as an ETL source).
-#   Reporting mode servers will forward RPC requests that require access to the
-#   peer to peer network (submit, fee, etc) to an ETL source.
-#
-#   [reporting]  Settings for Reporting Mode. If and only if this section is
-#                present, rippled will start in reporting mode. This section
-#                contains a list of ETL source names, and key-value pairs. The
-#                ETL source names each correspond to a configuration file
-#                section; the names must match exactly. The key-value pairs are
-#                optional.
-#
-#
-#   [<name>]
-#
-#   A series of key/value pairs that specify an ETL source.
-#
-#   source_ip = <IP-address>
-#
-#       Required. IP address of the ETL source. Can also be a DNS record.
-#
-#   source_ws_port = <number>
-#
-#       Required. Port on which ETL source is accepting unencrypted websocket
-#       connections.
-#
-#   source_grpc_port = <number>
-#
-#       Required for ETL. Port on which ETL source is accepting gRPC requests.
-#       If this option is ommitted, this ETL source cannot actually be used for
-#       ETL; the Reporting Mode server can still forward RPCs to this ETL
-#       source, but cannot extract data from this ETL source.
-#
-#
-#   Key-value pairs (all optional):
-#
-#       read_only    Valid values: 0, 1. Default is 0. If set to 1, the server
-#                    will start in strict read-only mode, and will not perform
-#                    ETL. The server will still handle RPC requests, and will
-#                    still forward RPC requests that require access to the p2p
-#                    network.
-#
-#      start_sequence
-#                   Sequence of first ledger to extract if the database is empty.
-#                   ETL extracts ledgers in order. If this setting is absent and
-#                   the database is empty, ETL will start with the next ledger
-#                   validated by the network. If this setting is present and the
-#                   database is not empty, an exception is thrown.
-#
-#     num_markers   Degree of parallelism used during the initial ledger
-#                   download. Only used if the database is empty. Valid values
-#                   are 1-256. A higher degree of parallelism results in a
-#                   faster download, but puts more load on the ETL source.
-#                   Default is 2.
-#
-#   Example:
-#
-#     [reporting]
-#     etl_source1
-#     etl_source2
-#     read_only=0
-#     start_sequence=32570
-#     num_markers=8
-#
-#     [etl_source1]
-#     source_ip=1.2.3.4
-#     source_ws_port=6005
-#     source_grpc_port=50051
-#
-#     [etl_source2]
-#     source_ip=5.6.7.8
-#     source_ws_port=6005
-#     source_grpc_port=50051
-#
-#   Minimal Example:
-#
-#     [reporting]
-#     etl_source1
-#
-#     [etl_source1]
-#     source_ip=1.2.3.4
-#     source_ws_port=6005
-#     source_grpc_port=50051
-#
-#
-#   Notes:
-#
-#   Reporting Mode requires Postgres (instead of SQLite). The Postgres
-#   connection info is specified under the [ledger_tx_tables] config section;
-#   see the Database section for further documentation.
-#
-#   Each ETL source specified must have gRPC enabled (by adding a [port_grpc]
-#   section to the config). It is recommended to add a secure_gateway entry to
-#   the gRPC section, in order to bypass the server's rate limiting.
-#   This section needs to be added to the config of the ETL source, not
-#   the config of the reporting node. In the example below, the
-#   reporting server is running at 127.0.0.1. Multiple IPs can be
-#   specified in secure_gateway via a comma separated list.
-#
-#   [port_grpc]
-#   ip = 0.0.0.0
-#   port = 50051
-#   secure_gateway = 127.0.0.1
-#
-#
-#-------------------------------------------------------------------------------
-#
 # 6. Database
 #
 #------------
@@ -1003,13 +894,7 @@
 #   rippled creates 4 SQLite database to hold bookkeeping information
 #   about transactions, local credentials, and various other things.
 #   It also creates the NodeDB, which holds all the objects that
-#   make up the current and historical ledgers. In Reporting Mode, rippled
-#   uses a Postgres database instead of SQLite.
-#
-#   The simplest way to work with Postgres is to install it locally.
-#   When it is running, execute the initdb.sh script in the current
-#   directory as: sudo -u postgres ./initdb.sh
-#   This will create the rippled user and an empty database of the same name.
+#   make up the current and historical ledgers.
 #
 #   The size of the NodeDB grows in proportion to the amount of new data and the
 #   amount of historical data (a configurable setting) so the performance of the
@@ -1051,33 +936,10 @@
 #       keeping full history is not advised, and using online delete is
 #       recommended.
 #
-#   type = Cassandra
-#
-#       Apache Cassandra is an open-source, distributed key-value store - see
-#       https://cassandra.apache.org/ for more details.
-#
-#       Cassandra is an alternative backend to be used only with Reporting Mode.
-#       See the Reporting Mode section for more details about Reporting Mode.
-#
 #   Required keys for NuDB and RocksDB:
 #
 #       path                Location to store the database
 #
-#   Required keys for Cassandra:
-#
-#      contact_points       IP of a node in the Cassandra cluster
-#
-#      port                 CQL Native Transport Port
-#
-#      secure_connect_bundle
-#                           Absolute path to a secure connect bundle. When using
-#                           a secure connect bundle, contact_points and port are
-#                           not required.
-#
-#      keyspace             Name of Cassandra keyspace to use
-#
-#      table_name           Name of table in above keyspace to use
-#
 #   Optional keys
 #
 #       cache_size          Size of cache for database records. Default is 16384.
@@ -1094,7 +956,7 @@
 #                           default value for the unspecified parameter.
 #
 #                           Note: the cache will not be created if online_delete
-#                           is specified, or if shards are used.
+#                           is specified.
 #
 #       fast_load           Boolean. If set, load the last persisted ledger
 #                           from disk upon process start before syncing to
@@ -1107,10 +969,6 @@
 #       earliest_seq        The default is 32570 to match the XRP ledger
 #                           network's earliest allowed sequence. Alternate
 #                           networks may set this value. Minimum value of 1.
-#                           If a [shard_db] section is defined, and this
-#                           value is present either [node_db] or [shard_db],
-#                           it must be defined with the same value in both
-#                           sections.
 #
 #       online_delete       Minimum value of 256. Enable automatic purging
 #                           of older ledger information. Maintain at least this
@@ -1157,25 +1015,6 @@
 #                           checking until healthy.
 #                           Default is 5.
 #
-#   Optional keys for Cassandra:
-#
-#       username            Username to use if Cassandra cluster requires
-#                           authentication
-#
-#       password            Password to use if Cassandra cluster requires
-#                           authentication
-#
-#       max_requests_outstanding
-#                           Limits the maximum number of concurrent database
-#                           writes. Default is 10 million. For slower clusters,
-#                           large numbers of concurrent writes can overload the
-#                           cluster. Setting this option can help eliminate
-#                           write timeouts and other write errors due to the
-#                           cluster being overloaded.
-#       io_threads
-#                           Set the number of IO threads used by the
-#                           Cassandra driver. Defaults to 4.
-#
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.
 #
@@ -1192,32 +1031,6 @@
 #   your rippled.cfg file.
 #   Partial pathnames are relative to the location of the rippled executable.
 #
-#   [shard_db]      Settings for the Shard Database (optional)
-#
-#   Format (without spaces):
-#       One or more lines of case-insensitive key / value pairs:
-#       <key> '=' <value>
-#       ...
-#
-#   Example:
-#       path=db/shards/nudb
-#
-#   Required keys:
-#       path                Location to store the database
-#
-#   Optional keys:
-#       max_historical_shards
-#                           The maximum number of historical shards
-#                           to store.
-#
-#   [historical_shard_paths]      Additional storage paths for the Shard Database (optional)
-#
-#   Format (without spaces):
-#       One or more lines, each expressing a full path for storing historical shards:
-#       /mnt/disk1
-#       /mnt/disk2
-#       ...
-#
 #   [sqlite]       Tuning settings for the SQLite databases (optional)
 #
 #   Format (without spaces):
@@ -1297,40 +1110,18 @@
 #                           This setting may not be combined with the
 #                           "safety_level" setting.
 #
-#  [ledger_tx_tables] (optional)
+#       page_size           Valid values: integer (MUST be power of 2 between 512 and 65536)
+#                           The default is 4096 bytes. This setting determines
+#                           the size of a page in the transaction.db file.
+#                           See https://www.sqlite.org/pragma.html#pragma_page_size 
+#                           for more details about the available options.
 #
-#      conninfo             Info for connecting to Postgres. Format is
-#                           postgres://[username]:[password]@[ip]/[database].
-#                           The database and user must already exist. If this
-#                           section is missing and rippled is running in
-#                           Reporting Mode, rippled will connect as the
-#                           user running rippled to a database with the
-#                           same name. On Linux and Mac OS X, the connection
-#                           will take place using the server's UNIX domain
-#                           socket. On Windows, through the localhost IP
-#                           address. Default is empty.
-#
-#      use_tx_tables        Valid values: 1, 0
-#                           The default is 1 (true). Determines whether to use
-#                           the SQLite transaction database. If set to 0,
-#                           rippled will not write to the transaction database,
-#                           and will reject tx, account_tx and tx_history RPCs.
-#                           In Reporting Mode, this setting is ignored.
-#
-#      max_connections      Valid values: any positive integer up to 64 bit
-#                           storage length. This configures the maximum
-#                           number of concurrent connections to postgres.
-#                           Default is the maximum possible value to
-#                           fit in a 64 bit integer.
-#
-#      timeout              Number of seconds after which idle postgres
-#                           connections are discconnected. If set to 0,
-#                           connections never timeout. Default is 600.
-#
-#
-#      remember_ip          Value values: 1, 0
-#                           Default is 1 (true). Whether to cache host and
-#                           port connection settings.
+#       journal_size_limit  Valid values: integer
+#                           The default is 1582080. This setting limits
+#                           the size of the journal for transaction.db file. When the limit is
+#                           reached, older entries will be deleted.
+#                           See https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+#                           for more details about the available options.
 #
 #
 #-------------------------------------------------------------------------------
@@ -1596,6 +1387,12 @@
 #       Admin level API commands over Secure Websockets, when originating
 #       from the same machine (via the loopback adapter at 127.0.0.1).
 #
+#   "grpc"
+#
+#       ETL commands for Clio. We recommend setting secure_gateway
+#       in this section to a comma-separated list of the addresses
+#       of your Clio servers, in order to bypass rippled's rate limiting.
+#
 #   This port is commented out but can be enabled by removing
 #   the '#' from each corresponding line including the entry under [server]
 #
@@ -1629,6 +1426,9 @@ admin = 127.0.0.1
 protocol = http
 
 [port_peer]
+# Many servers still use the legacy port of 51235, so for backward-compatibility
+# we maintain that port number here. However, for new servers we recommend
+# changing this to the default port of 2459.
 port = 51235
 ip = 0.0.0.0
 # alternatively, to accept connections on IPv4 + IPv6, use:
@@ -1674,34 +1474,10 @@ path=/var/lib/rippled/db/nudb
 online_delete=512
 advisory_delete=0
 
-# This is the persistent datastore for shards. It is important for the health
-# of the ripple network that rippled operators shard as much as practical.
-# NuDB requires SSD storage. Helpful information can be found at
-# https://xrpl.org/history-sharding.html
-#[shard_db]
-#path=/var/lib/rippled/db/shards/nudb
-#max_historical_shards=50
-#
-# This optional section can be configured with a list
-# of paths to use for storing historical shards. Each
-# path must correspond to a unique filesystem.
-#[historical_shard_paths]
-#/path/1
-#/path/2
-
 [database_path]
 /var/lib/rippled/db
 
 
-# To use Postgres, uncomment this section and fill in the appropriate connection
-# info. Postgres can only be used in Reporting Mode.
-# To disable writing to the transaction database, uncomment this section, and
-# set use_tx_tables=0
-# [ledger_tx_tables]
-# conninfo = postgres://[username]:[password]@[ip]/[database]
-# use_tx_tables=1
-
-
 # This needs to be an absolute directory reference, not a relative one.
 # Modify this value as required.
 [debug_logfile]
@@ -1729,15 +1505,3 @@ validators.txt
 # set to ssl_verify to 0.
 [ssl_verify]
 1
-
-
-# To run in Reporting Mode, uncomment this section and fill in the appropriate
-# connection info for one or more ETL sources.
-# [reporting]
-# etl_source
-#
-#
-# [etl_source]
-# source_grpc_port=50051
-# source_ws_port=6005
-# source_ip=127.0.0.1

cfg/validators-example.txt

@@ -26,7 +26,7 @@
 #
 #   Examples:
 #    https://vl.ripple.com
-#    https://vl.xrplf.org
+#    https://unl.xrplf.org
 #    http://127.0.0.1:8000
 #    file:///etc/opt/ripple/vl.txt
 #
@@ -54,13 +54,13 @@
 
 [validator_list_sites]
 https://vl.ripple.com
-https://vl.xrplf.org
+https://unl.xrplf.org
 
 [validator_list_keys]
 #vl.ripple.com
 ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734
-# vl.xrplf.org
-ED45D1840EE724BE327ABE9146503D5848EFD5F38B6D5FEDE71E80ACCE5E6E738B
+#unl.xrplf.org
+ED42AEC58B701EEBB77356FFFEC26F83C1F0407263530F068C7C73D392C7E06FD1
 
 # To use the test network (see https://xrpl.org/connect-your-rippled-to-the-xrp-test-net.html),
 # use the following configuration instead:
@@ -70,3 +70,21 @@ ED45D1840EE724BE327ABE9146503D5848EFD5F38B6D5FEDE71E80ACCE5E6E738B
 #
 # [validator_list_keys]
 # ED264807102805220DA0F312E71FC2C69E1552C9C5790F6C25E3729DEB573D5860
+
+
+# [validator_list_threshold]
+#
+#   Minimum number of validator lists on which a validator must be listed in
+#   order to be used.
+#
+#   This can be set explicitly to any positive integer number not greater than
+#   the size of [validator_list_keys]. If it is not set, or set to 0, the
+#   value will be calculated at startup from the size of [validator_list_keys],
+#   where the calculation is:
+#
+#     threshold = size(validator_list_keys) < 3
+#       ? 1
+#       : floor(size(validator_list_keys) / 2) + 1
+
+[validator_list_threshold]
+0

cmake/CodeCoverage.cmake

@@ -98,6 +98,9 @@
 # 2024-04-03, Bronek Kozicki
 #     - add support for output formats: jacoco, clover, lcov
 #
+# 2025-05-12, Jingchen Wu
+#     - add -fprofile-update=atomic to ensure atomic profile generation
+#
 # USAGE:
 #
 # 1. Copy this file into your cmake modules path.
@@ -200,15 +203,27 @@ set(COVERAGE_COMPILER_FLAGS "-g --coverage"
     CACHE INTERNAL "")
 if(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang)")
     include(CheckCXXCompilerFlag)
+    include(CheckCCompilerFlag)
+
     check_cxx_compiler_flag(-fprofile-abs-path HAVE_cxx_fprofile_abs_path)
     if(HAVE_cxx_fprofile_abs_path)
         set(COVERAGE_CXX_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
     endif()
-    include(CheckCCompilerFlag)
+
     check_c_compiler_flag(-fprofile-abs-path HAVE_c_fprofile_abs_path)
     if(HAVE_c_fprofile_abs_path)
         set(COVERAGE_C_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
     endif()
+
+    check_cxx_compiler_flag(-fprofile-update HAVE_cxx_fprofile_update)
+    if(HAVE_cxx_fprofile_update)
+        set(COVERAGE_CXX_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS}  -fprofile-update=atomic")
+    endif()
+
+    check_c_compiler_flag(-fprofile-update HAVE_c_fprofile_update)
+    if(HAVE_c_fprofile_update)
+        set(COVERAGE_C_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-update=atomic")
+    endif()
 endif()
 
 set(CMAKE_Fortran_FLAGS_COVERAGE

cmake/RippledCompiler.cmake

@@ -120,7 +120,7 @@ else ()
   target_link_libraries (common
     INTERFACE
       -rdynamic
-      $<$<BOOL:${is_linux}>:-Wl,-z,relro,-z,now>
+      $<$<BOOL:${is_linux}>:-Wl,-z,relro,-z,now,--build-id>
       # link to static libc/c++ iff:
       #   * static option set and
       #   * NOT APPLE (AppleClang does not support static libc/c++) and
@@ -131,6 +131,17 @@ else ()
       >)
 endif ()
 
+# Antithesis instrumentation will only be built and deployed using machines running Linux.
+if (voidstar)
+  if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message(FATAL_ERROR "Antithesis instrumentation requires Debug build type, aborting...")
+  elseif (NOT is_linux)
+    message(FATAL_ERROR "Antithesis instrumentation requires Linux, aborting...")
+  elseif (NOT (is_clang AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 16.0))
+    message(FATAL_ERROR "Antithesis instrumentation requires Clang version 16 or later, aborting...")
+  endif ()
+endif ()
+
 if (use_mold)
   # use mold linker if available
   execute_process (

cmake/RippledCore.cmake

@@ -9,6 +9,7 @@ include(target_protobuf_sources)
 # define a bunch of `static const` variables with the same names,
 # so we just build them as a separate library.
 add_library(xrpl.libpb)
+set_target_properties(xrpl.libpb PROPERTIES UNITY_BUILD OFF)
 target_protobuf_sources(xrpl.libpb xrpl/proto
   LANGUAGE cpp
   IMPORT_DIRS include/xrpl/proto
@@ -47,37 +48,11 @@ target_link_libraries(xrpl.libpb
     gRPC::grpc++
 )
 
-add_library(xrpl.libxrpl)
-set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl)
-if(unity)
-  set_target_properties(xrpl.libxrpl PROPERTIES UNITY_BUILD ON)
-endif()
+# TODO: Clean up the number of library targets later.
+add_library(xrpl.imports.main INTERFACE)
 
-add_library(xrpl::libxrpl ALIAS xrpl.libxrpl)
-
-file(GLOB_RECURSE sources CONFIGURE_DEPENDS
-  "${CMAKE_CURRENT_SOURCE_DIR}/src/libxrpl/*.cpp"
-)
-target_sources(xrpl.libxrpl PRIVATE ${sources})
-
-target_include_directories(xrpl.libxrpl
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:include>)
-
-target_compile_definitions(xrpl.libxrpl
-  PUBLIC
-    BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
-    BOOST_CONTAINER_FWD_BAD_DEQUE
-    HAS_UNCAUGHT_EXCEPTIONS=1)
-
-target_compile_options(xrpl.libxrpl
-  PUBLIC
-    $<$<BOOL:${is_gcc}>:-Wno-maybe-uninitialized>
-)
-
-target_link_libraries(xrpl.libxrpl
-  PUBLIC
+target_link_libraries(xrpl.imports.main
+  INTERFACE
     LibArchive::LibArchive
     OpenSSL::Crypto
     Ripple::boost
@@ -89,58 +64,131 @@ target_link_libraries(xrpl.libxrpl
     secp256k1::secp256k1
     xrpl.libpb
     xxHash::xxhash
+    $<$<BOOL:${voidstar}>:antithesis-sdk-cpp>
 )
 
-add_executable(rippled)
-if(unity)
-  set_target_properties(rippled PROPERTIES UNITY_BUILD ON)
-endif()
-if(tests)
-  target_compile_definitions(rippled PUBLIC ENABLE_TESTS)
-endif()
-target_include_directories(rippled
-  PRIVATE
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+include(add_module)
+include(target_link_modules)
+
+# Level 01
+add_module(xrpl beast)
+target_link_libraries(xrpl.libxrpl.beast PUBLIC
+  xrpl.imports.main
+  xrpl.libpb
 )
 
+# Level 02
+add_module(xrpl basics)
+target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast)
+
+# Level 03
+add_module(xrpl json)
+target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics)
+
+add_module(xrpl crypto)
+target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics)
+
+# Level 04
+add_module(xrpl protocol)
+target_link_libraries(xrpl.libxrpl.protocol PUBLIC
+  xrpl.libxrpl.crypto
+  xrpl.libxrpl.json
+)
+
+# Level 05
+add_module(xrpl resource)
+target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol)
+
+add_module(xrpl server)
+target_link_libraries(xrpl.libxrpl.server PUBLIC xrpl.libxrpl.protocol)
+
+
+add_library(xrpl.libxrpl)
+set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl)
+
+add_library(xrpl::libxrpl ALIAS xrpl.libxrpl)
+
 file(GLOB_RECURSE sources CONFIGURE_DEPENDS
-  "${CMAKE_CURRENT_SOURCE_DIR}/src/xrpld/*.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/src/libxrpl/*.cpp"
 )
-target_sources(rippled PRIVATE ${sources})
+target_sources(xrpl.libxrpl PRIVATE ${sources})
+
+target_link_modules(xrpl PUBLIC
+  basics
+  beast
+  crypto
+  json
+  protocol
+  resource
+  server
+)
+
+# All headers in libxrpl are in modules.
+# Uncomment this stanza if you have not yet moved new headers into a module.
+# target_include_directories(xrpl.libxrpl
+#   PRIVATE
+#     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+#   PUBLIC
+#     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+#     $<INSTALL_INTERFACE:include>)
+
+if(xrpld)
+  add_executable(rippled)
+  if(tests)
+    target_compile_definitions(rippled PUBLIC ENABLE_TESTS)
+    target_compile_definitions(rippled PRIVATE
+                                       UNIT_TEST_REFERENCE_FEE=${UNIT_TEST_REFERENCE_FEE}
+    )
+  endif()
+  target_include_directories(rippled
+    PRIVATE
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+  )
 
-if(tests)
   file(GLOB_RECURSE sources CONFIGURE_DEPENDS
-    "${CMAKE_CURRENT_SOURCE_DIR}/src/test/*.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/xrpld/*.cpp"
   )
   target_sources(rippled PRIVATE ${sources})
-endif()
 
-target_link_libraries(rippled
-  Ripple::boost
-  Ripple::opts
-  Ripple::libs
-  xrpl.libxrpl
-)
-exclude_if_included(rippled)
-# define a macro for tests that might need to
-# be exluded or run differently in CI environment
-if(is_ci)
-  target_compile_definitions(rippled PRIVATE RIPPLED_RUNNING_IN_CI)
-endif ()
+  if(tests)
+    file(GLOB_RECURSE sources CONFIGURE_DEPENDS
+      "${CMAKE_CURRENT_SOURCE_DIR}/src/test/*.cpp"
+    )
+    target_sources(rippled PRIVATE ${sources})
+  endif()
 
-if(reporting)
-  set(suffix -reporting)
-  set_target_properties(rippled PROPERTIES OUTPUT_NAME rippled-reporting)
-  get_target_property(BIN_NAME rippled OUTPUT_NAME)
-  message(STATUS "Reporting mode build: rippled renamed ${BIN_NAME}")
-  target_compile_definitions(rippled PRIVATE RIPPLED_REPORTING)
-endif()
+  target_link_libraries(rippled
+    Ripple::boost
+    Ripple::opts
+    Ripple::libs
+    xrpl.libxrpl
+  )
+  exclude_if_included(rippled)
+  # define a macro for tests that might need to
+  # be exluded or run differently in CI environment
+  if(is_ci)
+    target_compile_definitions(rippled PRIVATE RIPPLED_RUNNING_IN_CI)
+  endif ()
 
-# any files that don't play well with unity should be added here
-if(tests)
-  set_source_files_properties(
-    # these two seem to produce conflicts in beast teardown template methods
-    src/test/rpc/ValidatorRPC_test.cpp
-    src/test/rpc/ShardArchiveHandler_test.cpp
-    PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
+  if(voidstar)
+    target_compile_options(rippled
+      PRIVATE
+        -fsanitize-coverage=trace-pc-guard
+    )
+    # rippled requires access to antithesis-sdk-cpp implementation file
+    # antithesis_instrumentation.h, which is not exported as INTERFACE
+    target_include_directories(rippled
+      PRIVATE
+        ${CMAKE_SOURCE_DIR}/external/antithesis-sdk
+    )
+  endif()
+
+  # any files that don't play well with unity should be added here
+  if(tests)
+    set_source_files_properties(
+      # these two seem to produce conflicts in beast teardown template methods
+      src/test/rpc/ValidatorRPC_test.cpp
+      src/test/ledger/Invariants_test.cpp
+      PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
+  endif()
 endif()

cmake/RippledCov.cmake

@@ -33,6 +33,6 @@ setup_target_for_coverage_gcovr(
   FORMAT ${coverage_format}
   EXECUTABLE rippled
   EXECUTABLE_ARGS --unittest$<$<BOOL:${coverage_test}>:=${coverage_test}> --unittest-jobs ${coverage_test_parallelism} --quiet --unittest-log
-  EXCLUDE "src/test" "${CMAKE_BINARY_DIR}/proto_gen" "${CMAKE_BINARY_DIR}/proto_gen_grpc"
+  EXCLUDE "src/test" "include/xrpl/beast/test" "include/xrpl/beast/unit_test" "${CMAKE_BINARY_DIR}/pb-xrpl.libpb"
   DEPENDENCIES rippled
 )

cmake/RippledDocs.cmake

@@ -21,15 +21,17 @@ set(doxyfile "${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile")
 
 file(GLOB_RECURSE doxygen_input
   docs/*.md
-  src/ripple/*.h
-  src/ripple/*.cpp
-  src/ripple/*.md
-  src/test/*.h
-  src/test/*.md)
+  include/*.h
+  include/*.cpp
+  include/*.md
+  src/*.h
+  src/*.cpp
+  src/*.md
+  Builds/*.md
+  *.md)
 list(APPEND doxygen_input
-  README.md
-  RELEASENOTES.md
-  src/README.md)
+  external/README.md
+  )
 set(dependencies "${doxygen_input}" "${doxyfile}")
 
 function(verbose_find_path variable name)
@@ -51,9 +53,9 @@ set(download_script "${CMAKE_BINARY_DIR}/docs/download-cppreference.cmake")
 file(WRITE
   "${download_script}"
   "file(DOWNLOAD \
-    http://upload.cppreference.com/mwiki/images/b/b2/html_book_20190607.zip \
+    https://github.com/PeterFeicht/cppreference-doc/releases/download/v20250209/html-book-20250209.zip \
     ${CMAKE_BINARY_DIR}/docs/cppreference.zip \
-    EXPECTED_HASH MD5=82b3a612d7d35a83e3cb1195a63689ab \
+    EXPECTED_HASH MD5=bda585f72fbca4b817b29a3d5746567b \
   )\n \
   execute_process( \
     COMMAND \"${CMAKE_COMMAND}\" -E tar -xf cppreference.zip \

cmake/RippledInstall.cmake

@@ -2,14 +2,25 @@
    install stuff
 #]===================================================================]
 
+include(create_symbolic_link)
+
 install (
   TARGETS
     common
     opts
     ripple_syslibs
     ripple_boost
+    xrpl.imports.main
     xrpl.libpb
+    xrpl.libxrpl.basics
+    xrpl.libxrpl.beast
+    xrpl.libxrpl.crypto
+    xrpl.libxrpl.json
+    xrpl.libxrpl.protocol
+    xrpl.libxrpl.resource
+    xrpl.libxrpl.server
     xrpl.libxrpl
+    antithesis-sdk-cpp
   EXPORT RippleExports
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib
@@ -21,6 +32,13 @@ install(
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
 )
 
+install(CODE "
+  set(CMAKE_MODULE_PATH \"${CMAKE_MODULE_PATH}\")
+  include(create_symbolic_link)
+  create_symbolic_link(xrpl \
+    \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/ripple)
+")
+
 install (EXPORT RippleExports
   FILE RippleTargets.cmake
   NAMESPACE Ripple::
@@ -31,14 +49,9 @@ write_basic_package_version_file (
   VERSION ${rippled_version}
   COMPATIBILITY SameMajorVersion)
 
-if (is_root_project)
+if (is_root_project AND TARGET rippled)
   install (TARGETS rippled RUNTIME DESTINATION bin)
   set_target_properties(rippled PROPERTIES INSTALL_RPATH_USE_LINK_PATH ON)
-  install (
-    FILES
-      ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RippleConfig.cmake
-      ${CMAKE_CURRENT_BINARY_DIR}/RippleConfigVersion.cmake
-    DESTINATION lib/cmake/ripple)
   # sample configs should not overwrite existing files
   # install if-not-exists workaround as suggested by
   # https://cmake.org/Bug/view.php?id=12646
@@ -53,18 +66,16 @@ if (is_root_project)
     copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/rippled-example.cfg\" etc rippled.cfg)
     copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/validators-example.txt\" etc validators.txt)
   ")
+  install(CODE "
+    set(CMAKE_MODULE_PATH \"${CMAKE_MODULE_PATH}\")
+    include(create_symbolic_link)
+    create_symbolic_link(rippled${suffix} \
+      \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrpld${suffix})
+  ")
 endif ()
 
-if(NOT WIN32)
-  install(
-    CODE "file(CREATE_LINK xrpl \
-      \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/ripple SYMBOLIC)"
-  )
-endif()
-
-if(NOT WIN32)
-  install(
-    CODE "file(CREATE_LINK rippled${suffix} \
-      \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrpld${suffix} SYMBOLIC)"
-  )
-endif()
+install (
+  FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RippleConfig.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/RippleConfigVersion.cmake
+  DESTINATION lib/cmake/ripple)

cmake/RippledInterface.cmake

@@ -7,6 +7,9 @@ add_library (Ripple::opts ALIAS opts)
 target_compile_definitions (opts
   INTERFACE
     BOOST_ASIO_DISABLE_HANDLER_TYPE_REQUIREMENTS
+    BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
+    BOOST_CONTAINER_FWD_BAD_DEQUE
+    HAS_UNCAUGHT_EXCEPTIONS=1
     $<$<BOOL:${boost_show_deprecated}>:
       BOOST_ASIO_NO_DEPRECATED
       BOOST_FILESYSTEM_NO_DEPRECATED
@@ -18,10 +21,12 @@ target_compile_definitions (opts
     >
     $<$<BOOL:${beast_no_unit_test_inline}>:BEAST_NO_UNIT_TEST_INLINE=1>
     $<$<BOOL:${beast_disable_autolink}>:BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES=1>
-    $<$<BOOL:${single_io_service_thread}>:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>)
+    $<$<BOOL:${single_io_service_thread}>:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>
+    $<$<BOOL:${voidstar}>:ENABLE_VOIDSTAR>)
 target_compile_options (opts
   INTERFACE
     $<$<AND:$<BOOL:${is_gcc}>,$<COMPILE_LANGUAGE:CXX>>:-Wsuggest-override>
+    $<$<BOOL:${is_gcc}>:-Wno-maybe-uninitialized>
     $<$<BOOL:${perf}>:-fno-omit-frame-pointer>
     $<$<AND:$<BOOL:${is_gcc}>,$<BOOL:${coverage}>>:-g --coverage -fprofile-abs-path>
     $<$<AND:$<BOOL:${is_clang}>,$<BOOL:${coverage}>>:-g --coverage>

cmake/RippledSanity.cmake

@@ -2,16 +2,6 @@
    convenience variables and sanity checks
 #]===================================================================]
 
-include(ProcessorCount)
-
-if (NOT ep_procs)
-  ProcessorCount(ep_procs)
-  if (ep_procs GREATER 1)
-    # never use more than half of cores for EP builds
-    math (EXPR ep_procs "${ep_procs} / 2")
-    message (STATUS "Using ${ep_procs} cores for ExternalProject builds.")
-  endif ()
-endif ()
 get_property(is_multiconfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 
 set (CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE)

cmake/RippledSettings.cmake

@@ -8,15 +8,25 @@ ProcessorCount(PROCESSOR_COUNT)
 
 option(assert "Enables asserts, even in release builds" OFF)
 
-option(reporting "Build rippled with reporting mode enabled" OFF)
+option(xrpld "Build xrpld" ON)
 
 option(tests "Build tests" ON)
+if(tests)
+  # This setting allows making a separate workflow to test fees other than default 10
+  if(NOT UNIT_TEST_REFERENCE_FEE)
+    set(UNIT_TEST_REFERENCE_FEE "10" CACHE STRING "")
+  endif()
+endif()
 
-option(unity "Creates a build using UNITY support in cmake. This is the default" ON)
+option(unity "Creates a build using UNITY support in cmake." OFF)
 if(unity)
   if(NOT is_ci)
     set(CMAKE_UNITY_BUILD_BATCH_SIZE 15 CACHE STRING "")
   endif()
+  set(CMAKE_UNITY_BUILD ON CACHE BOOL "Do a unity build")
+endif()
+if(is_clang AND is_linux)
+  option(voidstar "Enable Antithesis instrumentation." OFF)
 endif()
 if(is_gcc OR is_clang)
   option(coverage "Generates coverage info." OFF)

cmake/add_module.cmake

@@ -0,0 +1,37 @@
+include(isolate_headers)
+
+# Create an OBJECT library target named
+#
+#   ${PROJECT_NAME}.lib${parent}.${name}
+#
+# with sources in src/lib${parent}/${name}
+# and headers in include/${parent}/${name}
+# that cannot include headers from other directories in include/
+# unless they come through linked libraries.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+function(add_module parent name)
+  set(target ${PROJECT_NAME}.lib${parent}.${name})
+  add_library(${target} OBJECT)
+  file(GLOB_RECURSE sources CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}/*.cpp"
+  )
+  target_sources(${target} PRIVATE ${sources})
+  target_include_directories(${target} PUBLIC
+    "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+  )
+  isolate_headers(
+    ${target}
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_SOURCE_DIR}/include/${parent}/${name}"
+    PUBLIC
+  )
+  isolate_headers(
+    ${target}
+    "${CMAKE_CURRENT_SOURCE_DIR}/src"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}"
+    PRIVATE
+  )
+endfunction()

cmake/create_symbolic_link.cmake

@@ -0,0 +1,20 @@
+# file(CREATE_SYMLINK) only works on Windows with administrator privileges.
+# https://stackoverflow.com/a/61244115/618906
+function(create_symbolic_link target link)
+  if(WIN32)
+    if(NOT IS_SYMLINK "${link}")
+      if(NOT IS_ABSOLUTE "${target}")
+        # Relative links work do not work on Windows.
+        set(target "${link}/../${target}")
+      endif()
+      file(TO_NATIVE_PATH "${target}" target)
+      file(TO_NATIVE_PATH "${link}" link)
+      execute_process(COMMAND cmd.exe /c mklink /J "${link}" "${target}")
+    endif()
+  else()
+    file(CREATE_LINK "${target}" "${link}" SYMBOLIC)
+  endif()
+  if(NOT IS_SYMLINK "${link}")
+    message(ERROR "failed to create symlink: <${link}>")
+  endif()
+endfunction()

cmake/deps/Boost.cmake

@@ -2,7 +2,6 @@ find_package(Boost 1.82 REQUIRED
   COMPONENTS
     chrono
     container
-    context
     coroutine
     date_time
     filesystem
@@ -24,7 +23,7 @@ endif()
 
 target_link_libraries(ripple_boost
   INTERFACE
-    Boost::boost
+    Boost::headers
     Boost::chrono
     Boost::container
     Boost::coroutine

cmake/isolate_headers.cmake

@@ -0,0 +1,48 @@
+include(create_symbolic_link)
+
+# Consider include directory B nested under prefix A:
+#
+#     /path/to/A/then/to/B/...
+#
+# Call C the relative path from A to B.
+# C is what we want to write in `#include` directives:
+#
+#     #include <then/to/B/...>
+#
+# Examples, all from the `jobqueue` module:
+#
+#   - Library public headers:
+#     B = /include/xrpl/jobqueue
+#     A = /include/
+#     C = xrpl/jobqueue
+#
+#   - Library private headers:
+#     B = /src/libxrpl/jobqueue
+#     A = /src/
+#     C = libxrpl/jobqueue
+#
+#   - Test private headers:
+#     B = /tests/jobqueue
+#     A = /
+#     C = tests/jobqueue
+#
+# To isolate headers from each other,
+# we want to create a symlink Y that points to B,
+# within a subdirectory X of the `CMAKE_BINARY_DIR`,
+# that has the same relative path C between X and Y,
+# and then add X as an include directory of the target,
+# sometimes `PUBLIC` and sometimes `PRIVATE`.
+# The Cs are all guaranteed to be unique.
+# We can guarantee a unique X per target by using
+# `${CMAKE_CURRENT_BINARY_DIR}/include/${target}`.
+#
+# isolate_headers(target A B scope)
+function(isolate_headers target A B scope)
+  file(RELATIVE_PATH C "${A}" "${B}")
+  set(X "${CMAKE_CURRENT_BINARY_DIR}/modules/${target}")
+  set(Y "${X}/${C}")
+  cmake_path(GET Y PARENT_PATH parent)
+  file(MAKE_DIRECTORY "${parent}")
+  create_symbolic_link("${B}" "${Y}")
+  target_include_directories(${target} ${scope} "$<BUILD_INTERFACE:${X}>")
+endfunction()

cmake/target_link_modules.cmake

@@ -0,0 +1,24 @@
+# Link a library to its modules (see: `add_module`)
+# and remove the module sources from the library's sources.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+# add_library(project.libparent)
+# target_link_modules(parent PUBLIC a b)
+function(target_link_modules parent scope)
+  set(library ${PROJECT_NAME}.lib${parent})
+  foreach(name ${ARGN})
+    set(module ${library}.${name})
+    get_target_property(sources ${library} SOURCES)
+    list(LENGTH sources before)
+    get_target_property(dupes ${module} SOURCES)
+    list(LENGTH dupes expected)
+    list(REMOVE_ITEM sources ${dupes})
+    list(LENGTH sources after)
+    math(EXPR actual "${before} - ${after}")
+    message(STATUS "${module} with ${expected} sources took ${actual} sources from ${library}")
+    set_target_properties(${library} PROPERTIES SOURCES "${sources}")
+    target_link_libraries(${library} ${scope} ${module})
+  endforeach()
+endfunction()

cmake/xrpl_add_test.cmake

@@ -0,0 +1,41 @@
+include(isolate_headers)
+
+function(xrpl_add_test name)
+  set(target ${PROJECT_NAME}.test.${name})
+
+  file(GLOB_RECURSE sources CONFIGURE_DEPENDS
+  "${CMAKE_CURRENT_SOURCE_DIR}/${name}/*.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/${name}.cpp"
+  )
+  add_executable(${target} EXCLUDE_FROM_ALL ${ARGN} ${sources})
+
+  isolate_headers(
+    ${target}
+    "${CMAKE_SOURCE_DIR}"
+    "${CMAKE_SOURCE_DIR}/tests/${name}"
+    PRIVATE
+  )
+
+  # Make sure the test isn't optimized away in unity builds
+  set_target_properties(${target} PROPERTIES
+    UNITY_BUILD_MODE GROUP
+    UNITY_BUILD_BATCH_SIZE 0)  # Adjust as needed
+
+  add_test(NAME ${target} COMMAND ${target})
+  set_tests_properties(
+    ${target} PROPERTIES
+    FIXTURES_REQUIRED ${target}_fixture
+  )
+
+  add_test(
+    NAME ${target}.build
+    COMMAND
+      ${CMAKE_COMMAND}
+      --build ${CMAKE_BINARY_DIR}
+      --config $<CONFIG>
+      --target ${target}
+  )
+  set_tests_properties(${target}.build PROPERTIES
+    FIXTURES_SETUP ${target}_fixture
+  )
+endfunction()

conan/profiles/default

@@ -0,0 +1,37 @@
+{% set os = detect_api.detect_os() %}
+{% set arch = detect_api.detect_arch() %}
+{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %}
+{% set compiler_version = version %}
+{% if os == "Linux" %}
+{% set compiler_version = detect_api.default_compiler_version(compiler, version) %}
+{% endif %}
+
+[settings]
+os={{ os }}
+arch={{ arch }}
+build_type=Debug
+compiler={{compiler}}
+compiler.version={{ compiler_version }}
+compiler.cppstd=20
+{% if os == "Windows" %}
+compiler.runtime=static
+{% else %}
+compiler.libcxx={{detect_api.detect_libcxx(compiler, version, compiler_exe)}}
+{% endif %}
+
+[conf]
+{% if compiler == "clang" and compiler_version >= 19 %}
+tools.build:cxxflags=['-Wno-missing-template-arg-list-after-template-kw']
+{% endif %}
+{% if compiler == "apple-clang" and compiler_version >= 17 %}
+tools.build:cxxflags=['-Wno-missing-template-arg-list-after-template-kw']
+{% endif %}
+{% if compiler == "clang" and compiler_version == 16 %}
+tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
+{% endif %}
+{% if compiler == "gcc" and compiler_version < 13 %}
+tools.build:cxxflags=['-Wno-restrict']
+{% endif %}
+
+[tool_requires]
+!cmake/*: cmake/[>=3 <4]

conanfile.py

@@ -1,4 +1,4 @@
-from conan import ConanFile
+from conan import ConanFile, __version__ as conan_version
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 import re
 
@@ -15,27 +15,29 @@ class Xrpl(ConanFile):
         'coverage': [True, False],
         'fPIC': [True, False],
         'jemalloc': [True, False],
-        'reporting': [True, False],
         'rocksdb': [True, False],
         'shared': [True, False],
         'static': [True, False],
         'tests': [True, False],
         'unity': [True, False],
+        'xrpld': [True, False],
     }
 
     requires = [
-        'date/3.0.1',
-        'grpc/1.50.1',
-        'libarchive/3.6.2',
-        'nudb/2.0.8',
-        'openssl/1.1.1u',
+        'grpc/1.72.0',
+        'libarchive/3.8.1',
+        'nudb/2.0.9',
+        'openssl/1.1.1w',
         'soci/4.0.3',
-        'xxhash/0.8.2',
-        'zlib/1.2.13',
+        'zlib/1.3.1',
+    ]
+
+    test_requires = [
+        'doctest/2.4.11',
     ]
 
     tool_requires = [
-        'protobuf/3.21.9',
+        'protobuf/6.30.1',
     ]
 
     default_options = {
@@ -43,15 +45,13 @@ class Xrpl(ConanFile):
         'coverage': False,
         'fPIC': True,
         'jemalloc': False,
-        'reporting': False,
         'rocksdb': True,
         'shared': False,
         'static': True,
-        'tests': True,
+        'tests': False,
         'unity': False,
+        'xrpld': False,
 
-        'cassandra-cpp-driver/*:shared': False,
-        'cassandra-cpp-driver/*:use_atomic': None,
         'date/*:header_only': True,
         'grpc/*:shared': False,
         'grpc/*:secure': True,
@@ -70,7 +70,6 @@ class Xrpl(ConanFile):
         'libarchive/*:with_pcreposix': False,
         'libarchive/*:with_xattr': False,
         'libarchive/*:with_zlib': False,
-        'libpq/*:shared': False,
         'lz4/*:shared': False,
         'openssl/*:shared': False,
         'protobuf/*:shared': False,
@@ -90,29 +89,31 @@ class Xrpl(ConanFile):
     }
 
     def set_version(self):
-        path = f'{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp'
-        regex = r'versionString\s?=\s?\"(.*)\"'
-        with open(path, 'r') as file:
-            matches = (re.search(regex, line) for line in file)
-            match = next(m for m in matches if m)
-            self.version = match.group(1)
+        if self.version is None:
+            path = f'{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp'
+            regex = r'versionString\s?=\s?\"(.*)\"'
+            with open(path, encoding='utf-8') as file:
+                matches = (re.search(regex, line) for line in file)
+                match = next(m for m in matches if m)
+                self.version = match.group(1)
 
     def configure(self):
         if self.settings.compiler == 'apple-clang':
             self.options['boost'].visibility = 'global'
 
     def requirements(self):
-        self.requires('boost/1.82.0', force=True)
-        self.requires('lz4/1.9.3', force=True)
-        self.requires('protobuf/3.21.9', force=True)
-        self.requires('sqlite3/3.42.0', force=True)
+        # Conan 2 requires transitive headers to be specified
+        transitive_headers_opt = {'transitive_headers': True} if conan_version.split('.')[0] == '2' else {}
+        self.requires('boost/1.83.0', force=True, **transitive_headers_opt)
+        self.requires('date/3.0.4', **transitive_headers_opt)
+        self.requires('lz4/1.10.0', force=True)
+        self.requires('protobuf/6.30.1', force=True)
+        self.requires('sqlite3/3.49.1', force=True)
         if self.options.jemalloc:
             self.requires('jemalloc/5.3.0')
-        if self.options.reporting:
-            self.requires('cassandra-cpp-driver/2.15.3')
-            self.requires('libpq/14.7')
         if self.options.rocksdb:
-            self.requires('rocksdb/6.29.5')
+            self.requires('rocksdb/9.7.3')
+        self.requires('xxhash/0.8.3', **transitive_headers_opt)
 
     exports_sources = (
         'CMakeLists.txt',
@@ -137,11 +138,11 @@ class Xrpl(ConanFile):
         tc.variables['assert'] = self.options.assertions
         tc.variables['coverage'] = self.options.coverage
         tc.variables['jemalloc'] = self.options.jemalloc
-        tc.variables['reporting'] = self.options.reporting
         tc.variables['rocksdb'] = self.options.rocksdb
         tc.variables['BUILD_SHARED_LIBS'] = self.options.shared
         tc.variables['static'] = self.options.static
         tc.variables['unity'] = self.options.unity
+        tc.variables['xrpld'] = self.options.xrpld
         tc.generate()
 
     def build(self):
@@ -167,7 +168,17 @@ class Xrpl(ConanFile):
         # `include/`, not `include/ripple/proto/`.
         libxrpl.includedirs = ['include', 'include/ripple/proto']
         libxrpl.requires = [
-            'boost::boost',
+            'boost::headers',
+            'boost::chrono',
+            'boost::container',
+            'boost::coroutine',
+            'boost::date_time',
+            'boost::filesystem',
+            'boost::json',
+            'boost::program_options',
+            'boost::regex',
+            'boost::system',
+            'boost::thread',
             'date::date',
             'grpc::grpc++',
             'libarchive::libarchive',

docs/Doxyfile

@@ -103,18 +103,17 @@ WARN_LOGFILE           =
 # Configuration options related to the input files
 #---------------------------------------------------------------------------
 INPUT                  = \
-    docs \
-    src/ripple \
-    src/test \
-    src/README.md \
-    README.md \
-    RELEASENOTES.md \
+    . \
 
 
 INPUT_ENCODING         = UTF-8
 FILE_PATTERNS          = *.h *.cpp *.md
 RECURSIVE              = YES
-EXCLUDE                =
+EXCLUDE                = \
+    .github \
+    external/ed25519-donna \
+    external/secp256k1 \
+
 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_PATTERNS       =
 EXCLUDE_SYMBOLS        =
@@ -130,7 +129,7 @@ INPUT_FILTER           =
 FILTER_PATTERNS        =
 FILTER_SOURCE_FILES    = NO
 FILTER_SOURCE_PATTERNS =
-USE_MDFILE_AS_MAINPAGE = src/README.md
+USE_MDFILE_AS_MAINPAGE = ./README.md
 
 #---------------------------------------------------------------------------
 # Configuration options related to source browsing

docs/build/environment.md

@@ -23,7 +23,7 @@ direction.
 
 ```
 apt update
-apt install --yes curl git libssl-dev python3.10-dev python3-pip make g++-11
+apt install --yes curl git libssl-dev pipx python3.10-dev python3-pip make g++-11 libprotobuf-dev protobuf-compiler
 
 curl --location --remote-name \
   "https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1.tar.gz"
@@ -35,7 +35,8 @@ make --jobs $(nproc)
 make install
 cd ..
 
-pip3 install 'conan<2'
+pipx install 'conan<2'
+pipx ensurepath
 ```
 
 [1]: https://github.com/thejohnfreeman/rippled-docker/blob/master/ubuntu-22.04/install.sh

docs/consensus.md

@@ -558,7 +558,7 @@ struct ConsensusResult
     ConsensusTimer roundTime;
 
     // Indicates state in which consensus ended.  Once in the accept phase
-    // will be either Yes or MovedOn
+    // will be either Yes or MovedOn or Expired
     ConsensusState state = ConsensusState::No;
 };
 

external/README.md

@@ -1,9 +1,12 @@
+# External Conan recipes
+
 The subdirectories in this directory contain either copies or Conan recipes
 of external libraries used by rippled.
 The Conan recipes include patches we have not yet pushed upstream.
 
 | Folder          | Upstream                                     | Description |
 |:----------------|:---------------------------------------------|:------------|
+| `antithesis-sdk`| [Project](https://github.com/antithesishq/antithesis-sdk-cpp/) | [Antithesis](https://antithesis.com/docs/using_antithesis/sdk/cpp/overview.html) SDK for C++ |
 | `ed25519-donna` | [Project](https://github.com/floodyberry/ed25519-donna) | [Ed25519](http://ed25519.cr.yp.to/) digital signatures |
 | `rocksdb`       | [Recipe](https://github.com/conan-io/conan-center-index/tree/master/recipes/rocksdb) | Fast key/value database. (Supports rotational disks better than NuDB.) |
 | `secp256k1`     | [Project](https://github.com/bitcoin-core/secp256k1)    | ECDSA digital signatures using the **secp256k1** curve |

external/antithesis-sdk/.clang-format

@@ -0,0 +1,3 @@
+---
+DisableFormat: true
+SortIncludes: false

external/antithesis-sdk/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.18)
+
+# Note, version set explicitly by rippled project
+project(antithesis-sdk-cpp VERSION 0.4.4 LANGUAGES CXX)
+
+add_library(antithesis-sdk-cpp INTERFACE antithesis_sdk.h)
+
+# Note, both sections below created by rippled project
+target_include_directories(antithesis-sdk-cpp INTERFACE
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+)
+
+install(
+  FILES antithesis_sdk.h
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)

external/antithesis-sdk/README.md

@@ -0,0 +1,8 @@
+# Antithesis C++ SDK
+
+This library provides methods for C++ programs to configure the [Antithesis](https://antithesis.com) platform. It contains three kinds of functionality:
+* Assertion macros that allow you to define test properties about your software or workload.
+* Randomness functions for requesting both structured and unstructured randomness from the Antithesis platform.
+* Lifecycle functions that inform the Antithesis environment that particular test phases or milestones have been reached.
+
+For general usage guidance see the [Antithesis C++ SDK Documentation](https://antithesis.com/docs/using_antithesis/sdk/cpp/overview/)

external/antithesis-sdk/antithesis_instrumentation.h

@@ -0,0 +1,113 @@
+#pragma once
+
+/*
+This header file enables code coverage instrumentation. It is distributed with the Antithesis C++ SDK.
+
+This header file can be used in both C and C++ programs. (The rest of the SDK works only for C++ programs.)
+
+You should include it in a single .cpp or .c file.
+
+The instructions (such as required compiler flags) and usage guidance are found at https://antithesis.com/docs/using_antithesis/sdk/cpp/overview/.
+*/
+
+#include <unistd.h>
+#include <string.h>
+#include <dlfcn.h>
+#include <stdint.h>
+#include <stdio.h>
+#ifndef __cplusplus
+#include <stdbool.h>
+#include <stddef.h>
+#endif
+
+// If the libvoidstar(determ) library is present, 
+// pass thru trace_pc_guard related callbacks to it
+typedef void (*trace_pc_guard_init_fn)(uint32_t *start, uint32_t *stop);
+typedef void (*trace_pc_guard_fn)(uint32_t *guard, uint64_t edge);
+
+static trace_pc_guard_init_fn trace_pc_guard_init = NULL;
+static trace_pc_guard_fn trace_pc_guard = NULL;
+static bool did_check_libvoidstar = false;
+static bool has_libvoidstar = false;
+
+static __attribute__((no_sanitize("coverage"))) void debug_message_out(const char *msg) {
+  (void)printf("%s\n", msg);
+  return;
+}
+
+extern
+#ifdef __cplusplus
+    "C"
+#endif
+__attribute__((no_sanitize("coverage"))) void antithesis_load_libvoidstar() {
+#ifdef __cplusplus
+    constexpr
+#endif
+    const char* LIB_PATH = "/usr/lib/libvoidstar.so";
+
+    if (did_check_libvoidstar) {
+      return;
+    }
+    debug_message_out("TRYING TO LOAD libvoidstar");
+    did_check_libvoidstar = true;
+    void* shared_lib = dlopen(LIB_PATH, RTLD_NOW);
+    if (!shared_lib) {
+        debug_message_out("Can not load the Antithesis native library");
+        return;
+    }
+
+    void* trace_pc_guard_init_sym = dlsym(shared_lib, "__sanitizer_cov_trace_pc_guard_init");
+    if (!trace_pc_guard_init_sym) {
+        debug_message_out("Can not forward calls to libvoidstar for __sanitizer_cov_trace_pc_guard_init");
+        return;
+    }
+
+    void* trace_pc_guard_sym = dlsym(shared_lib, "__sanitizer_cov_trace_pc_guard_internal");
+    if (!trace_pc_guard_sym) {
+        debug_message_out("Can not forward calls to libvoidstar for __sanitizer_cov_trace_pc_guard");
+        return;
+    }
+
+    trace_pc_guard_init = (trace_pc_guard_init_fn)(trace_pc_guard_init_sym);
+    trace_pc_guard = (trace_pc_guard_fn)(trace_pc_guard_sym);
+    has_libvoidstar = true;
+    debug_message_out("LOADED libvoidstar");
+}
+
+// The following symbols are indeed reserved identifiers, since we're implementing functions defined
+// in the compiler runtime. Not clear how to get Clang on board with that besides narrowly suppressing
+// the warning in this case. The sample code on the CoverageSanitizer documentation page fails this 
+// warning!
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreserved-identifier"
+extern 
+#ifdef __cplusplus
+    "C"
+#endif
+void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) {
+    debug_message_out("SDK forwarding to libvoidstar for __sanitizer_cov_trace_pc_guard_init()");
+    if (!did_check_libvoidstar) {
+        antithesis_load_libvoidstar();
+    }
+    if (has_libvoidstar) {
+        trace_pc_guard_init(start, stop);
+    }
+    return;
+}
+
+extern 
+#ifdef __cplusplus
+    "C"
+#endif
+void __sanitizer_cov_trace_pc_guard( uint32_t *guard ) {
+    if (has_libvoidstar) {
+        uint64_t edge = (uint64_t)(__builtin_return_address(0));
+        trace_pc_guard(guard, edge);
+    } else {
+        if (guard) {
+          *guard = 0;
+        }
+    }
+    return;
+}
+#pragma clang diagnostic pop

external/protobuf/conandata.yml

@@ -0,0 +1,54 @@
+sources:
+  "6.30.1":
+    url: "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.30.1.tar.gz"
+    sha256: "c97cc064278ef2b8c4da66c1f85613642ecbd5a0c4217c0defdf7ad1b3de9fa5"
+patches:
+  "6.30.1":
+    - patch_file: "patches/protobuf-6.30.1-change-empty-string.patch"
+      patch_description: "Change how we decide which empty string implementation to use"
+      patch_type: "backport"
+      patch_source: "https://github.com/protocolbuffers/protobuf/issues/20645"
+    - patch_file: "patches/protobuf-6.30.1-disable-fixed-string-MSVC.patch"
+      patch_description: "Disable the optimization for fixed_address_empty_string for MSVC"
+      patch_type: "backport"
+      patch_source: "https://github.com/protocolbuffers/protobuf/issues/21957"
+absl_deps:
+  # reference: https://github.com/protocolbuffers/protobuf/blob/main/cmake/abseil-cpp.cmake
+  "6.30.1":
+    - absl_absl_check
+    - absl_absl_log
+    - absl_algorithm
+    - absl_base
+    - absl_bind_front
+    - absl_bits
+    - absl_btree
+    - absl_cleanup
+    - absl_cord
+    - absl_core_headers
+    - absl_debugging
+    - absl_die_if_null
+    - absl_dynamic_annotations
+    - absl_flags
+    - absl_flat_hash_map
+    - absl_flat_hash_set
+    - absl_function_ref
+    - absl_hash
+    - absl_layout
+    - absl_log_initialize
+    - absl_log_globals
+    - absl_log_severity
+    - absl_memory
+    - absl_node_hash_map
+    - absl_node_hash_set
+    - absl_optional
+    - absl_random_distributions
+    - absl_random_random
+    - absl_span
+    - absl_status
+    - absl_statusor
+    - absl_strings
+    - absl_synchronization
+    - absl_time
+    - absl_type_traits
+    - absl_utility
+    - absl_variant

external/protobuf/conanfile.py

@@ -0,0 +1,330 @@
+from conan import ConanFile
+from conan.errors import ConanInvalidConfiguration
+from conan.tools.apple import is_apple_os
+from conan.tools.build import check_min_cppstd
+from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
+from conan.tools.files import copy, rename, get, apply_conandata_patches, export_conandata_patches, replace_in_file, rmdir, rm, save
+from conan.tools.microsoft import check_min_vs, msvc_runtime_flag, is_msvc, is_msvc_static_runtime
+from conan.tools.scm import Version
+
+import os
+
+required_conan_version = ">=1.53"
+
+
+class ProtobufConan(ConanFile):
+    name = "protobuf"
+    description = "Protocol Buffers - Google's data interchange format"
+    topics = ("protocol-buffers", "protocol-compiler", "serialization", "rpc", "protocol-compiler")
+    url = "https://github.com/conan-io/conan-center-index"
+    homepage = "https://github.com/protocolbuffers/protobuf"
+    license = "BSD-3-Clause"
+    package_type = "library"
+    settings = "os", "arch", "compiler", "build_type"
+    options = {
+        "shared": [True, False],
+        "fPIC": [True, False],
+        "with_zlib": [True, False],
+        "with_rtti": [True, False],
+        "lite": [True, False],
+        "upb": [True, False],
+        "debug_suffix": [True, False],
+    }
+    default_options = {
+        "shared": False,
+        "fPIC": True,
+        "with_zlib": True,
+        "with_rtti": True,
+        "lite": False,
+        "upb": False,
+        "debug_suffix": True,
+    }
+
+    short_paths = True
+
+    @property
+    def _is_clang_cl(self):
+        return self.settings.compiler == "clang" and self.settings.os == "Windows"
+
+    @property
+    def _is_clang_x86(self):
+        return self.settings.compiler == "clang" and self.settings.arch == "x86"
+
+    @property
+    def _protobuf_release(self):
+        current_ver = Version(self.version)
+        return Version(f"{current_ver.minor}.{current_ver.patch}")
+
+    def export_sources(self):
+        export_conandata_patches(self)
+        copy(self, "protobuf-conan-protoc-target.cmake", self.recipe_folder, os.path.join(self.export_sources_folder, "src"))
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            del self.options.fPIC
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+
+        if self._protobuf_release < "27.0":
+            self.options.rm_safe("upb")
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def requirements(self):
+        if self.options.with_zlib:
+            self.requires("zlib/[>=1.2.11 <2]")
+
+        if self._protobuf_release >= "22.0":
+            self.requires("abseil/[>=20230802.1 <=20250127.0]", transitive_headers=True)
+
+    @property
+    def _compilers_minimum_version(self):
+        return {
+            "gcc": "6",
+            "clang": "5",
+            "apple-clang": "10",
+            "Visual Studio": "15",
+            "msvc": "191",
+        }
+
+    def validate(self):
+        if self.options.shared and is_msvc_static_runtime(self):
+            raise ConanInvalidConfiguration("Protobuf can't be built with shared + MT(d) runtimes")
+
+        if is_msvc(self) and self._protobuf_release >= "22" and self.options.shared and \
+            not self.dependencies["abseil"].options.shared:
+            raise ConanInvalidConfiguration("When building protobuf as a shared library on Windows, "
+                                            "abseil needs to be a shared library too")
+        if self._protobuf_release >= "30.1":
+            check_min_cppstd(self, 17)
+        elif self._protobuf_release >= "22.0":
+            if self.settings.compiler.get_safe("cppstd"):
+                check_min_cppstd(self, 14)
+            else:
+                minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), None)
+                compiler_version = Version(self.settings.compiler.version)
+                if minimum_version and compiler_version < minimum_version:
+                    raise ConanInvalidConfiguration(
+                        f"{self.ref} requires C++14, which your compiler does not support.",
+                    )
+
+        check_min_vs(self, "190")
+
+        if self.settings.compiler == "clang":
+            if Version(self.settings.compiler.version) < "4":
+                raise ConanInvalidConfiguration(f"{self.ref} doesn't support clang < 4")
+
+        if "abseil" in self.dependencies.host:
+            abseil_cppstd = self.dependencies.host['abseil'].info.settings.compiler.cppstd
+            if abseil_cppstd != self.settings.compiler.cppstd:
+                raise ConanInvalidConfiguration(f"Protobuf and abseil must be built with the same compiler.cppstd setting")
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], strip_root=True)
+
+    def build_requirements(self):
+        if self._protobuf_release >= "30.1":
+            self.tool_requires("cmake/[>=3.16 <4]")
+
+    @property
+    def _cmake_install_base_path(self):
+        return os.path.join("lib", "cmake", "protobuf")
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        if self._protobuf_release >= "30.1":
+            tc.cache_variables["protobuf_LOCAL_DEPENDENCIES_ONLY"] = True
+        tc.cache_variables["CMAKE_INSTALL_CMAKEDIR"] = self._cmake_install_base_path.replace("\\", "/")
+        tc.cache_variables["protobuf_WITH_ZLIB"] = self.options.with_zlib
+        tc.cache_variables["protobuf_BUILD_TESTS"] = False
+        tc.cache_variables["protobuf_BUILD_PROTOC_BINARIES"] = self.settings.os != "tvOS"
+        if not self.options.debug_suffix:
+            tc.cache_variables["protobuf_DEBUG_POSTFIX"] = ""
+        tc.cache_variables["protobuf_BUILD_LIBPROTOC"] = self.settings.os != "tvOS"
+        tc.cache_variables["protobuf_DISABLE_RTTI"] = not self.options.with_rtti
+        tc.cache_variables["protobuf_BUILD_LIBUPB"] = self.options.get_safe("upb")
+        if self._protobuf_release >= "22.0":
+            tc.cache_variables["protobuf_ABSL_PROVIDER"] = "package"
+            if not self.settings.compiler.get_safe("cppstd") and self._protobuf_release >= "22.0":
+                tc.variables["CMAKE_CXX_STANDARD"] = 14
+        if is_msvc(self) or self._is_clang_cl:
+            runtime = self.settings.get_safe("compiler.runtime")
+            if runtime:
+                tc.cache_variables["protobuf_MSVC_STATIC_RUNTIME"] = runtime == "static"
+        if is_apple_os(self) and self.options.shared:
+            # Workaround against SIP on macOS for consumers while invoking protoc when protobuf lib is shared
+            tc.variables["CMAKE_INSTALL_RPATH"] = "@loader_path/../lib"
+
+        if self.settings.os == "Linux":
+            # Use RPATH instead of RUNPATH to help with specific case
+            # in the grpc recipe when grpc_cpp_plugin is run with protoc
+            # in the same build. RPATH ensures that the rpath in the binary
+            # is respected for transitive dependencies too
+            project_include = os.path.join(self.generators_folder, "protobuf_project_include.cmake")
+            save(self, project_include, "add_link_options(-Wl,--disable-new-dtags)")
+            tc.variables["CMAKE_PROJECT_INCLUDE"] = project_include
+            # Note: conan2 only could be:
+            # tc.extra_exelinkflags.append("-Wl,--disable-new-dtags")
+            # tc.extra_sharedlinkflags.append("-Wl,--disable-new-dtags")
+
+        tc.generate()
+
+        deps = CMakeDeps(self)
+        deps.generate()
+
+    def _patch_sources(self):
+        apply_conandata_patches(self)
+
+        if self._protobuf_release < "22.0":
+            # In older versions of protobuf, this file defines the `protobuf_generate` function
+            protobuf_config_cmake = os.path.join(self.source_folder, "cmake", "protobuf-config.cmake.in")
+            replace_in_file(self, protobuf_config_cmake, "@_protobuf_FIND_ZLIB@", "")
+            replace_in_file(self, protobuf_config_cmake,
+                "include(\"${CMAKE_CURRENT_LIST_DIR}/protobuf-targets.cmake\")",
+                ""
+            )
+
+        # Disable a potential warning in protobuf-module.cmake.in
+        # TODO: remove this patch? Is it really useful?
+        protobuf_module_cmake = os.path.join(self.source_folder, "cmake", "protobuf-module.cmake.in")
+        replace_in_file(self,
+            protobuf_module_cmake,
+            "if(DEFINED Protobuf_SRC_ROOT_FOLDER)",
+            "if(0)\nif(DEFINED Protobuf_SRC_ROOT_FOLDER)",
+        )
+        replace_in_file(self,
+            protobuf_module_cmake,
+            "# Define upper case versions of output variables",
+            "endif()",
+        )
+
+    def build(self):
+        self._patch_sources()
+        cmake = CMake(self)
+        cmake_root = "cmake" if Version(self.version) < "3.21" else None
+        cmake.configure(build_script_folder=cmake_root)
+        cmake.build()
+
+    def package(self):
+        copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
+        cmake = CMake(self)
+        cmake.install()
+        rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
+        rmdir(self, os.path.join(self.package_folder, "lib", "cmake", "utf8_range"))
+        if self._protobuf_release < "22.0":
+            rename(self, os.path.join(self.package_folder, self._cmake_install_base_path, "protobuf-config.cmake"),
+                      os.path.join(self.package_folder, self._cmake_install_base_path, "protobuf-generate.cmake"))
+
+        cmake_config_folder = os.path.join(self.package_folder, self._cmake_install_base_path)
+        rm(self, "protobuf-config*.cmake", folder=cmake_config_folder)
+        rm(self, "protobuf-targets*.cmake", folder=cmake_config_folder)
+        copy(self, "protobuf-conan-protoc-target.cmake", src=self.source_folder, dst=cmake_config_folder)
+
+        if not self.options.lite:
+            rm(self, "libprotobuf-lite*", os.path.join(self.package_folder, "lib"))
+            rm(self, "libprotobuf-lite*", os.path.join(self.package_folder, "bin"))
+
+    def package_info(self):
+        self.cpp_info.set_property("cmake_find_mode", "both")
+        self.cpp_info.set_property("cmake_module_file_name", "Protobuf")
+        self.cpp_info.set_property("cmake_file_name", "protobuf")
+        self.cpp_info.set_property("pkg_config_name", "protobuf_full_package") # unofficial, but required to avoid side effects (libprotobuf component "steals" the default global pkg_config name)
+
+        build_modules = [
+            os.path.join(self._cmake_install_base_path, "protobuf-generate.cmake"),
+            os.path.join(self._cmake_install_base_path, "protobuf-module.cmake"),
+            os.path.join(self._cmake_install_base_path, "protobuf-options.cmake"),
+            os.path.join(self._cmake_install_base_path, "protobuf-conan-protoc-target.cmake"),
+        ]
+        self.cpp_info.set_property("cmake_build_modules", build_modules)
+
+        lib_prefix = "lib" if (is_msvc(self) or self._is_clang_cl) else ""
+        lib_suffix = "d" if self.settings.build_type == "Debug" and self.options.debug_suffix else ""
+
+        if self._protobuf_release >= "22.0":
+            absl_deps = [f"abseil::{c}" for c in self.conan_data["absl_deps"][self.version]]
+
+        if self._protobuf_release >= "22.0" and (not self.options.shared or self.options.get_safe("upb")):
+            # utf8 libraries
+            # it's a private dependency and unconditionally built as a static library, should only
+            # be exposed when protobuf itself is static (or if upb is being built)
+            self.cpp_info.components["utf8_range"].set_property("cmake_target_name", "utf8_range::utf8_range")
+            self.cpp_info.components["utf8_validity"].set_property("cmake_target_name", "utf8_range::utf8_validity")
+            # https://github.com/protocolbuffers/protobuf/blob/0d815c5b74281f081c1ee4b431a4d5bbb1615c97/third_party/utf8_range/CMakeLists.txt#L24
+            if self._protobuf_release >= "30.1" and self.settings.os == "Windows":
+                self.cpp_info.components["utf8_range"].libs = ["libutf8_range"]
+                self.cpp_info.components["utf8_validity"].libs = ["libutf8_validity"]
+            else:
+                self.cpp_info.components["utf8_range"].libs = ["utf8_range"]
+                self.cpp_info.components["utf8_validity"].libs = ["utf8_validity"]
+            self.cpp_info.components["utf8_validity"].requires = ["abseil::absl_strings"]
+
+        if self.options.get_safe("upb"):
+            # upb libraries: note that these are unconditionally static
+            self.cpp_info.components["upb"].set_property("cmake_target_name", "protobuf::libupb")
+            self.cpp_info.components["upb"].libs = [lib_prefix + "upb" + lib_suffix]
+            self.cpp_info.components["upb"].requires = ["utf8_range"]
+
+        # libprotobuf
+        self.cpp_info.components["libprotobuf"].set_property("cmake_target_name", "protobuf::libprotobuf")
+        self.cpp_info.components["libprotobuf"].set_property("pkg_config_name", "protobuf")
+        self.cpp_info.components["libprotobuf"].builddirs.append(self._cmake_install_base_path)
+        self.cpp_info.components["libprotobuf"].libs = [lib_prefix + "protobuf" + lib_suffix]
+        if self.options.with_zlib:
+            self.cpp_info.components["libprotobuf"].requires = ["zlib::zlib"]
+        if self._protobuf_release >= "22.0":
+            self.cpp_info.components["libprotobuf"].requires.extend(absl_deps)
+            if not self.options.shared:
+                self.cpp_info.components["libprotobuf"].requires.extend(["utf8_validity"])
+
+        if self.settings.os in ["Linux", "FreeBSD"]:
+            self.cpp_info.components["libprotobuf"].system_libs.extend(["m", "pthread"])
+            if self._is_clang_x86 or "arm" in str(self.settings.arch):
+                self.cpp_info.components["libprotobuf"].system_libs.append("atomic")
+        if self.settings.os == "Android":
+            self.cpp_info.components["libprotobuf"].system_libs.append("log")
+        if self.settings.os == "Windows":
+            if self.options.shared:
+                self.cpp_info.components["libprotobuf"].defines = ["PROTOBUF_USE_DLLS"]
+
+        # libprotoc
+        if self.settings.os != "tvOS":
+            self.cpp_info.components["libprotoc"].set_property("cmake_target_name", "protobuf::libprotoc")
+            self.cpp_info.components["libprotoc"].libs = [lib_prefix + "protoc" + lib_suffix]
+            self.cpp_info.components["libprotoc"].requires = ["libprotobuf"]
+            if self._protobuf_release >= "22.0":
+                self.cpp_info.components["libprotoc"].requires.extend(absl_deps)
+
+        # libprotobuf-lite
+        if self.options.lite:
+            self.cpp_info.components["libprotobuf-lite"].set_property("cmake_target_name", "protobuf::libprotobuf-lite")
+            self.cpp_info.components["libprotobuf-lite"].set_property("pkg_config_name", "protobuf-lite")
+            self.cpp_info.components["libprotobuf-lite"].builddirs.append(self._cmake_install_base_path)
+            self.cpp_info.components["libprotobuf-lite"].libs = [lib_prefix + "protobuf-lite" + lib_suffix]
+            if self.settings.os in ["Linux", "FreeBSD"]:
+                self.cpp_info.components["libprotobuf-lite"].system_libs.extend(["m", "pthread"])
+                if self._is_clang_x86 or "arm" in str(self.settings.arch):
+                    self.cpp_info.components["libprotobuf-lite"].system_libs.append("atomic")
+            if self.settings.os == "Windows":
+                if self.options.shared:
+                    self.cpp_info.components["libprotobuf-lite"].defines = ["PROTOBUF_USE_DLLS"]
+            if self.settings.os == "Android":
+                self.cpp_info.components["libprotobuf-lite"].system_libs.append("log")
+            if self._protobuf_release >= "22.0":
+                self.cpp_info.components["libprotobuf-lite"].requires.extend(absl_deps)
+                if not self.options.shared:
+                    self.cpp_info.components["libprotobuf-lite"].requires.extend(["utf8_validity"])
+
+        # TODO: to remove in conan v2 once cmake_find_package* & pkg_config generators removed
+        self.cpp_info.filenames["cmake_find_package"] = "Protobuf"
+        self.cpp_info.filenames["cmake_find_package_multi"] = "protobuf"
+        self.cpp_info.names["pkg_config"] ="protobuf_full_package"
+        for generator in ["cmake_find_package", "cmake_find_package_multi"]:
+            self.cpp_info.components["libprotobuf"].build_modules[generator] = build_modules
+        if self.options.lite:
+            for generator in ["cmake_find_package", "cmake_find_package_multi"]:
+                self.cpp_info.components["libprotobuf-lite"].build_modules[generator] = build_modules
+        self.env_info.PATH.append(os.path.join(self.package_folder, "bin"))

external/protobuf/patches/protobuf-6.30.1-change-empty-string.patch

@@ -0,0 +1,70 @@
+diff --git a/src/google/protobuf/port.cc b/src/google/protobuf/port.cc
+index af668e9e2..d60c2b89f 100644
+--- a/src/google/protobuf/port.cc
++++ b/src/google/protobuf/port.cc
+@@ -97,14 +97,9 @@ void RealDebugCounter::Register(absl::string_view name) {
+   }
+ }
+ 
+-#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L
+-PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT const GlobalEmptyString
+-    fixed_address_empty_string{};
+-#else
+ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT
+     PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 GlobalEmptyString
+         fixed_address_empty_string{};
+-#endif
+ 
+ }  // namespace internal
+ }  // namespace protobuf
+diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h
+index 5f9e909a0..386ecc02a 100644
+--- a/src/google/protobuf/port.h
++++ b/src/google/protobuf/port.h
+@@ -494,20 +494,27 @@ class NoopDebugCounter {
+ // Default empty string object. Don't use this directly. Instead, call
+ // GetEmptyString() to get the reference. This empty string is aligned with a
+ // minimum alignment of 8 bytes to match the requirement of ArenaStringPtr.
+-#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L
++
+ // Take advantage of C++20 constexpr support in std::string.
+-class alignas(8) GlobalEmptyString {
++class alignas(8) GlobalEmptyStringConstexpr {
+  public:
+   const std::string& get() const { return value_; }
+   // Nothing to init, or destroy.
+   std::string* Init() const { return nullptr; }
+ 
++  template <typename T = std::string, bool = (T(), true)>
++  static constexpr std::true_type HasConstexprDefaultConstructor(int) {
++    return {};
++  }
++  static constexpr std::false_type HasConstexprDefaultConstructor(char) {
++    return {};
++  }
++
+  private:
+   std::string value_;
+ };
+-PROTOBUF_EXPORT extern const GlobalEmptyString fixed_address_empty_string;
+-#else
+-class alignas(8) GlobalEmptyString {
++
++class alignas(8) GlobalEmptyStringDynamicInit {
+  public:
+   const std::string& get() const {
+     return *reinterpret_cast<const std::string*>(internal::Launder(buffer_));
+@@ -519,8 +526,12 @@ class alignas(8) GlobalEmptyString {
+  private:
+   alignas(std::string) char buffer_[sizeof(std::string)];
+ };
++
++using GlobalEmptyString = std::conditional_t<
++    GlobalEmptyStringConstexpr::HasConstexprDefaultConstructor(0),
++    const GlobalEmptyStringConstexpr, GlobalEmptyStringDynamicInit>;
++
+ PROTOBUF_EXPORT extern GlobalEmptyString fixed_address_empty_string;
+-#endif
+ 
+ }  // namespace internal
+ }  // namespace protobuf

external/protobuf/patches/protobuf-6.30.1-disable-fixed-string-MSVC.patch

@@ -0,0 +1,21 @@
+diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h
+index 386ecc02a..32d260c42 100644
+--- a/src/google/protobuf/port.h
++++ b/src/google/protobuf/port.h
+@@ -502,10 +502,16 @@ class alignas(8) GlobalEmptyStringConstexpr {
+   // Nothing to init, or destroy.
+   std::string* Init() const { return nullptr; }
+ 
++  // Disable the optimization for MSVC.
++  // There are some builds where the default constructed string can't be used as
++  // `constinit` even though the constructor is `constexpr` and can be used
++  // during constant evaluation.
++#if !defined(_MSC_VER)
+   template <typename T = std::string, bool = (T(), true)>
+   static constexpr std::true_type HasConstexprDefaultConstructor(int) {
+     return {};
+   }
++#endif
+   static constexpr std::false_type HasConstexprDefaultConstructor(char) {
+     return {};
+   }

external/protobuf/protobuf-conan-protoc-target.cmake

@@ -0,0 +1,25 @@
+if(NOT TARGET protobuf::protoc)
+    # Locate protoc executable
+    ## Workaround for legacy "cmake" generator in case of cross-build
+    if(CMAKE_CROSSCOMPILING)
+        find_program(PROTOC_PROGRAM NAMES protoc PATHS ENV PATH NO_DEFAULT_PATH)
+    endif()
+    ## And here this will work fine with "CMakeToolchain" (for native & cross-build)
+    ## and legacy "cmake" generator in case of native build
+    if(NOT PROTOC_PROGRAM)
+        find_program(PROTOC_PROGRAM NAMES protoc)
+    endif()
+    ## Last resort: we search in package folder directly
+    if(NOT PROTOC_PROGRAM)
+        set(PROTOC_PROGRAM "${CMAKE_CURRENT_LIST_DIR}/../../../bin/protoc${CMAKE_EXECUTABLE_SUFFIX}")
+    endif()
+    get_filename_component(PROTOC_PROGRAM "${PROTOC_PROGRAM}" ABSOLUTE)
+
+    # Give opportunity to users to provide an external protoc executable
+    # (this is a feature of official FindProtobuf.cmake)
+    set(Protobuf_PROTOC_EXECUTABLE ${PROTOC_PROGRAM} CACHE FILEPATH "The protoc compiler")
+
+    # Create executable imported target protobuf::protoc
+    add_executable(protobuf::protoc IMPORTED)
+    set_property(TARGET protobuf::protoc PROPERTY IMPORTED_LOCATION ${Protobuf_PROTOC_EXECUTABLE})
+endif()

external/protobuf/test_package/CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.15)
+
+project(test_package LANGUAGES CXX)
+
+find_package(protobuf CONFIG REQUIRED)
+
+add_executable(${PROJECT_NAME} test_package.cpp)
+
+if(CONAN_TEST_USE_CXXSTD_14)
+    target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14)
+else()
+    target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_11)
+endif()
+
+if (protobuf_LITE)
+    target_link_libraries(${PROJECT_NAME} PRIVATE protobuf::libprotobuf-lite)
+    target_compile_definitions(${PROJECT_NAME} PRIVATE CONANTEST_PROTOBUF_LITE=1)
+else()
+    target_link_libraries(${PROJECT_NAME} PRIVATE protobuf::libprotobuf)
+endif()
+
+if(NOT TARGET protobuf::protoc)
+    message(FATAL_ERROR "protoc executable should have been defined as part of find_package(protobuf)")
+endif()
+
+if(NOT COMMAND protobuf_generate)
+    message(FATAL_ERROR "protobuf_generate should have been defined as part of find_package(protobuf)")
+endif()

external/protobuf/test_package/addressbook.proto

@@ -0,0 +1,64 @@
+// See README.txt for information and build instructions.
+//
+// Note: START and END tags are used in comments to define sections used in
+// tutorials.  They are not part of the syntax for Protocol Buffers.
+//
+// To get an in-depth walkthrough of this file and the related examples, see:
+// https://developers.google.com/protocol-buffers/docs/tutorials
+
+// [START declaration]
+syntax = "proto3";
+package tutorial;
+
+import "google/protobuf/timestamp.proto";
+// [END declaration]
+
+// [START java_declaration]
+option java_package = "com.example.tutorial";
+option java_outer_classname = "AddressBookProtos";
+// [END java_declaration]
+
+// [START csharp_declaration]
+option csharp_namespace = "Google.Protobuf.Examples.AddressBook";
+// [END csharp_declaration]
+
+// [START messages]
+message Person {
+  string name = 1;
+  int32 id = 2;  // Unique ID number for this person.
+  string email = 3;
+
+  enum PhoneType {
+    MOBILE = 0;
+    HOME = 1;
+    WORK = 2;
+  }
+
+  message PhoneNumber {
+    string number = 1;
+    PhoneType type = 2;
+  }
+
+  repeated PhoneNumber phones = 4;
+
+  google.protobuf.Timestamp last_updated = 5;
+}
+
+// Our address book file is just one of these.
+message AddressBook {
+  repeated Person people = 1;
+}
+// [END messages]
+
+// Artibrary message with a GID type, to cause 
+// the .pb.h file to define `GID_MAX`, which
+// may conflict with a macro defined by system
+// headers on macOS.
+message FooBar {
+  enum GID {
+    FOO = 0;
+  }
+  enum UID {
+    BAR = 0;
+  }
+}

external/protobuf/test_package/conanfile.py

@@ -0,0 +1,40 @@
+from conan import ConanFile
+from conan.tools.build import can_run
+from conan.tools.cmake import cmake_layout, CMake, CMakeToolchain
+from conan.tools.scm import Version
+import os
+
+
+class TestPackageConan(ConanFile):
+    settings = "os", "arch", "compiler", "build_type"
+    generators = "CMakeDeps", "VirtualBuildEnv", "VirtualRunEnv"
+    test_type = "explicit"
+
+    def layout(self):
+        cmake_layout(self)
+
+    def requirements(self):
+        # note `run=True` so that the runenv can find protoc
+        self.requires(self.tested_reference_str, run=True)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.cache_variables["protobuf_LITE"] = self.dependencies[self.tested_reference_str].options.lite
+        protobuf_version = Version(self.dependencies[self.tested_reference_str].ref.version)
+        tc.cache_variables["CONAN_TEST_USE_CXXSTD_14"] = protobuf_version >= "3.22"
+        tc.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def test(self):
+        if can_run(self):
+            bin_path = os.path.join(self.cpp.build.bindirs[0], "test_package")
+            self.run(bin_path, env="conanrun")
+
+            # Invoke protoc in the same way CMake would
+            self.run(f"protoc --proto_path={self.source_folder} --cpp_out={self.build_folder} {self.source_folder}/addressbook.proto", env="conanrun")
+            assert os.path.exists(os.path.join(self.build_folder,"addressbook.pb.cc"))
+            assert os.path.exists(os.path.join(self.build_folder,"addressbook.pb.h"))

external/protobuf/test_package/test_package.cpp

@@ -0,0 +1,23 @@
+#include <cstdlib>
+#include <iostream>
+
+#if !defined(CONANTEST_PROTOBUF_LITE)
+#include <google/protobuf/timestamp.pb.h>
+#include <google/protobuf/util/time_util.h>
+#else 
+#include <google/protobuf/message_lite.h>
+#endif
+
+int main()
+{
+
+#if !defined(CONANTEST_PROTOBUF_LITE)
+	google::protobuf::Timestamp ts;
+	google::protobuf::util::TimeUtil::FromString("1972-01-01T10:00:20.021Z", &ts);
+	const auto nanoseconds = ts.nanos();
+	std::cout << "1972-01-01T10:00:20.021Z in nanoseconds: " << nanoseconds << "\n";
+#else
+	google::protobuf::ShutdownProtobufLibrary();
+#endif
+	return EXIT_SUCCESS;
+}

external/rocksdb/conandata.yml

@@ -1,27 +1,12 @@
 sources:
-  "6.29.5":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz"
-    sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0"
-  "6.27.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz"
-    sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58"
-  "6.20.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz"
-    sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca"
-  "8.8.1":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz"
-    sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e"
+  "9.7.3":
+    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v9.7.3.tar.gz"
+    sha256: "acfabb989cbfb5b5c4d23214819b059638193ec33dad2d88373c46448d16d38b"
 patches:
-  "6.29.5":
-    - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
-      patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"
-    - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch"
+  "9.7.3":
+    - patch_file: "patches/9.x.x-0001-exclude-thirdparty.patch"
       patch_description: "Do not include thirdparty.inc"
       patch_type: "portability"
-  "6.27.3":
-    - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
+    - patch_file: "patches/9.7.3-0001-memory-leak.patch"
+      patch_description: "Fix a leak of obsolete blob files left open until DB::Close()"
       patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"

external/rocksdb/conanfile.py

@@ -15,10 +15,10 @@ required_conan_version = ">=1.53.0"
 
 class RocksDBConan(ConanFile):
     name = "rocksdb"
-    homepage = "https://github.com/facebook/rocksdb"
+    description = "A library that provides an embeddable, persistent key-value store for fast storage"
     license = ("GPL-2.0-only", "Apache-2.0")
     url = "https://github.com/conan-io/conan-center-index"
-    description = "A library that provides an embeddable, persistent key-value store for fast storage"
+    homepage = "https://github.com/facebook/rocksdb"
     topics = ("database", "leveldb", "facebook", "key-value")
     package_type = "library"
     settings = "os", "arch", "compiler", "build_type"
@@ -58,12 +58,12 @@ class RocksDBConan(ConanFile):
     @property
     def _compilers_minimum_version(self):
         return {} if self._min_cppstd == "11" else {
-                "apple-clang": "10",
-                "clang": "7",
-                "gcc": "7",
-                "msvc": "191",
-                "Visual Studio": "15",
-            }
+            "apple-clang": "10",
+            "clang": "7",
+            "gcc": "7",
+            "msvc": "191",
+            "Visual Studio": "15",
+        }
 
     def export_sources(self):
         export_conandata_patches(self)
@@ -89,13 +89,13 @@ class RocksDBConan(ConanFile):
         if self.options.with_snappy:
             self.requires("snappy/1.1.10")
         if self.options.with_lz4:
-            self.requires("lz4/1.9.4")
+            self.requires("lz4/1.10.0")
         if self.options.with_zlib:
             self.requires("zlib/[>=1.2.11 <2]")
         if self.options.with_zstd:
-            self.requires("zstd/1.5.5")
+            self.requires("zstd/1.5.6")
         if self.options.get_safe("with_tbb"):
-            self.requires("onetbb/2021.10.0")
+            self.requires("onetbb/2021.12.0")
         if self.options.with_jemalloc:
             self.requires("jemalloc/5.3.0")
 
@@ -115,9 +115,9 @@ class RocksDBConan(ConanFile):
         check_min_vs(self, "191")
 
         if self.version == "6.20.3" and \
-           self.settings.os == "Linux" and \
-           self.settings.compiler == "gcc" and \
-           Version(self.settings.compiler.version) < "5":
+                self.settings.os == "Linux" and \
+                self.settings.compiler == "gcc" and \
+                Version(self.settings.compiler.version) < "5":
             raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522
 
     def source(self):
@@ -163,6 +163,8 @@ class RocksDBConan(ConanFile):
         if self.options.with_jemalloc:
             deps.set_property("jemalloc", "cmake_file_name", "JeMalloc")
             deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc")
+        if self.options.with_zstd:
+            deps.set_property("zstd", "cmake_target_name", "zstd::zstd")
         deps.generate()
 
     def build(self):

external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch

@@ -1,30 +0,0 @@
---- a/include/rocksdb/utilities/checkpoint.h
-+++ b/include/rocksdb/utilities/checkpoint.h
-@@ -8,6 +8,7 @@
- #pragma once
- #ifndef ROCKSDB_LITE
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- #include "rocksdb/status.h"
---- a/table/block_based/data_block_hash_index.h
-+++ b/table/block_based/data_block_hash_index.h
-@@ -5,6 +5,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- 
---- a/util/string_util.h
-+++ b/util/string_util.h
-@@ -6,6 +6,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <sstream>
- #include <string>
- #include <unordered_map>

external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch

@@ -1,16 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index ec59d4491..35577c998 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -101 +100,0 @@ if(MSVC)
--  option(WITH_GFLAGS "build with GFlags" OFF)
-@@ -103,2 +102,2 @@ if(MSVC)
--  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
--else()
-+endif()
-+
-@@ -117 +116 @@ else()
--  if(MINGW)
-+  if(MINGW OR MSVC)
-@@ -183 +181,0 @@ else()
--endif()

external/rocksdb/patches/9.7.3-0001-memory-leak.patch

@@ -0,0 +1,319 @@
+diff --git a/HISTORY.md b/HISTORY.md
+index 36d472229..05ad1a202 100644
+--- a/HISTORY.md
++++ b/HISTORY.md
+@@ -1,6 +1,10 @@
+ # Rocksdb Change Log
+ > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
+ 
++## 9.7.4 (10/31/2024)
++### Bug Fixes
++* Fix a leak of obsolete blob files left open until DB::Close(). This bug was introduced in version 9.4.0.
++
+ ## 9.7.3 (10/16/2024)
+ ### Behavior Changes
+ * OPTIONS file to be loaded by remote worker is now preserved so that it does not get purged by the primary host. A similar technique as how we are preserving new SST files from getting purged is used for this. min_options_file_numbers_ is tracked like pending_outputs_ is tracked.
+diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc
+index 5f340aadf..1b9faa238 100644
+--- a/db/blob/blob_file_cache.cc
++++ b/db/blob/blob_file_cache.cc
+@@ -42,6 +42,7 @@ Status BlobFileCache::GetBlobFileReader(
+   assert(blob_file_reader);
+   assert(blob_file_reader->IsEmpty());
+ 
++  // NOTE: sharing same Cache with table_cache
+   const Slice key = GetSliceForKey(&blob_file_number);
+ 
+   assert(cache_);
+@@ -98,4 +99,13 @@ Status BlobFileCache::GetBlobFileReader(
+   return Status::OK();
+ }
+ 
++void BlobFileCache::Evict(uint64_t blob_file_number) {
++  // NOTE: sharing same Cache with table_cache
++  const Slice key = GetSliceForKey(&blob_file_number);
++
++  assert(cache_);
++
++  cache_.get()->Erase(key);
++}
++
+ }  // namespace ROCKSDB_NAMESPACE
+diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h
+index 740e67ada..6858d012b 100644
+--- a/db/blob/blob_file_cache.h
++++ b/db/blob/blob_file_cache.h
+@@ -36,6 +36,15 @@ class BlobFileCache {
+                            uint64_t blob_file_number,
+                            CacheHandleGuard<BlobFileReader>* blob_file_reader);
+ 
++  // Called when a blob file is obsolete to ensure it is removed from the cache
++  // to avoid effectively leaking the open file and assicated memory
++  void Evict(uint64_t blob_file_number);
++
++  // Used to identify cache entries for blob files (not normally useful)
++  static const Cache::CacheItemHelper* GetHelper() {
++    return CacheInterface::GetBasicHelper();
++  }
++
+  private:
+   using CacheInterface =
+       BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
+diff --git a/db/column_family.h b/db/column_family.h
+index e4b7adde8..86637736a 100644
+--- a/db/column_family.h
++++ b/db/column_family.h
+@@ -401,6 +401,7 @@ class ColumnFamilyData {
+                          SequenceNumber earliest_seq);
+ 
+   TableCache* table_cache() const { return table_cache_.get(); }
++  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
+   BlobSource* blob_source() const { return blob_source_.get(); }
+ 
+   // See documentation in compaction_picker.h
+diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc
+index 261593423..06573ac2e 100644
+--- a/db/db_impl/db_impl.cc
++++ b/db/db_impl/db_impl.cc
+@@ -659,8 +659,9 @@ Status DBImpl::CloseHelper() {
+   // We need to release them before the block cache is destroyed. The block
+   // cache may be destroyed inside versions_.reset(), when column family data
+   // list is destroyed, so leaving handles in table cache after
+-  // versions_.reset() may cause issues.
+-  // Here we clean all unreferenced handles in table cache.
++  // versions_.reset() may cause issues. Here we clean all unreferenced handles
++  // in table cache, and (for certain builds/conditions) assert that no obsolete
++  // files are hanging around unreferenced (leak) in the table/blob file cache.
+   // Now we assume all user queries have finished, so only version set itself
+   // can possibly hold the blocks from block cache. After releasing unreferenced
+   // handles here, only handles held by version set left and inside
+@@ -668,6 +669,9 @@ Status DBImpl::CloseHelper() {
+   // time a handle is released, we erase it from the cache too. By doing that,
+   // we can guarantee that after versions_.reset(), table cache is empty
+   // so the cache can be safely destroyed.
++#ifndef NDEBUG
++  TEST_VerifyNoObsoleteFilesCached(/*db_mutex_already_held=*/true);
++#endif  // !NDEBUG
+   table_cache_->EraseUnRefEntries();
+ 
+   for (auto& txn_entry : recovered_transactions_) {
+@@ -3227,6 +3231,8 @@ Status DBImpl::MultiGetImpl(
+       s = Status::Aborted();
+       break;
+     }
++    // This could be a long-running operation
++    ROCKSDB_THREAD_YIELD_HOOK();
+   }
+ 
+   // Post processing (decrement reference counts and record statistics)
+diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h
+index 5e4fa310b..ccc0abfa7 100644
+--- a/db/db_impl/db_impl.h
++++ b/db/db_impl/db_impl.h
+@@ -1241,9 +1241,14 @@ class DBImpl : public DB {
+   static Status TEST_ValidateOptions(const DBOptions& db_options) {
+     return ValidateOptions(db_options);
+   }
+-
+ #endif  // NDEBUG
+ 
++  // In certain configurations, verify that the table/blob file cache only
++  // contains entries for live files, to check for effective leaks of open
++  // files. This can only be called when purging of obsolete files has
++  // "settled," such as during parts of DB Close().
++  void TEST_VerifyNoObsoleteFilesCached(bool db_mutex_already_held) const;
++
+   // persist stats to column family "_persistent_stats"
+   void PersistStats();
+ 
+diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc
+index 790a50d7a..67f5b4aaf 100644
+--- a/db/db_impl/db_impl_debug.cc
++++ b/db/db_impl/db_impl_debug.cc
+@@ -9,6 +9,7 @@
+ 
+ #ifndef NDEBUG
+ 
++#include "db/blob/blob_file_cache.h"
+ #include "db/column_family.h"
+ #include "db/db_impl/db_impl.h"
+ #include "db/error_handler.h"
+@@ -328,5 +329,49 @@ size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
+   InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
+   return EstimateInMemoryStatsHistorySize();
+ }
++
++void DBImpl::TEST_VerifyNoObsoleteFilesCached(
++    bool db_mutex_already_held) const {
++  // This check is somewhat expensive and obscure to make a part of every
++  // unit test in every build variety. Thus, we only enable it for ASAN builds.
++  if (!kMustFreeHeapAllocations) {
++    return;
++  }
++
++  std::optional<InstrumentedMutexLock> l;
++  if (db_mutex_already_held) {
++    mutex_.AssertHeld();
++  } else {
++    l.emplace(&mutex_);
++  }
++
++  std::vector<uint64_t> live_files;
++  for (auto cfd : *versions_->GetColumnFamilySet()) {
++    if (cfd->IsDropped()) {
++      continue;
++    }
++    // Sneakily add both SST and blob files to the same list
++    cfd->current()->AddLiveFiles(&live_files, &live_files);
++  }
++  std::sort(live_files.begin(), live_files.end());
++
++  auto fn = [&live_files](const Slice& key, Cache::ObjectPtr, size_t,
++                          const Cache::CacheItemHelper* helper) {
++    if (helper != BlobFileCache::GetHelper()) {
++      // Skip non-blob files for now
++      // FIXME: diagnose and fix the leaks of obsolete SST files revealed in
++      // unit tests.
++      return;
++    }
++    // See TableCache and BlobFileCache
++    assert(key.size() == sizeof(uint64_t));
++    uint64_t file_number;
++    GetUnaligned(reinterpret_cast<const uint64_t*>(key.data()), &file_number);
++    // Assert file is in sorted live_files
++    assert(
++        std::binary_search(live_files.begin(), live_files.end(), file_number));
++  };
++  table_cache_->ApplyToAllEntries(fn, {});
++}
+ }  // namespace ROCKSDB_NAMESPACE
+ #endif  // NDEBUG
+diff --git a/db/db_iter.cc b/db/db_iter.cc
+index e02586377..bf4749eb9 100644
+--- a/db/db_iter.cc
++++ b/db/db_iter.cc
+@@ -540,6 +540,8 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
+     } else {
+       iter_.Next();
+     }
++    // This could be a long-running operation due to tombstones, etc.
++    ROCKSDB_THREAD_YIELD_HOOK();
+   } while (iter_.Valid());
+ 
+   valid_ = false;
+diff --git a/db/table_cache.cc b/db/table_cache.cc
+index 71fc29c32..8a5be75e8 100644
+--- a/db/table_cache.cc
++++ b/db/table_cache.cc
+@@ -164,6 +164,7 @@ Status TableCache::GetTableReader(
+ }
+ 
+ Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&file_number);
+   return cache->Lookup(key);
+ }
+@@ -179,6 +180,7 @@ Status TableCache::FindTable(
+     size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) {
+   PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock);
+   uint64_t number = file_meta.fd.GetNumber();
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&number);
+   *handle = cache_.Lookup(key);
+   TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
+diff --git a/db/version_builder.cc b/db/version_builder.cc
+index ed8ab8214..c98f53f42 100644
+--- a/db/version_builder.cc
++++ b/db/version_builder.cc
+@@ -24,6 +24,7 @@
+ #include <vector>
+ 
+ #include "cache/cache_reservation_manager.h"
++#include "db/blob/blob_file_cache.h"
+ #include "db/blob/blob_file_meta.h"
+ #include "db/dbformat.h"
+ #include "db/internal_stats.h"
+@@ -744,12 +745,9 @@ class VersionBuilder::Rep {
+       return Status::Corruption("VersionBuilder", oss.str());
+     }
+ 
+-    // Note: we use C++11 for now but in C++14, this could be done in a more
+-    // elegant way using generalized lambda capture.
+-    VersionSet* const vs = version_set_;
+-    const ImmutableCFOptions* const ioptions = ioptions_;
+-
+-    auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) {
++    auto deleter = [vs = version_set_, ioptions = ioptions_,
++                    bc = cfd_ ? cfd_->blob_file_cache()
++                              : nullptr](SharedBlobFileMetaData* shared_meta) {
+       if (vs) {
+         assert(ioptions);
+         assert(!ioptions->cf_paths.empty());
+@@ -758,6 +756,9 @@ class VersionBuilder::Rep {
+         vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(),
+                                 ioptions->cf_paths.front().path);
+       }
++      if (bc) {
++        bc->Evict(shared_meta->GetBlobFileNumber());
++      }
+ 
+       delete shared_meta;
+     };
+@@ -766,7 +767,7 @@ class VersionBuilder::Rep {
+         blob_file_number, blob_file_addition.GetTotalBlobCount(),
+         blob_file_addition.GetTotalBlobBytes(),
+         blob_file_addition.GetChecksumMethod(),
+-        blob_file_addition.GetChecksumValue(), deleter);
++        blob_file_addition.GetChecksumValue(), std::move(deleter));
+ 
+     mutable_blob_file_metas_.emplace(
+         blob_file_number, MutableBlobFileMetaData(std::move(shared_meta)));
+diff --git a/db/version_set.h b/db/version_set.h
+index 9336782b1..024f869e7 100644
+--- a/db/version_set.h
++++ b/db/version_set.h
+@@ -1514,7 +1514,6 @@ class VersionSet {
+   void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+ 
+   void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
+-    // TODO: Erase file from BlobFileCache?
+     obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
+   }
+ 
+diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h
+index 2a19796b8..0afa2cab1 100644
+--- a/include/rocksdb/version.h
++++ b/include/rocksdb/version.h
+@@ -13,7 +13,7 @@
+ // minor or major version number planned for release.
+ #define ROCKSDB_MAJOR 9
+ #define ROCKSDB_MINOR 7
+-#define ROCKSDB_PATCH 3
++#define ROCKSDB_PATCH 4
+ 
+ // Do not use these. We made the mistake of declaring macros starting with
+ // double underscore. Now we have to live with our choice. We'll deprecate these
+diff --git a/port/port.h b/port/port.h
+index 13aa56d47..141716e5b 100644
+--- a/port/port.h
++++ b/port/port.h
+@@ -19,3 +19,19 @@
+ #elif defined(OS_WIN)
+ #include "port/win/port_win.h"
+ #endif
++
++#ifdef OS_LINUX
++// A temporary hook into long-running RocksDB threads to support modifying their
++// priority etc. This should become a public API hook once the requirements
++// are better understood.
++extern "C" void RocksDbThreadYield() __attribute__((__weak__));
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {                                 \
++    if (RocksDbThreadYield) {       \
++      RocksDbThreadYield();         \
++    }                               \
++  }
++#else
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {}
++#endif

external/rocksdb/patches/9.x.x-0001-exclude-thirdparty.patch

@@ -0,0 +1,30 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 93b884d..b715cb6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -106,14 +106,9 @@ endif()
+ include(CMakeDependentOption)
+
+ if(MSVC)
+-  option(WITH_GFLAGS "build with GFlags" OFF)
+   option(WITH_XPRESS "build with windows built in compression" OFF)
+-  option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF)
+-
+-  if(NOT ROCKSDB_SKIP_THIRDPARTY)
+-    include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+-  endif()
+-else()
++endif()
++if(TRUE)
+   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
+     # FreeBSD has jemalloc as default malloc
+     # but it does not have all the jemalloc files in include/...
+@@ -126,7 +121,7 @@ else()
+     endif()
+   endif()
+
+-  if(MINGW)
++  if(MSVC OR MINGW)
+     option(WITH_GFLAGS "build with GFlags" OFF)
+   else()
+     option(WITH_GFLAGS "build with GFlags" ON)

external/secp256k1/.cirrus.yml

@@ -10,8 +10,8 @@ env:
   MAKEFLAGS: -j4
   BUILD: check
   ### secp256k1 config
-  ECMULTWINDOW: auto
-  ECMULTGENPRECISION: auto
+  ECMULTWINDOW: 15
+  ECMULTGENKB: 22
   ASM: no
   WIDEMUL: auto
   WITH_VALGRIND: yes
@@ -20,20 +20,18 @@ env:
   EXPERIMENTAL: no
   ECDH: no
   RECOVERY: no
+  EXTRAKEYS: no
   SCHNORRSIG: no
+  MUSIG: no
+  ELLSWIFT: no
   ### test options
-  SECP256K1_TEST_ITERS:
+  SECP256K1_TEST_ITERS: 64
   BENCH: yes
   SECP256K1_BENCH_ITERS: 2
   CTIMETESTS: yes
   # Compile and run the tests
   EXAMPLES: yes
 
-# https://cirrus-ci.org/pricing/#compute-credits
-credits_snippet: &CREDITS
-  # Don't use any credits for now.
-  use_compute_credits: false
-
 cat_logs_snippet: &CAT_LOGS
   always:
     cat_tests_log_script:
@@ -53,357 +51,51 @@ cat_logs_snippet: &CAT_LOGS
     cat_ci_env_script:
       - env
 
-merge_base_script_snippet: &MERGE_BASE
-  merge_base_script:
-    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
-    - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
-    - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
-
-linux_container_snippet: &LINUX_CONTAINER
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    # Reduce number of CPUs to be able to do more builds in parallel.
-    cpu: 1
-    # Gives us more CPUs for free if they're available.
-    greedy: true
-    # More than enough for our scripts.
-    memory: 1G
-
-task:
-  name: "x86_64: Linux (Debian stable)"
-  << : *LINUX_CONTAINER
-  matrix: &ENV_MATRIX
-    - env: {WIDEMUL:  int64,  RECOVERY: yes}
-    - env: {WIDEMUL:  int64,                 ECDH: yes, SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128}
-    - env: {WIDEMUL: int128_struct}
-    - env: {WIDEMUL: int128,  RECOVERY: yes,            SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128,                 ECDH: yes, SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128,  ASM: x86_64}
-    - env: {                  RECOVERY: yes,            SCHNORRSIG: yes}
-    - env: {CTIMETESTS: no,    RECOVERY: yes, ECDH: yes, SCHNORRSIG: yes, CPPFLAGS: -DVERIFY}
-    - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETESTS: no, BENCH: no}
-    - env: {CPPFLAGS: -DDETERMINISTIC}
-    - env: {CFLAGS: -O0, CTIMETESTS: no}
-    - env: { ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 }
-    - env: { ECMULTGENPRECISION: 8, ECMULTWINDOW: 4 }
-  matrix:
-    - env:
-        CC: gcc
-    - env:
-        CC: clang
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "i686: Linux (Debian stable)"
-  << : *LINUX_CONTAINER
-  env:
-    HOST: i686-linux-gnu
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-  matrix:
-    - env:
-        CC: i686-linux-gnu-gcc
-    - env:
-        CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "arm64: macOS Ventura"
-  macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-base:latest
-  env:
-    HOMEBREW_NO_AUTO_UPDATE: 1
-    HOMEBREW_NO_INSTALL_CLEANUP: 1
-    # Cirrus gives us a fixed number of 4 virtual CPUs. Not that we even have that many jobs at the moment...
-    MAKEFLAGS: -j5
-  matrix:
-    << : *ENV_MATRIX
-  env:
-    ASM: no
-    WITH_VALGRIND: no
-    CTIMETESTS: no
-  matrix:
-    - env:
-        CC: gcc
-    - env:
-        CC: clang
-  brew_script:
-    - brew install automake libtool gcc
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-  << : *CREDITS
-
-task:
-  name: "s390x (big-endian): Linux (Debian stable, QEMU)"
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: qemu-s390x
-    SECP256K1_TEST_ITERS: 16
-    HOST: s390x-linux-gnu
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  << : *MERGE_BASE
-  test_script:
-    # https://sourceware.org/bugzilla/show_bug.cgi?id=27008
-    - rm /etc/ld.so.cache
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "ARM32: Linux (Debian stable, QEMU)"
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: qemu-arm
-    SECP256K1_TEST_ITERS: 16
-    HOST: arm-linux-gnueabihf
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  matrix:
-    - env: {}
-    - env: {EXPERIMENTAL: yes, ASM: arm32}
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "ARM64: Linux (Debian stable, QEMU)"
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: qemu-aarch64
-    SECP256K1_TEST_ITERS: 16
-    HOST: aarch64-linux-gnu
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "ppc64le: Linux (Debian stable, QEMU)"
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: qemu-ppc64le
-    SECP256K1_TEST_ITERS: 16
-    HOST: powerpc64le-linux-gnu
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: wine
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  matrix:
-    - name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)"
-      env:
-        HOST: x86_64-w64-mingw32
-    - name: "i686 (mingw32-w64): Windows (Debian stable, Wine)"
-      env:
-        HOST: i686-w64-mingw32
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  << : *LINUX_CONTAINER
-  env:
-    WRAPPER_CMD: wine
-    WERROR_CFLAGS: -WX
-    WITH_VALGRIND: no
-    ECDH: yes
-    RECOVERY: yes
-    EXPERIMENTAL: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-    # Use a MinGW-w64 host to tell ./configure we're building for Windows.
-    # This will detect some MinGW-w64 tools but then make will need only
-    # the MSVC tools CC, AR and NM as specified below.
-    HOST: x86_64-w64-mingw32
-    CC: /opt/msvc/bin/x64/cl
-    AR: /opt/msvc/bin/x64/lib
-    NM: /opt/msvc/bin/x64/dumpbin -symbols -headers
-    # Set non-essential options that affect the CLI messages here.
-    # (They depend on the user's taste, so we don't want to set them automatically in configure.ac.)
-    CFLAGS: -nologo -diagnostics:caret
-    LDFLAGS: -Xlinker -Xlinker -Xlinker -nologo
-  matrix:
-    - name: "x86_64 (MSVC): Windows (Debian stable, Wine)"
-    - name: "x86_64 (MSVC): Windows (Debian stable, Wine, int128_struct)"
-      env:
-        WIDEMUL: int128_struct
-    - name: "x86_64 (MSVC): Windows (Debian stable, Wine, int128_struct with __(u)mulh)"
-      env:
-        WIDEMUL: int128_struct
-        CPPFLAGS: -DSECP256K1_MSVC_MULH_TEST_OVERRIDE
-    - name: "i686 (MSVC): Windows (Debian stable, Wine)"
-      env:
-        HOST: i686-w64-mingw32
-        CC: /opt/msvc/bin/x86/cl
-        AR: /opt/msvc/bin/x86/lib
-        NM: /opt/msvc/bin/x86/dumpbin -symbols -headers
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-# Sanitizers
-task:
-  << : *LINUX_CONTAINER
-  env:
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: no
-  matrix:
-    - name: "Valgrind (memcheck)"
-      container:
-        cpu: 2
-      env:
-        # The `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html)
-        WRAPPER_CMD: "valgrind --error-exitcode=42"
-        SECP256K1_TEST_ITERS: 2
-    - name: "UBSan, ASan, LSan"
-      container:
-        memory: 2G
-      env:
-        CFLAGS: "-fsanitize=undefined,address -g"
-        UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
-        ASAN_OPTIONS: "strict_string_checks=1:detect_stack_use_after_return=1:detect_leaks=1"
-        LSAN_OPTIONS: "use_unaligned=1"
-        SECP256K1_TEST_ITERS: 32
-  # Try to cover many configurations with just a tiny matrix.
-  matrix:
-    - env:
-        ASM: auto
-    - env:
-        ASM: no
-        ECMULTGENPRECISION: 2
-        ECMULTWINDOW: 2
-  matrix:
-    - env:
-        CC: clang
-    - env:
-        HOST: i686-linux-gnu
-        CC: i686-linux-gnu-gcc
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-# Memory sanitizers
-task:
-  << : *LINUX_CONTAINER
-  name: "MSan"
-  env:
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-    CTIMETESTS: yes
-    CC: clang
-    SECP256K1_TEST_ITERS: 32
-    ASM: no
-    WITH_VALGRIND: no
-  container:
-    memory: 2G
-  matrix:
-    - env:
-        CFLAGS: "-fsanitize=memory -g"
-    - env:
-        ECMULTGENPRECISION: 2
-        ECMULTWINDOW: 2
-        CFLAGS: "-fsanitize=memory -g -O3"
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "C++ -fpermissive (entire project)"
-  << : *LINUX_CONTAINER
-  env:
-    CC: g++
-    CFLAGS: -fpermissive -g
-    CPPFLAGS: -DSECP256K1_CPLUSPLUS_TEST_OVERRIDE
-    WERROR_CFLAGS:
-    ECDH: yes
-    RECOVERY: yes
-    SCHNORRSIG: yes
-  << : *MERGE_BASE
-  test_script:
-    - ./ci/cirrus.sh
-  << : *CAT_LOGS
-
-task:
-  name: "C++ (public headers)"
-  << : *LINUX_CONTAINER
-  test_script:
-    - g++ -Werror include/*.h
-    - clang -Werror -x c++-header include/*.h
-    - /opt/msvc/bin/x64/cl.exe -c -WX -TP include/*.h
-
-task:
-  name: "sage prover"
-  << : *LINUX_CONTAINER
-  test_script:
-    - cd sage
-    - sage prove_group_implementations.sage
-
-task:
-  name: "x86_64: Windows (VS 2022)"
-  windows_container:
-    image: cirrusci/windowsservercore:visualstudio2022
-    cpu: 4
-    memory: 3840MB
-  env:
-    PATH: '%CIRRUS_WORKING_DIR%\build\src\RelWithDebInfo;%PATH%'
-    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
-    # Ignore MSBuild warning MSB8029.
-    # See: https://learn.microsoft.com/en-us/visualstudio/msbuild/errors/msb8029?view=vs-2022
-    IgnoreWarnIntDirInTempDetected: 'true'
-  merge_script:
-    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; }
-  configure_script:
-    - '%x64_NATIVE_TOOLS%'
-    - cmake -E env CFLAGS="/WX" cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_BUILD_EXAMPLES=ON
+linux_arm64_container_snippet: &LINUX_ARM64_CONTAINER
+  env_script:
+    - env | tee /tmp/env
   build_script:
-    - '%x64_NATIVE_TOOLS%'
-    - cmake --build build --config RelWithDebInfo -- -property:UseMultiToolTask=true;CL_MPcount=5
-  check_script:
-    - '%x64_NATIVE_TOOLS%'
-    - ctest -C RelWithDebInfo --test-dir build -j 5
-    - build\src\RelWithDebInfo\bench_ecmult.exe
-    - build\src\RelWithDebInfo\bench_internal.exe
-    - build\src\RelWithDebInfo\bench.exe
+    - DOCKER_BUILDKIT=1 docker build --file "ci/linux-debian.Dockerfile" --tag="ci_secp256k1_arm"
+    - docker image prune --force  # Cleanup stale layers
+  test_script:
+    - docker run --rm --mount "type=bind,src=./,dst=/ci_secp256k1" --env-file /tmp/env --replace --name "ci_secp256k1_arm" "ci_secp256k1_arm" bash -c "cd /ci_secp256k1/ && ./ci/ci.sh"
+
+task:
+  name: "ARM64: Linux (Debian stable)"
+  persistent_worker:
+    labels:
+      type: arm64
+  env:
+    ECDH: yes
+    RECOVERY: yes
+    EXTRAKEYS: yes
+    SCHNORRSIG: yes
+    MUSIG: yes
+    ELLSWIFT: yes
+  matrix:
+     # Currently only gcc-snapshot, the other compilers are tested on GHA with QEMU
+     - env: { CC: 'gcc-snapshot' }
+  << : *LINUX_ARM64_CONTAINER
+  << : *CAT_LOGS
+
+task:
+  name: "ARM64: Linux (Debian stable), Valgrind"
+  persistent_worker:
+    labels:
+      type: arm64
+  env:
+    ECDH: yes
+    RECOVERY: yes
+    EXTRAKEYS: yes
+    SCHNORRSIG: yes
+    MUSIG: yes
+    ELLSWIFT: yes
+    WRAPPER_CMD: 'valgrind --error-exitcode=42'
+    SECP256K1_TEST_ITERS: 2
+  matrix:
+     - env: { CC: 'gcc' }
+     - env: { CC: 'clang' }
+     - env: { CC: 'gcc-snapshot' }
+     - env: { CC: 'clang-snapshot' }
+  << : *LINUX_ARM64_CONTAINER
+  << : *CAT_LOGS

external/secp256k1/.gitignore

@@ -10,6 +10,8 @@ ctime_tests
 ecdh_example
 ecdsa_example
 schnorr_example
+ellswift_example
+musig_example
 *.exe
 *.so
 *.a

external/secp256k1/CHANGELOG.md

@@ -5,6 +5,83 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.0] - 2024-11-04
+
+#### Added
+ - New module `musig` implements the MuSig2 multisignature scheme according to the [BIP 327 specification](https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki). See:
+   - Header file `include/secp256k1_musig.h` which defines the new API.
+   - Document `doc/musig.md` for further notes on API usage.
+   - Usage example `examples/musig.c`.
+ - New CMake variable `SECP256K1_APPEND_LDFLAGS` for appending linker flags to the build command.
+
+#### Changed
+ - API functions now use a significantly more robust method to clear secrets from the stack before returning. However, secret clearing remains a best-effort security measure and cannot guarantee complete removal.
+ - Any type `secp256k1_foo` can now be forward-declared using `typedef struct secp256k1_foo secp256k1_foo;` (or also `struct secp256k1_foo;` in C++).
+ - Organized CMake build artifacts into dedicated directories (`bin/` for executables, `lib/` for libraries) to improve build output structure and Windows shared library compatibility.
+
+#### Removed
+ - Removed the `secp256k1_scratch_space` struct and its associated functions `secp256k1_scratch_space_create` and `secp256k1_scratch_space_destroy` because the scratch space was unused in the API.
+
+#### ABI Compatibility
+The symbols `secp256k1_scratch_space_create` and `secp256k1_scratch_space_destroy` were removed.
+Otherwise, the library maintains backward compatibility with versions 0.3.x through 0.5.x.
+
+## [0.5.1] - 2024-08-01
+
+#### Added
+ - Added usage example for an ElligatorSwift key exchange.
+
+#### Changed
+ - The default size of the precomputed table for signing was changed from 22 KiB to 86 KiB. The size can be changed with the configure option `--ecmult-gen-kb` (`SECP256K1_ECMULT_GEN_KB` for CMake).
+ - "auto" is no longer an accepted value for the `--with-ecmult-window` and `--with-ecmult-gen-kb` configure options (this also applies to  `SECP256K1_ECMULT_WINDOW_SIZE` and `SECP256K1_ECMULT_GEN_KB` in CMake). To achieve the same configuration as previously provided by the "auto" value, omit setting the configure option explicitly.
+
+#### Fixed
+ - Fixed compilation when the extrakeys module is disabled.
+
+#### ABI Compatibility
+The ABI is backward compatible with versions 0.5.0, 0.4.x and 0.3.x.
+
+## [0.5.0] - 2024-05-06
+
+#### Added
+ - New function `secp256k1_ec_pubkey_sort` that sorts public keys using lexicographic (of compressed serialization) order.
+
+#### Changed
+ - The implementation of the point multiplication algorithm used for signing and public key generation was changed, resulting in improved performance for those operations.
+   - The related configure option `--ecmult-gen-precision` was replaced with `--ecmult-gen-kb` (`SECP256K1_ECMULT_GEN_KB` for CMake).
+   - This changes the supported precomputed table sizes for these operations. The new supported sizes are 2 KiB, 22 KiB, or 86 KiB (while the old supported sizes were 32 KiB, 64 KiB, or 512 KiB).
+
+#### ABI Compatibility
+The ABI is backward compatible with versions 0.4.x and 0.3.x.
+
+## [0.4.1] - 2023-12-21
+
+#### Changed
+ - The point multiplication algorithm used for ECDH operations (module `ecdh`) was replaced with a slightly faster one.
+ - Optional handwritten x86_64 assembly for field operations was removed because modern C compilers are able to output more efficient assembly. This change results in a significant speedup of some library functions when handwritten x86_64 assembly is enabled (`--with-asm=x86_64` in GNU Autotools, `-DSECP256K1_ASM=x86_64` in CMake), which is the default on x86_64. Benchmarks with GCC 10.5.0 show a 10% speedup for `secp256k1_ecdsa_verify` and `secp256k1_schnorrsig_verify`.
+
+#### ABI Compatibility
+The ABI is backward compatible with versions 0.4.0 and 0.3.x.
+
+## [0.4.0] - 2023-09-04
+
+#### Added
+ - New module `ellswift` implements ElligatorSwift encoding for public keys and x-only Diffie-Hellman key exchange for them.
+   ElligatorSwift permits representing secp256k1 public keys as 64-byte arrays which cannot be distinguished from uniformly random. See:
+   - Header file `include/secp256k1_ellswift.h` which defines the new API.
+   - Document `doc/ellswift.md` which explains the mathematical background of the scheme.
+   - The [paper](https://eprint.iacr.org/2022/759) on which the scheme is based.
+ - We now test the library with unreleased development snapshots of GCC and Clang. This gives us an early chance to catch miscompilations and constant-time issues introduced by the compiler (such as those that led to the previous two releases).
+
+#### Fixed
+ - Fixed symbol visibility in Windows DLL builds, where three internal library symbols were wrongly exported.
+
+#### Changed
+ - When consuming libsecp256k1 as a static library on Windows, the user must now define the `SECP256K1_STATIC` macro before including `secp256k1.h`.
+
+#### ABI Compatibility
+This release is backward compatible with the ABI of 0.3.0, 0.3.1, and 0.3.2. Symbol visibility is now believed to be handled properly on supported platforms and is now considered to be part of the ABI. Please report any improperly exported symbols as a bug.
+
 ## [0.3.2] - 2023-05-13
 We strongly recommend updating to 0.3.2 if you use or plan to use GCC >=13 to compile libsecp256k1. When in doubt, check the GCC version using `gcc -v`.
 
@@ -85,7 +162,11 @@ This version was in fact never released.
 The number was given by the build system since the introduction of autotools in Jan 2014 (ea0fe5a5bf0c04f9cc955b2966b614f5f378c6f6).
 Therefore, this version number does not uniquely identify a set of source files.
 
-[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.2...HEAD
+[0.6.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.5.1...v0.6.0
+[0.5.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.5.0...v0.5.1
+[0.5.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.1...v0.5.0
+[0.4.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.0...v0.4.1
+[0.4.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.2...v0.4.0
 [0.3.2]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.1...v0.3.2
 [0.3.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.2.0...v0.3.0

external/secp256k1/CMakeLists.txt

@@ -1,32 +1,29 @@
-cmake_minimum_required(VERSION 3.13)
-
-if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.15)
-  # MSVC runtime library flags are selected by the CMAKE_MSVC_RUNTIME_LIBRARY abstraction.
-  cmake_policy(SET CMP0091 NEW)
-  # MSVC warning flags are not in CMAKE_<LANG>_FLAGS by default.
-  cmake_policy(SET CMP0092 NEW)
-endif()
+cmake_minimum_required(VERSION 3.16)
 
+#=============================
+# Project / Package metadata
+#=============================
 project(libsecp256k1
   # The package (a.k.a. release) version is based on semantic versioning 2.0.0 of
   # the API. All changes in experimental modules are treated as
   # backwards-compatible and therefore at most increase the minor version.
-  VERSION 0.3.2
+  VERSION 0.6.0
   DESCRIPTION "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1."
   HOMEPAGE_URL "https://github.com/bitcoin-core/secp256k1"
   LANGUAGES C
 )
+enable_testing()
+list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 if(CMAKE_VERSION VERSION_LESS 3.21)
-  get_directory_property(parent_directory PARENT_DIRECTORY)
-  if(parent_directory)
-    set(PROJECT_IS_TOP_LEVEL OFF CACHE INTERNAL "Emulates CMake 3.21+ behavior.")
-    set(${PROJECT_NAME}_IS_TOP_LEVEL OFF CACHE INTERNAL "Emulates CMake 3.21+ behavior.")
+  # Emulates CMake 3.21+ behavior.
+  if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+    set(PROJECT_IS_TOP_LEVEL ON)
+    set(${PROJECT_NAME}_IS_TOP_LEVEL ON)
   else()
-    set(PROJECT_IS_TOP_LEVEL ON CACHE INTERNAL "Emulates CMake 3.21+ behavior.")
-    set(${PROJECT_NAME}_IS_TOP_LEVEL ON CACHE INTERNAL "Emulates CMake 3.21+ behavior.")
+    set(PROJECT_IS_TOP_LEVEL OFF)
+    set(${PROJECT_NAME}_IS_TOP_LEVEL OFF)
   endif()
-  unset(parent_directory)
 endif()
 
 # The library version is based on libtool versioning of the ABI. The set of
@@ -34,15 +31,19 @@ endif()
 # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
 # All changes in experimental modules are treated as if they don't affect the
 # interface and therefore only increase the revision.
-set(${PROJECT_NAME}_LIB_VERSION_CURRENT 2)
-set(${PROJECT_NAME}_LIB_VERSION_REVISION 2)
+set(${PROJECT_NAME}_LIB_VERSION_CURRENT 5)
+set(${PROJECT_NAME}_LIB_VERSION_REVISION 0)
 set(${PROJECT_NAME}_LIB_VERSION_AGE 0)
 
+#=============================
+# Language setup
+#=============================
 set(CMAKE_C_STANDARD 90)
 set(CMAKE_C_EXTENSIONS OFF)
 
-list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-
+#=============================
+# Configurable options
+#=============================
 option(BUILD_SHARED_LIBS "Build shared libraries." ON)
 option(SECP256K1_DISABLE_SHARED "Disable shared library. Overrides BUILD_SHARED_LIBS." OFF)
 if(SECP256K1_DISABLE_SHARED)
@@ -51,24 +52,49 @@ endif()
 
 option(SECP256K1_INSTALL "Enable installation." ${PROJECT_IS_TOP_LEVEL})
 
+## Modules
+
+# We declare all options before processing them, to make sure we can express
+# dependendencies while processing.
 option(SECP256K1_ENABLE_MODULE_ECDH "Enable ECDH module." ON)
-if(SECP256K1_ENABLE_MODULE_ECDH)
-  add_compile_definitions(ENABLE_MODULE_ECDH=1)
+option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF)
+option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON)
+option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON)
+option(SECP256K1_ENABLE_MODULE_MUSIG "Enable musig module." ON)
+option(SECP256K1_ENABLE_MODULE_ELLSWIFT "Enable ElligatorSwift module." ON)
+
+# Processing must be done in a topological sorting of the dependency graph
+# (dependent module first).
+if(SECP256K1_ENABLE_MODULE_ELLSWIFT)
+  add_compile_definitions(ENABLE_MODULE_ELLSWIFT=1)
+endif()
+
+if(SECP256K1_ENABLE_MODULE_MUSIG)
+  if(DEFINED SECP256K1_ENABLE_MODULE_SCHNORRSIG AND NOT SECP256K1_ENABLE_MODULE_SCHNORRSIG)
+    message(FATAL_ERROR "Module dependency error: You have disabled the schnorrsig module explicitly, but it is required by the musig module.")
+  endif()
+  set(SECP256K1_ENABLE_MODULE_SCHNORRSIG ON)
+  add_compile_definitions(ENABLE_MODULE_MUSIG=1)
+endif()
+
+if(SECP256K1_ENABLE_MODULE_SCHNORRSIG)
+  if(DEFINED SECP256K1_ENABLE_MODULE_EXTRAKEYS AND NOT SECP256K1_ENABLE_MODULE_EXTRAKEYS)
+    message(FATAL_ERROR "Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.")
+  endif()
+  set(SECP256K1_ENABLE_MODULE_EXTRAKEYS ON)
+  add_compile_definitions(ENABLE_MODULE_SCHNORRSIG=1)
+endif()
+
+if(SECP256K1_ENABLE_MODULE_EXTRAKEYS)
+  add_compile_definitions(ENABLE_MODULE_EXTRAKEYS=1)
 endif()
 
-option(SECP256K1_ENABLE_MODULE_RECOVERY "Enable ECDSA pubkey recovery module." OFF)
 if(SECP256K1_ENABLE_MODULE_RECOVERY)
   add_compile_definitions(ENABLE_MODULE_RECOVERY=1)
 endif()
 
-option(SECP256K1_ENABLE_MODULE_EXTRAKEYS "Enable extrakeys module." ON)
-option(SECP256K1_ENABLE_MODULE_SCHNORRSIG "Enable schnorrsig module." ON)
-if(SECP256K1_ENABLE_MODULE_SCHNORRSIG)
-  set(SECP256K1_ENABLE_MODULE_EXTRAKEYS ON)
-  add_compile_definitions(ENABLE_MODULE_SCHNORRSIG=1)
-endif()
-if(SECP256K1_ENABLE_MODULE_EXTRAKEYS)
-  add_compile_definitions(ENABLE_MODULE_EXTRAKEYS=1)
+if(SECP256K1_ENABLE_MODULE_ECDH)
+  add_compile_definitions(ENABLE_MODULE_ECDH=1)
 endif()
 
 option(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS "Enable external default callback functions." OFF)
@@ -76,22 +102,25 @@ if(SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS)
   add_compile_definitions(USE_EXTERNAL_DEFAULT_CALLBACKS=1)
 endif()
 
-set(SECP256K1_ECMULT_WINDOW_SIZE "AUTO" CACHE STRING "Window size for ecmult precomputation for verification, specified as integer in range [2..24]. \"AUTO\" is a reasonable setting for desktop machines (currently 15). [default=AUTO]")
-set_property(CACHE SECP256K1_ECMULT_WINDOW_SIZE PROPERTY STRINGS "AUTO" 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24)
+set(SECP256K1_ECMULT_WINDOW_SIZE 15 CACHE STRING "Window size for ecmult precomputation for verification, specified as integer in range [2..24]. The default value is a reasonable setting for desktop machines (currently 15). [default=15]")
+set_property(CACHE SECP256K1_ECMULT_WINDOW_SIZE PROPERTY STRINGS 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24)
 include(CheckStringOptionValue)
 check_string_option_value(SECP256K1_ECMULT_WINDOW_SIZE)
-if(SECP256K1_ECMULT_WINDOW_SIZE STREQUAL "AUTO")
-  set(SECP256K1_ECMULT_WINDOW_SIZE 15)
-endif()
 add_compile_definitions(ECMULT_WINDOW_SIZE=${SECP256K1_ECMULT_WINDOW_SIZE})
 
-set(SECP256K1_ECMULT_GEN_PREC_BITS "AUTO" CACHE STRING "Precision bits to tune the precomputed table size for signing, specified as integer 2, 4 or 8. \"AUTO\" is a reasonable setting for desktop machines (currently 4). [default=AUTO]")
-set_property(CACHE SECP256K1_ECMULT_GEN_PREC_BITS PROPERTY STRINGS "AUTO" 2 4 8)
-check_string_option_value(SECP256K1_ECMULT_GEN_PREC_BITS)
-if(SECP256K1_ECMULT_GEN_PREC_BITS STREQUAL "AUTO")
-  set(SECP256K1_ECMULT_GEN_PREC_BITS 4)
+set(SECP256K1_ECMULT_GEN_KB 86 CACHE STRING "The size of the precomputed table for signing in multiples of 1024 bytes (on typical platforms). Larger values result in possibly better signing or key generation performance at the cost of a larger table. Valid choices are 2, 22, 86. The default value is a reasonable setting for desktop machines (currently 86). [default=86]")
+set_property(CACHE SECP256K1_ECMULT_GEN_KB PROPERTY STRINGS 2 22 86)
+check_string_option_value(SECP256K1_ECMULT_GEN_KB)
+if(SECP256K1_ECMULT_GEN_KB EQUAL 2)
+  add_compile_definitions(COMB_BLOCKS=2)
+  add_compile_definitions(COMB_TEETH=5)
+elseif(SECP256K1_ECMULT_GEN_KB EQUAL 22)
+  add_compile_definitions(COMB_BLOCKS=11)
+  add_compile_definitions(COMB_TEETH=6)
+elseif(SECP256K1_ECMULT_GEN_KB EQUAL 86)
+  add_compile_definitions(COMB_BLOCKS=43)
+  add_compile_definitions(COMB_TEETH=6)
 endif()
-add_compile_definitions(ECMULT_GEN_PREC_BITS=${SECP256K1_ECMULT_GEN_PREC_BITS})
 
 set(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY "OFF" CACHE STRING "Test-only override of the (autodetected by the C code) \"widemul\" setting. Legal values are: \"OFF\", \"int128_struct\", \"int128\" or \"int64\". [default=OFF]")
 set_property(CACHE SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY PROPERTY STRINGS "OFF" "int128_struct" "int128" "int64")
@@ -102,7 +131,7 @@ if(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY)
 endif()
 mark_as_advanced(FORCE SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY)
 
-set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly optimizations to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm32\" (experimental). [default=AUTO]")
+set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm32\" (experimental). [default=AUTO]")
 set_property(CACHE SECP256K1_ASM PROPERTY STRINGS "AUTO" "OFF" "x86_64" "arm32")
 check_string_option_value(SECP256K1_ASM)
 if(SECP256K1_ASM STREQUAL "arm32")
@@ -112,7 +141,7 @@ if(SECP256K1_ASM STREQUAL "arm32")
   if(HAVE_ARM32_ASM)
     add_compile_definitions(USE_EXTERNAL_ASM=1)
   else()
-    message(FATAL_ERROR "ARM32 assembly optimization requested but not available.")
+    message(FATAL_ERROR "ARM32 assembly requested but not available.")
   endif()
 elseif(SECP256K1_ASM)
   include(CheckX86_64Assembly)
@@ -123,14 +152,14 @@ elseif(SECP256K1_ASM)
   elseif(SECP256K1_ASM STREQUAL "AUTO")
     set(SECP256K1_ASM "OFF")
   else()
-    message(FATAL_ERROR "x86_64 assembly optimization requested but not available.")
+    message(FATAL_ERROR "x86_64 assembly requested but not available.")
   endif()
 endif()
 
 option(SECP256K1_EXPERIMENTAL "Allow experimental configuration options." OFF)
 if(NOT SECP256K1_EXPERIMENTAL)
   if(SECP256K1_ASM STREQUAL "arm32")
-    message(FATAL_ERROR "ARM32 assembly optimization is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.")
+    message(FATAL_ERROR "ARM32 assembly is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.")
   endif()
 endif()
 
@@ -167,7 +196,7 @@ else()
   string(REGEX REPLACE "-DNDEBUG[ \t\r\n]*" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
   string(REGEX REPLACE "-DNDEBUG[ \t\r\n]*" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
   # Prefer -O2 optimization level. (-O3 is CMake's default for Release for many compilers.)
-  string(REGEX REPLACE "-O3[ \t\r\n]*" "-O2" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+  string(REGEX REPLACE "-O3( |$)" "-O2\\1" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
 endif()
 
 # Define custom "Coverage" build type.
@@ -189,31 +218,37 @@ mark_as_advanced(
   CMAKE_SHARED_LINKER_FLAGS_COVERAGE
 )
 
-get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
-set(default_build_type "RelWithDebInfo")
-if(is_multi_config)
-  set(CMAKE_CONFIGURATION_TYPES "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage" CACHE STRING
-    "Supported configuration types."
-    FORCE
-  )
-else()
-  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY
-    STRINGS "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage"
-  )
-  if(NOT CMAKE_BUILD_TYPE)
-    message(STATUS "Setting build type to \"${default_build_type}\" as none was specified")
-    set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING
-      "Choose the type of build."
+if(PROJECT_IS_TOP_LEVEL)
+  get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+  set(default_build_type "RelWithDebInfo")
+  if(is_multi_config)
+    set(CMAKE_CONFIGURATION_TYPES "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage" CACHE STRING
+      "Supported configuration types."
       FORCE
     )
+  else()
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY
+      STRINGS "${default_build_type}" "Release" "Debug" "MinSizeRel" "Coverage"
+    )
+    if(NOT CMAKE_BUILD_TYPE)
+      message(STATUS "Setting build type to \"${default_build_type}\" as none was specified")
+      set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING
+        "Choose the type of build."
+        FORCE
+      )
+    endif()
   endif()
 endif()
 
 include(TryAppendCFlags)
 if(MSVC)
   # Keep the following commands ordered lexicographically.
-  try_append_c_flags(/W2) # Moderate warning level.
+  try_append_c_flags(/W3) # Production quality warning level.
   try_append_c_flags(/wd4146) # Disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned".
+  try_append_c_flags(/wd4244) # Disable warning C4244 "'conversion' conversion from 'type1' to 'type2', possible loss of data".
+  try_append_c_flags(/wd4267) # Disable warning C4267 "'var' : conversion from 'size_t' to 'type', possible loss of data".
+  # Eliminate deprecation warnings for the older, less secure functions.
+  add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
 else()
   # Keep the following commands ordered lexicographically.
   try_append_c_flags(-pedantic)
@@ -234,17 +269,41 @@ endif()
 
 set(CMAKE_C_VISIBILITY_PRESET hidden)
 
-# Ask CTest to create a "check" target (e.g., make check) as alias for the "test" target.
-# CTEST_TEST_TARGET_ALIAS is not documented but supposed to be user-facing.
-# See: https://gitlab.kitware.com/cmake/cmake/-/commit/816c9d1aa1f2b42d40c81a991b68c96eb12b6d2
-set(CTEST_TEST_TARGET_ALIAS check)
-include(CTest)
-# We do not use CTest's BUILD_TESTING because a single toggle for all tests is too coarse for our needs.
-mark_as_advanced(BUILD_TESTING)
-if(SECP256K1_BUILD_BENCHMARK OR SECP256K1_BUILD_TESTS OR SECP256K1_BUILD_EXHAUSTIVE_TESTS OR SECP256K1_BUILD_CTIME_TESTS OR SECP256K1_BUILD_EXAMPLES)
-  enable_testing()
+set(print_msan_notice)
+if(SECP256K1_BUILD_CTIME_TESTS)
+  include(CheckMemorySanitizer)
+  check_memory_sanitizer(msan_enabled)
+  if(msan_enabled)
+    try_append_c_flags(-fno-sanitize-memory-param-retval)
+    set(print_msan_notice YES)
+  endif()
+  unset(msan_enabled)
 endif()
 
+set(SECP256K1_APPEND_CFLAGS "" CACHE STRING "Compiler flags that are appended to the command line after all other flags added by the build system. This variable is intended for debugging and special builds.")
+if(SECP256K1_APPEND_CFLAGS)
+  # Appending to this low-level rule variable is the only way to
+  # guarantee that the flags appear at the end of the command line.
+  string(APPEND CMAKE_C_COMPILE_OBJECT " ${SECP256K1_APPEND_CFLAGS}")
+endif()
+
+set(SECP256K1_APPEND_LDFLAGS "" CACHE STRING "Linker flags that are appended to the command line after all other flags added by the build system. This variable is intended for debugging and special builds.")
+if(SECP256K1_APPEND_LDFLAGS)
+  # Appending to this low-level rule variable is the only way to
+  # guarantee that the flags appear at the end of the command line.
+  string(APPEND CMAKE_C_CREATE_SHARED_LIBRARY " ${SECP256K1_APPEND_LDFLAGS}")
+  string(APPEND CMAKE_C_LINK_EXECUTABLE " ${SECP256K1_APPEND_LDFLAGS}")
+endif()
+
+if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin)
+endif()
+if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
+endif()
+if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY)
+  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
+endif()
 add_subdirectory(src)
 if(SECP256K1_BUILD_EXAMPLES)
   add_subdirectory(examples)
@@ -266,11 +325,13 @@ message("  ECDH ................................ ${SECP256K1_ENABLE_MODULE_ECDH}
 message("  ECDSA pubkey recovery ............... ${SECP256K1_ENABLE_MODULE_RECOVERY}")
 message("  extrakeys ........................... ${SECP256K1_ENABLE_MODULE_EXTRAKEYS}")
 message("  schnorrsig .......................... ${SECP256K1_ENABLE_MODULE_SCHNORRSIG}")
+message("  musig ............................... ${SECP256K1_ENABLE_MODULE_MUSIG}")
+message("  ElligatorSwift ...................... ${SECP256K1_ENABLE_MODULE_ELLSWIFT}")
 message("Parameters:")
 message("  ecmult window size .................. ${SECP256K1_ECMULT_WINDOW_SIZE}")
-message("  ecmult gen precision bits ........... ${SECP256K1_ECMULT_GEN_PREC_BITS}")
+message("  ecmult gen table size ............... ${SECP256K1_ECMULT_GEN_KB} KiB")
 message("Optional features:")
-message("  assembly optimization ............... ${SECP256K1_ASM}")
+message("  assembly ............................ ${SECP256K1_ASM}")
 message("  external callbacks .................. ${SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS}")
 if(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY)
   message("  wide multiplication (test-only) ..... ${SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY}")
@@ -297,7 +358,7 @@ message("Valgrind .............................. ${SECP256K1_VALGRIND}")
 get_directory_property(definitions COMPILE_DEFINITIONS)
 string(REPLACE ";" " " definitions "${definitions}")
 message("Preprocessor defined macros ........... ${definitions}")
-message("C compiler ............................ ${CMAKE_C_COMPILER}")
+message("C compiler ............................ ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}, ${CMAKE_C_COMPILER}")
 message("CFLAGS ................................ ${CMAKE_C_FLAGS}")
 get_directory_property(compile_options COMPILE_OPTIONS)
 string(REPLACE ";" " " compile_options "${compile_options}")
@@ -320,7 +381,20 @@ else()
   message(" - LDFLAGS for executables ............ ${CMAKE_EXE_LINKER_FLAGS_DEBUG}")
   message(" - LDFLAGS for shared libraries ....... ${CMAKE_SHARED_LINKER_FLAGS_DEBUG}")
 endif()
-message("\n")
+if(SECP256K1_APPEND_CFLAGS)
+  message("SECP256K1_APPEND_CFLAGS ............... ${SECP256K1_APPEND_CFLAGS}")
+endif()
+if(SECP256K1_APPEND_LDFLAGS)
+  message("SECP256K1_APPEND_LDFLAGS .............. ${SECP256K1_APPEND_LDFLAGS}")
+endif()
+message("")
+if(print_msan_notice)
+  message(
+    "Note:\n"
+    "  MemorySanitizer detected, tried to add -fno-sanitize-memory-param-retval to compile options\n"
+    "  to avoid false positives in ctime_tests. Pass -DSECP256K1_BUILD_CTIME_TESTS=OFF to avoid this.\n"
+  )
+endif()
 if(SECP256K1_EXPERIMENTAL)
   message(
     "  ******\n"

external/secp256k1/CONTRIBUTING.md

@@ -0,0 +1,108 @@
+# Contributing to libsecp256k1
+
+## Scope
+
+libsecp256k1 is a library for elliptic curve cryptography on the curve secp256k1, not a general-purpose cryptography library.
+The library primarily serves the needs of the Bitcoin Core project but provides additional functionality for the benefit of the wider Bitcoin ecosystem.
+
+## Adding new functionality or modules
+
+The libsecp256k1 project welcomes contributions in the form of new functionality or modules, provided they are within the project's scope.
+
+It is the responsibility of the contributors to convince the maintainers that the proposed functionality is within the project's scope, high-quality and maintainable.
+Contributors are recommended to provide the following in addition to the new code:
+
+* **Specification:**
+    A specification can help significantly in reviewing the new code as it provides documentation and context.
+    It may justify various design decisions, give a motivation and outline security goals.
+    If the specification contains pseudocode, a reference implementation or test vectors, these can be used to compare with the proposed libsecp256k1 code.
+* **Security Arguments:**
+    In addition to a defining the security goals, it should be argued that the new functionality meets these goals.
+    Depending on the nature of the new functionality, a wide range of security arguments are acceptable, ranging from being "obviously secure" to rigorous proofs of security.
+* **Relevance Arguments:**
+    The relevance of the new functionality for the Bitcoin ecosystem should be argued by outlining clear use cases.
+
+These are not the only factors taken into account when considering to add new functionality.
+The proposed new libsecp256k1 code must be of high quality, including API documentation and tests, as well as featuring a misuse-resistant API design.
+
+We recommend reaching out to other contributors (see [Communication Channels](#communication-channels)) and get feedback before implementing new functionality.
+
+## Communication channels
+
+Most communication about libsecp256k1 occurs on the GitHub repository: in issues, pull request or on the discussion board.
+
+Additionally, there is an IRC channel dedicated to libsecp256k1, with biweekly meetings (see channel topic).
+The channel is `#secp256k1` on Libera Chat.
+The easiest way to participate on IRC is with the web client, [web.libera.chat](https://web.libera.chat/#secp256k1).
+Chat history logs can be found at https://gnusha.org/secp256k1/.
+
+## Contributor workflow & peer review
+
+The Contributor Workflow & Peer Review in libsecp256k1 are similar to Bitcoin Core's workflow and review processes described in its [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md).
+
+### Coding conventions
+
+In addition, libsecp256k1 tries to maintain the following coding conventions:
+
+* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Moreover, it should be possible to use the library without any heap allocations.
+* The tests should cover all lines and branches of the library (see [Test coverage](#coverage)).
+* Operations involving secret data should be tested for being constant time with respect to the secrets (see [src/ctime_tests.c](src/ctime_tests.c)).
+* Local variables containing secret data should be cleared explicitly to try to delete secrets from memory.
+* Use `secp256k1_memcmp_var` instead of `memcmp` (see [#823](https://github.com/bitcoin-core/secp256k1/issues/823)).
+* As a rule of thumb, the default values for configuration options should target standard desktop machines and align with Bitcoin Core's defaults, and the tests should mostly exercise the default configuration (see [#1549](https://github.com/bitcoin-core/secp256k1/issues/1549#issuecomment-2200559257)).
+
+#### Style conventions
+
+* Commits should be atomic and diffs should be easy to read. For this reason, do not mix any formatting fixes or code moves with actual code changes. Make sure each individual commit is hygienic: that it builds successfully on its own without warnings, errors, regressions, or test failures.
+* New code should adhere to the style of existing, in particular surrounding, code. Other than that, we do not enforce strict rules for code formatting.
+* The code conforms to C89. Most notably, that means that only `/* ... */` comments are allowed (no `//` line comments). Moreover, any declarations in a `{ ... }` block (e.g., a function) must appear at the beginning of the block before any statements. When you would like to declare a variable in the middle of a block, you can open a new block:
+    ```C
+    void secp256k_foo(void) {
+        unsigned int x;              /* declaration */
+        int y = 2*x;                 /* declaration */
+        x = 17;                      /* statement */
+        {
+            int a, b;                /* declaration */
+            a = x + y;               /* statement */
+            secp256k_bar(x, &b);     /* statement */
+        }
+    }
+    ```
+* Use `unsigned int` instead of just `unsigned`.
+* Use `void *ptr` instead of `void* ptr`.
+* Arguments of the publicly-facing API must have a specific order defined in [include/secp256k1.h](include/secp256k1.h).
+* User-facing comment lines in headers should be limited to 80 chars if possible.
+* All identifiers in file scope should start with `secp256k1_`.
+* Avoid trailing whitespace.
+
+### Tests
+
+#### Coverage
+
+This library aims to have full coverage of reachable lines and branches.
+
+To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary):
+
+    $ ./configure --enable-coverage
+
+Run the tests:
+
+    $ make check
+
+To create a report, `gcovr` is recommended, as it includes branch coverage reporting:
+
+    $ gcovr --exclude 'src/bench*' --print-summary
+
+To create a HTML report with coloured and annotated source code:
+
+    $ mkdir -p coverage
+    $ gcovr --exclude 'src/bench*' --html --html-details -o coverage/coverage.html
+
+#### Exhaustive tests
+
+There are tests of several functions in which a small group replaces secp256k1.
+These tests are *exhaustive* since they provide all elements and scalars of the small group as input arguments (see [src/tests_exhaustive.c](src/tests_exhaustive.c)).
+
+### Benchmarks
+
+See `src/bench*.c` for examples of benchmarks.

external/secp256k1/Makefile.am

@@ -37,7 +37,6 @@ noinst_HEADERS += src/field_10x26_impl.h
 noinst_HEADERS += src/field_5x52.h
 noinst_HEADERS += src/field_5x52_impl.h
 noinst_HEADERS += src/field_5x52_int128_impl.h
-noinst_HEADERS += src/field_5x52_asm_impl.h
 noinst_HEADERS += src/modinv32.h
 noinst_HEADERS += src/modinv32_impl.h
 noinst_HEADERS += src/modinv64.h
@@ -46,6 +45,7 @@ noinst_HEADERS += src/precomputed_ecmult.h
 noinst_HEADERS += src/precomputed_ecmult_gen.h
 noinst_HEADERS += src/assumptions.h
 noinst_HEADERS += src/checkmem.h
+noinst_HEADERS += src/testutil.h
 noinst_HEADERS += src/util.h
 noinst_HEADERS += src/int128.h
 noinst_HEADERS += src/int128_impl.h
@@ -64,6 +64,8 @@ noinst_HEADERS += src/field.h
 noinst_HEADERS += src/field_impl.h
 noinst_HEADERS += src/bench.h
 noinst_HEADERS += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h
+noinst_HEADERS += src/hsort.h
+noinst_HEADERS += src/hsort_impl.h
 noinst_HEADERS += contrib/lax_der_parsing.h
 noinst_HEADERS += contrib/lax_der_parsing.c
 noinst_HEADERS += contrib/lax_der_privatekey_parsing.h
@@ -153,7 +155,7 @@ endif
 if USE_EXAMPLES
 noinst_PROGRAMS += ecdsa_example
 ecdsa_example_SOURCES = examples/ecdsa.c
-ecdsa_example_CPPFLAGS = -I$(top_srcdir)/include
+ecdsa_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
 ecdsa_example_LDADD = libsecp256k1.la
 ecdsa_example_LDFLAGS = -static
 if BUILD_WINDOWS
@@ -163,7 +165,7 @@ TESTS += ecdsa_example
 if ENABLE_MODULE_ECDH
 noinst_PROGRAMS += ecdh_example
 ecdh_example_SOURCES = examples/ecdh.c
-ecdh_example_CPPFLAGS = -I$(top_srcdir)/include
+ecdh_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
 ecdh_example_LDADD = libsecp256k1.la
 ecdh_example_LDFLAGS = -static
 if BUILD_WINDOWS
@@ -174,7 +176,7 @@ endif
 if ENABLE_MODULE_SCHNORRSIG
 noinst_PROGRAMS += schnorr_example
 schnorr_example_SOURCES = examples/schnorr.c
-schnorr_example_CPPFLAGS = -I$(top_srcdir)/include
+schnorr_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
 schnorr_example_LDADD = libsecp256k1.la
 schnorr_example_LDFLAGS = -static
 if BUILD_WINDOWS
@@ -182,6 +184,28 @@ schnorr_example_LDFLAGS += -lbcrypt
 endif
 TESTS += schnorr_example
 endif
+if ENABLE_MODULE_ELLSWIFT
+noinst_PROGRAMS += ellswift_example
+ellswift_example_SOURCES = examples/ellswift.c
+ellswift_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+ellswift_example_LDADD = libsecp256k1.la
+ellswift_example_LDFLAGS = -static
+if BUILD_WINDOWS
+ellswift_example_LDFLAGS += -lbcrypt
+endif
+TESTS += ellswift_example
+endif
+if ENABLE_MODULE_MUSIG
+noinst_PROGRAMS += musig_example
+musig_example_SOURCES = examples/musig.c
+musig_example_CPPFLAGS = -I$(top_srcdir)/include -DSECP256K1_STATIC
+musig_example_LDADD = libsecp256k1.la
+musig_example_LDFLAGS = -static
+if BUILD_WINDOWS
+musig_example_LDFLAGS += -lbcrypt
+endif
+TESTS += musig_example
+endif
 endif
 
 ### Precomputed tables
@@ -189,11 +213,11 @@ EXTRA_PROGRAMS = precompute_ecmult precompute_ecmult_gen
 CLEANFILES = $(EXTRA_PROGRAMS)
 
 precompute_ecmult_SOURCES = src/precompute_ecmult.c
-precompute_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+precompute_ecmult_CPPFLAGS = $(SECP_CONFIG_DEFINES) -DVERIFY
 precompute_ecmult_LDADD = $(COMMON_LIB)
 
 precompute_ecmult_gen_SOURCES = src/precompute_ecmult_gen.c
-precompute_ecmult_gen_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+precompute_ecmult_gen_CPPFLAGS = $(SECP_CONFIG_DEFINES) -DVERIFY
 precompute_ecmult_gen_LDADD = $(COMMON_LIB)
 
 # See Automake manual, Section "Errors with distclean".
@@ -241,6 +265,7 @@ maintainer-clean-local: clean-testvectors
 ### Additional files to distribute
 EXTRA_DIST = autogen.sh CHANGELOG.md SECURITY.md
 EXTRA_DIST += doc/release-process.md doc/safegcd_implementation.md
+EXTRA_DIST += doc/ellswift.md doc/musig.md
 EXTRA_DIST += examples/EXAMPLES_COPYING
 EXTRA_DIST += sage/gen_exhaustive_groups.sage
 EXTRA_DIST += sage/gen_split_lambda_constants.sage
@@ -267,3 +292,11 @@ endif
 if ENABLE_MODULE_SCHNORRSIG
 include src/modules/schnorrsig/Makefile.am.include
 endif
+
+if ENABLE_MODULE_MUSIG
+include src/modules/musig/Makefile.am.include
+endif
+
+if ENABLE_MODULE_ELLSWIFT
+include src/modules/ellswift/Makefile.am.include
+endif

external/secp256k1/README.md

@@ -1,11 +1,10 @@
 libsecp256k1
 ============
 
-[![Build Status](https://api.cirrus-ci.com/github/bitcoin-core/secp256k1.svg?branch=master)](https://cirrus-ci.com/github/bitcoin-core/secp256k1)
 ![Dependencies: None](https://img.shields.io/badge/dependencies-none-success)
 [![irc.libera.chat #secp256k1](https://img.shields.io/badge/irc.libera.chat-%23secp256k1-success)](https://web.libera.chat/#secp256k1)
 
-Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1.
+High-performance high-assurance C library for digital signatures and other cryptographic primitives on the secp256k1 elliptic curve.
 
 This library is intended to be the highest quality publicly available library for cryptography on the secp256k1 curve. However, the primary focus of its development has been for usage in the Bitcoin system and usage unlike Bitcoin's may be less well tested, verified, or suffer from a less well thought out interface. Correct usage requires some care and consideration that the library is fit for your application's purpose.
 
@@ -21,6 +20,8 @@ Features:
 * Optional module for public key recovery.
 * Optional module for ECDH key exchange.
 * Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki).
+* Optional module for ElligatorSwift key exchange according to [BIP-324](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki).
+* Optional module for MuSig2 Schnorr multi-signatures according to [BIP-327](https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki).
 
 Implementation details
 ----------------------
@@ -34,7 +35,7 @@ Implementation details
   * Expose only higher level interfaces to minimize the API surface and improve application security. ("Be difficult to use insecurely.")
 * Field operations
   * Optimized implementation of arithmetic modulo the curve's field size (2^256 - 0x1000003D1).
-    * Using 5 52-bit limbs (including hand-optimized assembly for x86_64, by Diederik Huys).
+    * Using 5 52-bit limbs
     * Using 10 26-bit limbs (including hand-optimized assembly for 32-bit ARM, by Wladimir J. van der Laan).
       * This is an experimental feature that has not received enough scrutiny to satisfy the standard of quality of this library but is made available for testing and review by the community.
 * Scalar operations
@@ -80,9 +81,9 @@ To maintain a pristine source tree, CMake encourages to perform an out-of-source
 
     $ mkdir build && cd build
     $ cmake ..
-    $ make
-    $ make check  # run the test suite
-    $ sudo make install  # optional
+    $ cmake --build .
+    $ ctest  # run the test suite
+    $ sudo cmake --install .  # optional
 
 To compile optional modules (such as Schnorr signatures), you need to run `cmake` with additional flags (such as `-DSECP256K1_ENABLE_MODULE_SCHNORRSIG=ON`). Run `cmake .. -LH` to see the full list of available flags.
 
@@ -114,31 +115,10 @@ Usage examples can be found in the [examples](examples) directory. To compile th
   * [ECDSA example](examples/ecdsa.c)
   * [Schnorr signatures example](examples/schnorr.c)
   * [Deriving a shared secret (ECDH) example](examples/ecdh.c)
+  * [ElligatorSwift key exchange example](examples/ellswift.c)
 
 To compile the Schnorr signature and ECDH examples, you also need to configure with `--enable-module-schnorrsig` and `--enable-module-ecdh`.
 
-Test coverage
------------
-
-This library aims to have full coverage of the reachable lines and branches.
-
-To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary):
-
-    $ ./configure --enable-coverage
-
-Run the tests:
-
-    $ make check
-
-To create a report, `gcovr` is recommended, as it includes branch coverage reporting:
-
-    $ gcovr --exclude 'src/bench*' --print-summary
-
-To create a HTML report with coloured and annotated source code:
-
-    $ mkdir -p coverage
-    $ gcovr --exclude 'src/bench*' --html --html-details -o coverage/coverage.html
-
 Benchmark
 ------------
 If configured with `--enable-benchmark` (which is the default), binaries for benchmarking the libsecp256k1 functions will be present in the root directory after the build.
@@ -155,3 +135,8 @@ Reporting a vulnerability
 ------------
 
 See [SECURITY.md](SECURITY.md)
+
+Contributing to libsecp256k1
+------------
+
+See [CONTRIBUTING.md](CONTRIBUTING.md)

external/secp256k1/build-aux/m4/bitcoin_secp.m4

@@ -45,6 +45,22 @@ fi
 AC_MSG_RESULT($has_valgrind)
 ])
 
+AC_DEFUN([SECP_MSAN_CHECK], [
+AC_MSG_CHECKING(whether MemorySanitizer is enabled)
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+  #if defined(__has_feature)
+  #  if __has_feature(memory_sanitizer)
+       /* MemorySanitizer is enabled. */
+  #  elif
+  #    error "MemorySanitizer is disabled."
+  #  endif
+  #else
+  #  error "__has_feature is not defined."
+  #endif
+  ]])], [msan_enabled=yes], [msan_enabled=no])
+AC_MSG_RESULT([$msan_enabled])
+])
+
 dnl SECP_TRY_APPEND_CFLAGS(flags, VAR)
 dnl Append flags to VAR if CC accepts them.
 AC_DEFUN([SECP_TRY_APPEND_CFLAGS], [

external/secp256k1/ci/ci.sh

@@ -4,19 +4,21 @@ set -eux
 
 export LC_ALL=C
 
-# Print relevant CI environment to allow reproducing the job outside of CI.
+# Print commit and relevant CI environment to allow reproducing the job outside of CI.
+git show --no-patch
 print_environment() {
     # Turn off -x because it messes up the output
     set +x
     # There are many ways to print variable names and their content. This one
     # does not rely on bash.
     for var in WERROR_CFLAGS MAKEFLAGS BUILD \
-            ECMULTWINDOW ECMULTGENPRECISION ASM WIDEMUL WITH_VALGRIND EXTRAFLAGS \
-            EXPERIMENTAL ECDH RECOVERY SCHNORRSIG \
+            ECMULTWINDOW ECMULTGENKB ASM WIDEMUL WITH_VALGRIND EXTRAFLAGS \
+            EXPERIMENTAL ECDH RECOVERY EXTRAKEYS MUSIG SCHNORRSIG ELLSWIFT \
             SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETESTS\
             EXAMPLES \
             HOST WRAPPER_CMD \
-            CC CFLAGS CPPFLAGS AR NM
+            CC CFLAGS CPPFLAGS AR NM \
+            UBSAN_OPTIONS ASAN_OPTIONS LSAN_OPTIONS
     do
         eval "isset=\${$var+x}"
         if [ -n "$isset" ]; then
@@ -30,19 +32,15 @@ print_environment() {
 }
 print_environment
 
-# Start persistent wineserver if necessary.
-# This speeds up jobs with many invocations of wine (e.g., ./configure with MSVC) tremendously.
-case "$WRAPPER_CMD" in
-    *wine*)
-        # Make sure to shutdown wineserver whenever we exit.
-        trap "wineserver -k || true" EXIT INT HUP
-        # This is apparently only reliable when we run a dummy command such as "hh.exe" afterwards.
-        wineserver -p && wine hh.exe
+env >> test_env.log
+
+# If gcc is requested, assert that it's in fact gcc (and not some symlinked Apple clang).
+case "${CC:-undefined}" in
+    *gcc*)
+        $CC -v 2>&1 | grep -q "gcc version" || exit 1;
         ;;
 esac
 
-env >> test_env.log
-
 if [ -n "${CC+x}" ]; then
     # The MSVC compiler "cl" doesn't understand "-v"
     $CC -v || true
@@ -54,22 +52,55 @@ if [ -n "$WRAPPER_CMD" ]; then
     $WRAPPER_CMD --version
 fi
 
+# Workaround for https://bugs.kde.org/show_bug.cgi?id=452758 (fixed in valgrind 3.20.0).
+case "${CC:-undefined}" in
+    clang*)
+        if [ "$CTIMETESTS" = "yes" ] && [ "$WITH_VALGRIND" = "yes" ]
+        then
+            export CFLAGS="${CFLAGS:+$CFLAGS }-gdwarf-4"
+        else
+            case "$WRAPPER_CMD" in
+                valgrind*)
+                    export CFLAGS="${CFLAGS:+$CFLAGS }-gdwarf-4"
+                    ;;
+            esac
+        fi
+        ;;
+esac
+
 ./autogen.sh
 
 ./configure \
     --enable-experimental="$EXPERIMENTAL" \
     --with-test-override-wide-multiply="$WIDEMUL" --with-asm="$ASM" \
     --with-ecmult-window="$ECMULTWINDOW" \
-    --with-ecmult-gen-precision="$ECMULTGENPRECISION" \
+    --with-ecmult-gen-kb="$ECMULTGENKB" \
     --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \
+    --enable-module-ellswift="$ELLSWIFT" \
+    --enable-module-extrakeys="$EXTRAKEYS" \
     --enable-module-schnorrsig="$SCHNORRSIG" \
+    --enable-module-musig="$MUSIG" \
     --enable-examples="$EXAMPLES" \
     --enable-ctime-tests="$CTIMETESTS" \
     --with-valgrind="$WITH_VALGRIND" \
     --host="$HOST" $EXTRAFLAGS
 
 # We have set "-j<n>" in MAKEFLAGS.
-make
+build_exit_code=0
+make > make.log 2>&1 || build_exit_code=$?
+cat make.log
+if [ $build_exit_code -ne 0 ]; then
+    case "${CC:-undefined}" in
+        *snapshot*)
+            # Ignore internal compiler errors in gcc-snapshot and clang-snapshot
+            grep -e "internal compiler error:" -e "PLEASE submit a bug report" make.log
+            return $?;
+            ;;
+        *)
+            return 1;
+            ;;
+    esac
+fi
 
 # Print information about binaries so that we can see that the architecture is correct
 file *tests* || true

external/secp256k1/ci/linux-debian.Dockerfile

@@ -1,4 +1,17 @@
-FROM debian:stable
+FROM debian:stable-slim
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /root
+
+# A too high maximum number of file descriptors (with the default value
+# inherited from the docker host) can cause issues with some of our tools:
+#  - sanitizers hanging: https://github.com/google/sanitizers/issues/1662 
+#  - valgrind crashing: https://stackoverflow.com/a/75293014
+# This is not be a problem on our CI hosts, but developers who run the image
+# on their machines may run into this (e.g., on Arch Linux), so warn them.
+# (Note that .bashrc is only executed in interactive bash shells.)
+RUN echo 'if [[ $(ulimit -n) -gt 200000 ]]; then echo "WARNING: Very high value reported by \"ulimit -n\". Consider passing \"--ulimit nofile=32768\" to \"docker run\"."; fi' >> /root/.bashrc
 
 RUN dpkg --add-architecture i386 && \
     dpkg --add-architecture s390x && \
@@ -11,27 +24,56 @@ RUN dpkg --add-architecture i386 && \
 RUN apt-get update && apt-get install --no-install-recommends -y \
         git ca-certificates \
         make automake libtool pkg-config dpkg-dev valgrind qemu-user \
-        gcc clang llvm libc6-dbg \
+        gcc clang llvm libclang-rt-dev libc6-dbg \
         g++ \
-        gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libubsan1:i386 libasan6:i386 \
+        gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libubsan1:i386 libasan8:i386 \
         gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \
         gcc-arm-linux-gnueabihf libc6-dev-armhf-cross libc6-dbg:armhf \
-        gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 \
         gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross libc6-dbg:ppc64el \
         gcc-mingw-w64-x86-64-win32 wine64 wine \
         gcc-mingw-w64-i686-win32 wine32 \
-        sagemath
+        python3 && \
+        if ! ( dpkg --print-architecture | grep --quiet "arm64" ) ; then \
+         apt-get install --no-install-recommends -y \
+         gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 ;\
+        fi && \
+        apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root
-# The "wine" package provides a convience wrapper that we need
-RUN apt-get update && apt-get install --no-install-recommends -y \
-        git ca-certificates wine64 wine python3-simplejson python3-six msitools winbind procps && \
-    git clone https://github.com/mstorsjo/msvc-wine && \
-    mkdir /opt/msvc && \
-    python3 msvc-wine/vsdownload.py --accept-license --dest /opt/msvc Microsoft.VisualStudio.Workload.VCTools && \
-    msvc-wine/install.sh /opt/msvc
+# Build and install gcc snapshot
+ARG GCC_SNAPSHOT_MAJOR=15
+RUN apt-get update && apt-get install --no-install-recommends -y wget libgmp-dev libmpfr-dev libmpc-dev flex && \
+    mkdir gcc && cd gcc && \
+    wget --progress=dot:giga --https-only --recursive --accept '*.tar.xz' --level 1 --no-directories "https://gcc.gnu.org/pub/gcc/snapshots/LATEST-${GCC_SNAPSHOT_MAJOR}" && \
+    wget "https://gcc.gnu.org/pub/gcc/snapshots/LATEST-${GCC_SNAPSHOT_MAJOR}/sha512.sum" && \
+    sha512sum --check --ignore-missing sha512.sum && \
+    # We should have downloaded exactly one tar.xz file
+    ls && \
+    [ $(ls *.tar.xz | wc -l) -eq "1" ] && \
+    tar xf *.tar.xz && \
+    mkdir gcc-build && cd gcc-build && \
+    ../*/configure --prefix=/opt/gcc-snapshot --enable-languages=c --disable-bootstrap --disable-multilib --without-isl && \
+    make -j $(nproc) && \
+    make install && \
+    cd ../.. && rm -rf gcc && \
+    ln -s /opt/gcc-snapshot/bin/gcc /usr/bin/gcc-snapshot && \
+    apt-get autoremove -y wget libgmp-dev libmpfr-dev libmpc-dev flex && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install clang snapshot, see https://apt.llvm.org/
+RUN \
+    # Setup GPG keys of LLVM repository
+    apt-get update && apt-get install --no-install-recommends -y wget && \
+    wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \
+    # Add repository for this Debian release
+    . /etc/os-release && echo "deb http://apt.llvm.org/${VERSION_CODENAME} llvm-toolchain-${VERSION_CODENAME} main" >> /etc/apt/sources.list && \
+    apt-get update && \
+    # Determine the version number of the LLVM development branch
+    LLVM_VERSION=$(apt-cache search --names-only '^clang-[0-9]+$' | sort -V | tail -1 | cut -f1 -d" " | cut -f2 -d"-" ) && \
+    # Install
+    apt-get install --no-install-recommends -y "clang-${LLVM_VERSION}" && \
+    # Create symlink
+    ln -s "/usr/bin/clang-${LLVM_VERSION}" /usr/bin/clang-snapshot && \
+    # Clean up
+    apt-get autoremove -y wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# Initialize the wine environment. Wait until the wineserver process has
-# exited before closing the session, to avoid corrupting the wine prefix.
-RUN wine64 wineboot --init && \
-    while (ps -A | grep wineserver) > /dev/null; do sleep 1; done

external/secp256k1/cmake/CheckArm32Assembly.cmake

@@ -1,6 +1,6 @@
 function(check_arm32_assembly)
   try_compile(HAVE_ARM32_ASM
-    ${CMAKE_BINARY_DIR}/check_arm32_assembly
-    SOURCES ${CMAKE_SOURCE_DIR}/cmake/source_arm32.s
+    ${PROJECT_BINARY_DIR}/check_arm32_assembly
+    SOURCES ${PROJECT_SOURCE_DIR}/cmake/source_arm32.s
   )
 endfunction()

external/secp256k1/cmake/CheckMemorySanitizer.cmake

@@ -0,0 +1,18 @@
+include_guard(GLOBAL)
+include(CheckCSourceCompiles)
+
+function(check_memory_sanitizer output)
+  set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+  check_c_source_compiles("
+    #if defined(__has_feature)
+    #  if __has_feature(memory_sanitizer)
+         /* MemorySanitizer is enabled. */
+    #  elif
+    #    error \"MemorySanitizer is disabled.\"
+    #  endif
+    #else
+    #  error \"__has_feature is not defined.\"
+    #endif
+  " HAVE_MSAN)
+  set(${output} ${HAVE_MSAN} PARENT_SCOPE)
+endfunction()

external/secp256k1/cmake/GeneratePkgConfigFile.cmake

@@ -0,0 +1,8 @@
+function(generate_pkg_config_file in_file)
+  set(prefix ${CMAKE_INSTALL_PREFIX})
+  set(exec_prefix \${prefix})
+  set(libdir \${exec_prefix}/${CMAKE_INSTALL_LIBDIR})
+  set(includedir \${prefix}/${CMAKE_INSTALL_INCLUDEDIR})
+  set(PACKAGE_VERSION ${PROJECT_VERSION})
+  configure_file(${in_file} ${PROJECT_NAME}.pc @ONLY)
+endfunction()

external/secp256k1/configure.ac

@@ -4,8 +4,8 @@ AC_PREREQ([2.60])
 # the API. All changes in experimental modules are treated as
 # backwards-compatible and therefore at most increase the minor version.
 define(_PKG_VERSION_MAJOR, 0)
-define(_PKG_VERSION_MINOR, 3)
-define(_PKG_VERSION_PATCH, 2)
+define(_PKG_VERSION_MINOR, 6)
+define(_PKG_VERSION_PATCH, 0)
 define(_PKG_VERSION_IS_RELEASE, true)
 
 # The library version is based on libtool versioning of the ABI. The set of
@@ -13,8 +13,8 @@ define(_PKG_VERSION_IS_RELEASE, true)
 # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
 # All changes in experimental modules are treated as if they don't affect the
 # interface and therefore only increase the revision.
-define(_LIB_VERSION_CURRENT, 2)
-define(_LIB_VERSION_REVISION, 2)
+define(_LIB_VERSION_CURRENT, 5)
+define(_LIB_VERSION_REVISION, 0)
 define(_LIB_VERSION_AGE, 0)
 
 AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1])
@@ -121,13 +121,12 @@ AC_DEFUN([SECP_TRY_APPEND_DEFAULT_CFLAGS], [
     # libtool makes the same assumption internally.
     # Note that "/opt" and "-opt" are equivalent for MSVC; we use "-opt" because "/opt" looks like a path.
     if test x"$GCC" != x"yes" && test x"$build_windows" = x"yes"; then
-      SECP_TRY_APPEND_CFLAGS([-W2 -wd4146], $1) # Moderate warning level, disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned"
-      # We pass -ignore:4217 to the MSVC linker to suppress warning 4217 when
-      # importing variables from a statically linked secp256k1.
-      # (See the libtool manual, section "Windows DLLs" for background.)
-      # Unfortunately, libtool tries to be too clever and strips "-Xlinker arg"
-      # into "arg", so this will be " -Xlinker -ignore:4217" after stripping.
-      LDFLAGS="-Xlinker -Xlinker -Xlinker -ignore:4217 $LDFLAGS" 
+      SECP_TRY_APPEND_CFLAGS([-W3], $1) # Production quality warning level.
+      SECP_TRY_APPEND_CFLAGS([-wd4146], $1) # Disable warning C4146 "unary minus operator applied to unsigned type, result still unsigned".
+      SECP_TRY_APPEND_CFLAGS([-wd4244], $1) # Disable warning C4244 "'conversion' conversion from 'type1' to 'type2', possible loss of data".
+      SECP_TRY_APPEND_CFLAGS([-wd4267], $1) # Disable warning C4267 "'var' : conversion from 'size_t' to 'type', possible loss of data".
+      # Eliminate deprecation warnings for the older, less secure functions.
+      CPPFLAGS="-D_CRT_SECURE_NO_WARNINGS $CPPFLAGS"
     fi
 ])
 SECP_TRY_APPEND_DEFAULT_CFLAGS(SECP_CFLAGS)
@@ -185,6 +184,14 @@ AC_ARG_ENABLE(module_schnorrsig,
     AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module [default=yes]]), [],
     [SECP_SET_DEFAULT([enable_module_schnorrsig], [yes], [yes])])
 
+AC_ARG_ENABLE(module_musig,
+    AS_HELP_STRING([--enable-module-musig],[enable MuSig2 module [default=yes]]), [],
+    [SECP_SET_DEFAULT([enable_module_musig], [yes], [yes])])
+
+AC_ARG_ENABLE(module_ellswift,
+    AS_HELP_STRING([--enable-module-ellswift],[enable ElligatorSwift module [default=yes]]), [],
+    [SECP_SET_DEFAULT([enable_module_ellswift], [yes], [yes])])
+
 AC_ARG_ENABLE(external_default_callbacks,
     AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), [],
     [SECP_SET_DEFAULT([enable_external_default_callbacks], [no], [no])])
@@ -198,25 +205,24 @@ AC_ARG_ENABLE(external_default_callbacks,
 AC_ARG_WITH([test-override-wide-multiply], [] ,[set_widemul=$withval], [set_widemul=auto])
 
 AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm32|no|auto],
-[assembly optimizations to use (experimental: arm32) [default=auto]])],[req_asm=$withval], [req_asm=auto])
+[assembly to use (experimental: arm32) [default=auto]])],[req_asm=$withval], [req_asm=auto])
 
-AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE|auto],
+AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE],
 [window size for ecmult precomputation for verification, specified as integer in range [2..24].]
 [Larger values result in possibly better performance at the cost of an exponentially larger precomputed table.]
 [The table will store 2^(SIZE-1) * 64 bytes of data but can be larger in memory due to platform-specific padding and alignment.]
 [A window size larger than 15 will require you delete the prebuilt precomputed_ecmult.c file so that it can be rebuilt.]
 [For very large window sizes, use "make -j 1" to reduce memory use during compilation.]
-["auto" is a reasonable setting for desktop machines (currently 15). [default=auto]]
+[The default value is a reasonable setting for desktop machines (currently 15). [default=15]]
 )],
-[req_ecmult_window=$withval], [req_ecmult_window=auto])
+[set_ecmult_window=$withval], [set_ecmult_window=15])
 
-AC_ARG_WITH([ecmult-gen-precision], [AS_HELP_STRING([--with-ecmult-gen-precision=2|4|8|auto],
-[Precision bits to tune the precomputed table size for signing.]
-[The size of the table is 32kB for 2 bits, 64kB for 4 bits, 512kB for 8 bits of precision.]
-[A larger table size usually results in possible faster signing.]
-["auto" is a reasonable setting for desktop machines (currently 4). [default=auto]]
+AC_ARG_WITH([ecmult-gen-kb], [AS_HELP_STRING([--with-ecmult-gen-kb=2|22|86],
+[The size of the precomputed table for signing in multiples of 1024 bytes (on typical platforms).]
+[Larger values result in possibly better signing/keygeneration performance at the cost of a larger table.]
+[The default value is a reasonable setting for desktop machines (currently 86). [default=86]]
 )],
-[req_ecmult_gen_precision=$withval], [req_ecmult_gen_precision=auto])
+[set_ecmult_gen_kb=$withval], [set_ecmult_gen_kb=86])
 
 AC_ARG_WITH([valgrind], [AS_HELP_STRING([--with-valgrind=yes|no|auto],
 [Build with extra checks for running inside Valgrind [default=auto]]
@@ -245,6 +251,20 @@ if test x"$enable_ctime_tests" = x"auto"; then
     enable_ctime_tests=$enable_valgrind
 fi
 
+print_msan_notice=no
+if test x"$enable_ctime_tests" = x"yes"; then
+  SECP_MSAN_CHECK
+  # MSan on Clang >=16 reports unitialized memory in function parameters and return values, even if
+  # the uninitalized variable is never actually "used". This is called "eager" checking, and it's
+  # sounds like good idea for normal use of MSan. However, it yields many false positives in the
+  # ctime_tests because many return values depend on secret (i.e., "uninitialized") values, and
+  # we're only interested in detecting branches (which count as "uses") on secret data.
+  if test x"$msan_enabled" = x"yes"; then
+    SECP_TRY_APPEND_CFLAGS([-fno-sanitize-memory-param-retval], SECP_CFLAGS)
+    print_msan_notice=yes
+  fi
+fi
+
 if test x"$enable_coverage" = x"yes"; then
     SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DCOVERAGE=1"
     SECP_CFLAGS="-O0 --coverage $SECP_CFLAGS"
@@ -276,24 +296,24 @@ else
   x86_64)
     SECP_X86_64_ASM_CHECK
     if test x"$has_x86_64_asm" != x"yes"; then
-      AC_MSG_ERROR([x86_64 assembly optimization requested but not available])
+      AC_MSG_ERROR([x86_64 assembly requested but not available])
     fi
     ;;
   arm32)
     SECP_ARM32_ASM_CHECK
     if test x"$has_arm32_asm" != x"yes"; then
-      AC_MSG_ERROR([ARM32 assembly optimization requested but not available])
+      AC_MSG_ERROR([ARM32 assembly requested but not available])
     fi
     ;;
   no)
     ;;
   *)
-    AC_MSG_ERROR([invalid assembly optimization selection])
+    AC_MSG_ERROR([invalid assembly selection])
     ;;
   esac
 fi
 
-# Select assembly optimization
+# Select assembly
 enable_external_asm=no
 
 case $set_asm in
@@ -306,7 +326,7 @@ arm32)
 no)
   ;;
 *)
-  AC_MSG_ERROR([invalid assembly optimizations])
+  AC_MSG_ERROR([invalid assembly selection])
   ;;
 esac
 
@@ -333,14 +353,7 @@ auto)
   ;;
 esac
 
-# Set ecmult window size
-if test x"$req_ecmult_window" = x"auto"; then
-  set_ecmult_window=15
-else
-  set_ecmult_window=$req_ecmult_window
-fi
-
-error_window_size=['window size for ecmult precomputation not an integer in range [2..24] or "auto"']
+error_window_size=['window size for ecmult precomputation not an integer in range [2..24]']
 case $set_ecmult_window in
 ''|*[[!0-9]]*)
   # no valid integer
@@ -355,19 +368,18 @@ case $set_ecmult_window in
   ;;
 esac
 
-# Set ecmult gen precision
-if test x"$req_ecmult_gen_precision" = x"auto"; then
-  set_ecmult_gen_precision=4
-else
-  set_ecmult_gen_precision=$req_ecmult_gen_precision
-fi
-
-case $set_ecmult_gen_precision in
-2|4|8)
-  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DECMULT_GEN_PREC_BITS=$set_ecmult_gen_precision"
+case $set_ecmult_gen_kb in
+2)
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DCOMB_BLOCKS=2 -DCOMB_TEETH=5"
+  ;;
+22)
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DCOMB_BLOCKS=11 -DCOMB_TEETH=6"
+  ;;
+86)
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DCOMB_BLOCKS=43 -DCOMB_TEETH=6"
   ;;
 *)
-  AC_MSG_ERROR(['ecmult gen precision not 2, 4, 8 or "auto"'])
+  AC_MSG_ERROR(['ecmult gen table size not 2, 22 or 86'])
   ;;
 esac
 
@@ -384,23 +396,38 @@ SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS"
 ### Handle module options
 ###
 
-if test x"$enable_module_ecdh" = x"yes"; then
-  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1"
+# Processing must be done in a reverse topological sorting of the dependency graph
+# (dependent module first).
+if test x"$enable_module_ellswift" = x"yes"; then
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1"
+fi
+
+if test x"$enable_module_musig" = x"yes"; then
+  if test x"$enable_module_schnorrsig" = x"no"; then
+    AC_MSG_ERROR([Module dependency error: You have disabled the schnorrsig module explicitly, but it is required by the musig module.])
+  fi
+  enable_module_schnorrsig=yes
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_MUSIG=1"
+fi
+
+if test x"$enable_module_schnorrsig" = x"yes"; then
+  if test x"$enable_module_extrakeys" = x"no"; then
+    AC_MSG_ERROR([Module dependency error: You have disabled the extrakeys module explicitly, but it is required by the schnorrsig module.])
+  fi
+  enable_module_extrakeys=yes
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1"
+fi
+
+if test x"$enable_module_extrakeys" = x"yes"; then
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_EXTRAKEYS=1"
 fi
 
 if test x"$enable_module_recovery" = x"yes"; then
   SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1"
 fi
 
-if test x"$enable_module_schnorrsig" = x"yes"; then
-  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1"
-  enable_module_extrakeys=yes
-fi
-
-# Test if extrakeys is set after the schnorrsig module to allow the schnorrsig
-# module to set enable_module_extrakeys=yes
-if test x"$enable_module_extrakeys" = x"yes"; then
-  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_EXTRAKEYS=1"
+if test x"$enable_module_ecdh" = x"yes"; then
+  SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1"
 fi
 
 if test x"$enable_external_default_callbacks" = x"yes"; then
@@ -411,14 +438,9 @@ fi
 ### Check for --enable-experimental if necessary
 ###
 
-if test x"$enable_experimental" = x"yes"; then
-  AC_MSG_NOTICE([******])
-  AC_MSG_NOTICE([WARNING: experimental build])
-  AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.])
-  AC_MSG_NOTICE([******])
-else
+if test x"$enable_experimental" = x"no"; then
   if test x"$set_asm" = x"arm32"; then
-    AC_MSG_ERROR([ARM32 assembly optimization is experimental. Use --enable-experimental to allow.])
+    AC_MSG_ERROR([ARM32 assembly is experimental. Use --enable-experimental to allow.])
   fi
 fi
 
@@ -439,6 +461,8 @@ AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"])
 AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"])
 AM_CONDITIONAL([ENABLE_MODULE_EXTRAKEYS], [test x"$enable_module_extrakeys" = x"yes"])
 AM_CONDITIONAL([ENABLE_MODULE_SCHNORRSIG], [test x"$enable_module_schnorrsig" = x"yes"])
+AM_CONDITIONAL([ENABLE_MODULE_MUSIG], [test x"$enable_module_musig" = x"yes"])
+AM_CONDITIONAL([ENABLE_MODULE_ELLSWIFT], [test x"$enable_module_ellswift" = x"yes"])
 AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$enable_external_asm" = x"yes"])
 AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm32"])
 AM_CONDITIONAL([BUILD_WINDOWS], [test "$build_windows" = "yes"])
@@ -460,10 +484,12 @@ echo "  module ecdh             = $enable_module_ecdh"
 echo "  module recovery         = $enable_module_recovery"
 echo "  module extrakeys        = $enable_module_extrakeys"
 echo "  module schnorrsig       = $enable_module_schnorrsig"
+echo "  module musig            = $enable_module_musig"
+echo "  module ellswift         = $enable_module_ellswift"
 echo
 echo "  asm                     = $set_asm"
 echo "  ecmult window size      = $set_ecmult_window"
-echo "  ecmult gen prec. bits   = $set_ecmult_gen_precision"
+echo "  ecmult gen table size   = $set_ecmult_gen_kb KiB"
 # Hide test-only options unless they're used.
 if test x"$set_widemul" != xauto; then
 echo "  wide multiplication     = $set_widemul"
@@ -475,3 +501,17 @@ echo "  CPPFLAGS                = $CPPFLAGS"
 echo "  SECP_CFLAGS             = $SECP_CFLAGS"
 echo "  CFLAGS                  = $CFLAGS"
 echo "  LDFLAGS                 = $LDFLAGS"
+
+if test x"$print_msan_notice" = x"yes"; then
+  echo
+  echo "Note:"
+  echo "  MemorySanitizer detected, tried to add -fno-sanitize-memory-param-retval to SECP_CFLAGS"
+  echo "  to avoid false positives in ctime_tests. Pass --disable-ctime-tests to avoid this."
+fi
+
+if test x"$enable_experimental" = x"yes"; then
+  echo
+  echo "WARNING: Experimental build"
+  echo "  Experimental features do not have stable APIs or properties, and may not be safe for"
+  echo "  production use."
+fi

external/secp256k1/contrib/lax_der_parsing.h

@@ -67,8 +67,8 @@ extern "C" {
  *
  *  Returns: 1 when the signature could be parsed, 0 otherwise.
  *  Args: ctx:      a secp256k1 context object
- *  Out:  sig:      a pointer to a signature object
- *  In:   input:    a pointer to the signature to be parsed
+ *  Out:  sig:      pointer to a signature object
+ *  In:   input:    pointer to the signature to be parsed
  *        inputlen: the length of the array pointed to be input
  *
  *  This function will accept any valid DER encoded signature, even if the

external/secp256k1/doc/ellswift.md

@@ -0,0 +1,483 @@
+# ElligatorSwift for secp256k1 explained
+
+In this document we explain how the `ellswift` module implementation is related to the
+construction in the
+["SwiftEC: Shallue–van de Woestijne Indifferentiable Function To Elliptic Curves"](https://eprint.iacr.org/2022/759)
+paper by Jorge Chávez-Saab, Francisco Rodríguez-Henríquez, and Mehdi Tibouchi.
+
+* [1. Introduction](#1-introduction)
+* [2. The decoding function](#2-the-decoding-function)
+  + [2.1 Decoding for `secp256k1`](#21-decoding-for-secp256k1)
+* [3. The encoding function](#3-the-encoding-function)
+  + [3.1 Switching to *v, w* coordinates](#31-switching-to-v-w-coordinates)
+  + [3.2 Avoiding computing all inverses](#32-avoiding-computing-all-inverses)
+  + [3.3 Finding the inverse](#33-finding-the-inverse)
+  + [3.4 Dealing with special cases](#34-dealing-with-special-cases)
+  + [3.5 Encoding for `secp256k1`](#35-encoding-for-secp256k1)
+* [4. Encoding and decoding full *(x, y)* coordinates](#4-encoding-and-decoding-full-x-y-coordinates)
+  + [4.1 Full *(x, y)* coordinates for `secp256k1`](#41-full-x-y-coordinates-for-secp256k1)
+
+## 1. Introduction
+
+The `ellswift` module effectively introduces a new 64-byte public key format, with the property
+that (uniformly random) public keys can be encoded as 64-byte arrays which are computationally
+indistinguishable from uniform byte arrays. The module provides functions to convert public keys
+from and to this format, as well as convenience functions for key generation and ECDH that operate
+directly on ellswift-encoded keys.
+
+The encoding consists of the concatenation of two (32-byte big endian) encoded field elements $u$
+and $t.$ Together they encode an x-coordinate on the curve $x$, or (see further) a full point $(x, y)$ on
+the curve.
+
+**Decoding** consists of decoding the field elements $u$ and $t$ (values above the field size $p$
+are taken modulo $p$), and then evaluating $F_u(t)$, which for every $u$ and $t$ results in a valid
+x-coordinate on the curve. The functions $F_u$ will be defined in [Section 2](#2-the-decoding-function).
+
+**Encoding** a given $x$ coordinate is conceptually done as follows:
+* Loop:
+  * Pick a uniformly random field element $u.$
+  * Compute the set $L = F_u^{-1}(x)$ of $t$ values for which $F_u(t) = x$, which may have up to *8* elements.
+  * With probability $1 - \dfrac{\\#L}{8}$, restart the loop.
+  * Select a uniformly random $t \in L$ and return $(u, t).$
+
+This is the *ElligatorSwift* algorithm, here given for just x-coordinates. An extension to full
+$(x, y)$ points will be given in [Section 4](#4-encoding-and-decoding-full-x-y-coordinates).
+The algorithm finds a uniformly random $(u, t)$ among (almost all) those
+for which $F_u(t) = x.$ Section 3.2 in the paper proves that the number of such encodings for
+almost all x-coordinates on the curve (all but at most 39) is close to two times the field size
+(specifically, it lies in the range $2q \pm (22\sqrt{q} + O(1))$, where $q$ is the size of the field).
+
+## 2. The decoding function
+
+First some definitions:
+* $\mathbb{F}$ is the finite field of size $q$, of characteristic 5 or more, and $q \equiv 1 \mod 3.$
+  * For `secp256k1`, $q = 2^{256} - 2^{32} - 977$, which satisfies that requirement.
+* Let $E$ be the elliptic curve of points $(x, y) \in \mathbb{F}^2$ for which $y^2 = x^3 + ax + b$, with $a$ and $b$
+  public constants, for which $\Delta_E = -16(4a^3 + 27b^2)$ is a square, and at least one of $(-b \pm \sqrt{-3 \Delta_E} / 36)/2$ is a square.
+  This implies that the order of $E$ is either odd, or a multiple of *4*.
+  If $a=0$, this condition is always fulfilled.
+  * For `secp256k1`, $a=0$ and $b=7.$
+* Let the function $g(x) = x^3 + ax + b$, so the $E$ curve equation is also $y^2 = g(x).$
+* Let the function $h(x) = 3x^3 + 4a.$
+* Define $V$ as the set of solutions $(x_1, x_2, x_3, z)$ to $z^2 = g(x_1)g(x_2)g(x_3).$
+* Define $S_u$ as the set of solutions $(X, Y)$ to $X^2 + h(u)Y^2 = -g(u)$ and $Y \neq 0.$
+* $P_u$ is a function from $\mathbb{F}$ to $S_u$ that will be defined below.
+* $\psi_u$ is a function from $S_u$ to $V$ that will be defined below.
+
+**Note**: In the paper:
+* $F_u$ corresponds to $F_{0,u}$ there.
+* $P_u(t)$ is called $P$ there.
+* All $S_u$ sets together correspond to $S$ there.
+* All $\psi_u$ functions together (operating on elements of $S$) correspond to $\psi$ there.
+
+Note that for $V$, the left hand side of the equation $z^2$ is square, and thus the right
+hand must also be square. As multiplying non-squares results in a square in $\mathbb{F}$,
+out of the three right-hand side factors an even number must be non-squares.
+This implies that exactly *1* or exactly *3* out of
+$\\{g(x_1), g(x_2), g(x_3)\\}$ must be square, and thus that for any $(x_1,x_2,x_3,z) \in V$,
+at least one of $\\{x_1, x_2, x_3\\}$ must be a valid x-coordinate on $E.$ There is one exception
+to this, namely when $z=0$, but even then one of the three values is a valid x-coordinate.
+
+**Define** the decoding function $F_u(t)$ as:
+* Let $(x_1, x_2, x_3, z) = \psi_u(P_u(t)).$
+* Return the first element $x$ of $(x_3, x_2, x_1)$ which is a valid x-coordinate on $E$ (i.e., $g(x)$ is square).
+
+$P_u(t) = (X(u, t), Y(u, t))$, where:
+
+$$
+\begin{array}{lcl}
+X(u, t) & = & \left\\{\begin{array}{ll}
+  \dfrac{g(u) - t^2}{2t} & a = 0 \\
+  \dfrac{g(u) + h(u)(Y_0(u) - X_0(u)t)^2}{X_0(u)(1 + h(u)t^2)} & a \neq 0
+\end{array}\right. \\
+Y(u, t) & = & \left\\{\begin{array}{ll}
+  \dfrac{X(u, t) + t}{u \sqrt{-3}} = \dfrac{g(u) + t^2}{2tu\sqrt{-3}} & a = 0 \\
+  Y_0(u) + t(X(u, t) - X_0(u)) & a \neq 0
+\end{array}\right.
+\end{array}
+$$
+
+$P_u(t)$ is defined:
+* For $a=0$, unless:
+  * $u = 0$ or $t = 0$ (division by zero)
+  * $g(u) = -t^2$ (would give $Y=0$).
+* For $a \neq 0$, unless:
+  * $X_0(u) = 0$ or $h(u)t^2 = -1$ (division by zero)
+  * $Y_0(u) (1 - h(u)t^2) = 2X_0(u)t$ (would give $Y=0$).
+
+The functions $X_0(u)$ and $Y_0(u)$ are defined in Appendix A of the paper, and depend on various properties of $E.$
+
+The function $\psi_u$ is the same for all curves: $\psi_u(X, Y) = (x_1, x_2, x_3, z)$, where:
+
+$$
+\begin{array}{lcl}
+  x_1 & = & \dfrac{X}{2Y} - \dfrac{u}{2} && \\
+  x_2 & = & -\dfrac{X}{2Y} - \dfrac{u}{2} && \\
+  x_3 & = & u + 4Y^2 && \\
+  z   & = & \dfrac{g(x_3)}{2Y}(u^2 + ux_1 + x_1^2 + a) = \dfrac{-g(u)g(x_3)}{8Y^3}
+\end{array}
+$$
+
+### 2.1 Decoding for `secp256k1`
+
+Put together and specialized for $a=0$ curves, decoding $(u, t)$ to an x-coordinate is:
+
+**Define** $F_u(t)$ as:
+* Let $X = \dfrac{u^3 + b - t^2}{2t}.$
+* Let $Y = \dfrac{X + t}{u\sqrt{-3}}.$
+* Return the first $x$ in $(u + 4Y^2, \dfrac{-X}{2Y} - \dfrac{u}{2}, \dfrac{X}{2Y} - \dfrac{u}{2})$ for which $g(x)$ is square.
+
+To make sure that every input decodes to a valid x-coordinate, we remap the inputs in case
+$P_u$ is not defined (when $u=0$, $t=0$, or $g(u) = -t^2$):
+
+**Define** $F_u(t)$ as:
+* Let $u'=u$ if $u \neq 0$; $1$ otherwise (guaranteeing $u' \neq 0$).
+* Let $t'=t$ if $t \neq 0$; $1$ otherwise (guaranteeing $t' \neq 0$).
+* Let $t''=t'$ if $g(u') \neq -t'^2$; $2t'$ otherwise (guaranteeing $t'' \neq 0$ and $g(u') \neq -t''^2$).
+* Let $X = \dfrac{u'^3 + b - t''^2}{2t''}.$
+* Let $Y = \dfrac{X + t''}{u'\sqrt{-3}}.$
+* Return the first $x$ in $(u' + 4Y^2, \dfrac{-X}{2Y} - \dfrac{u'}{2}, \dfrac{X}{2Y} - \dfrac{u'}{2})$ for which $x^3 + b$ is square.
+
+The choices here are not strictly necessary. Just returning a fixed constant in any of the undefined cases would suffice,
+but the approach here is simple enough and gives fairly uniform output even in these cases.
+
+**Note**: in the paper these conditions result in $\infty$ as output, due to the use of projective coordinates there.
+We wish to avoid the need for callers to deal with this special case.
+
+This is implemented in `secp256k1_ellswift_xswiftec_frac_var` (which decodes to an x-coordinate represented as a fraction), and
+in `secp256k1_ellswift_xswiftec_var` (which outputs the actual x-coordinate).
+
+## 3. The encoding function
+
+To implement $F_u^{-1}(x)$, the function to find the set of inverses $t$ for which $F_u(t) = x$, we have to reverse the process:
+* Find all the $(X, Y) \in S_u$ that could have given rise to $x$, through the $x_1$, $x_2$, or $x_3$ formulas in $\psi_u.$
+* Map those $(X, Y)$ solutions to $t$ values using $P_u^{-1}(X, Y).$
+* For each of the found $t$ values, verify that $F_u(t) = x.$
+* Return the remaining $t$ values.
+
+The function $P_u^{-1}$, which finds $t$ given $(X, Y) \in S_u$, is significantly simpler than $P_u:$
+
+$$
+P_u^{-1}(X, Y) = \left\\{\begin{array}{ll}
+Yu\sqrt{-3} - X & a = 0 \\
+\dfrac{Y-Y_0(u)}{X-X_0(u)} & a \neq 0 \land X \neq X_0(u) \\
+\dfrac{-X_0(u)}{h(u)Y_0(u)} & a \neq 0 \land X = X_0(u) \land Y = Y_0(u)
+\end{array}\right.
+$$
+
+The third step above, verifying that $F_u(t) = x$, is necessary because for the $(X, Y)$ values found through the $x_1$ and $x_2$ expressions,
+it is possible that decoding through $\psi_u(X, Y)$ yields a valid $x_3$ on the curve, which would take precedence over the
+$x_1$ or $x_2$ decoding. These $(X, Y)$ solutions must be rejected.
+
+Since we know that exactly one or exactly three out of $\\{x_1, x_2, x_3\\}$ are valid x-coordinates for any $t$,
+the case where either $x_1$ or $x_2$ is valid and in addition also $x_3$ is valid must mean that all three are valid.
+This means that instead of checking whether $x_3$ is on the curve, it is also possible to check whether the other one out of
+$x_1$ and $x_2$ is on the curve. This is significantly simpler, as it turns out.
+
+Observe that $\psi_u$ guarantees that $x_1 + x_2 = -u.$ So given either $x = x_1$ or $x = x_2$, the other one of the two can be computed as
+$-u - x.$ Thus, when encoding $x$ through the $x_1$ or $x_2$ expressions, one can simply check whether $g(-u-x)$ is a square,
+and if so, not include the corresponding $t$ values in the returned set. As this does not need $X$, $Y$, or $t$, this condition can be determined
+before those values are computed.
+
+It is not possible that an encoding found through the $x_1$ expression decodes to a different valid x-coordinate using $x_2$ (which would
+take precedence), for the same reason: if both $x_1$ and $x_2$ decodings were valid, $x_3$ would be valid as well, and thus take
+precedence over both. Because of this, the $g(-u-x)$ being square test for $x_1$ and $x_2$ is the only test necessary to guarantee the found $t$
+values round-trip back to the input $x$ correctly. This is the reason for choosing the $(x_3, x_2, x_1)$ precedence order in the decoder;
+any order which does not place $x_3$ first requires more complicated round-trip checks in the encoder.
+
+### 3.1 Switching to *v, w* coordinates
+
+Before working out the formulas for all this, we switch to different variables for $S_u.$ Let $v = (X/Y - u)/2$, and
+$w = 2Y.$ Or in the other direction, $X = w(u/2 + v)$ and $Y = w/2:$
+* $S_u'$ becomes the set of $(v, w)$ for which $w^2 (u^2 + uv + v^2 + a) = -g(u)$ and $w \neq 0.$
+* For $a=0$ curves, $P_u^{-1}$ can be stated for $(v,w)$ as $P_u^{'-1}(v, w) = w\left(\frac{\sqrt{-3}-1}{2}u - v\right).$
+* $\psi_u$ can be stated for $(v, w)$ as $\psi_u'(v, w) = (x_1, x_2, x_3, z)$, where
+
+$$
+\begin{array}{lcl}
+  x_1 & = & v \\
+  x_2 & = & -u - v \\
+  x_3 & = & u + w^2 \\
+  z   & = & \dfrac{g(x_3)}{w}(u^2 + uv + v^2 + a) = \dfrac{-g(u)g(x_3)}{w^3}
+\end{array}
+$$
+
+We can now write the expressions for finding $(v, w)$ given $x$ explicitly, by solving each of the $\\{x_1, x_2, x_3\\}$
+expressions for $v$ or $w$, and using the $S_u'$ equation to find the other variable:
+* Assuming $x = x_1$, we find $v = x$ and $w = \pm\sqrt{-g(u)/(u^2 + uv + v^2 + a)}$ (two solutions).
+* Assuming $x = x_2$, we find $v = -u-x$ and $w = \pm\sqrt{-g(u)/(u^2 + uv + v^2 + a)}$ (two solutions).
+* Assuming $x = x_3$, we find $w = \pm\sqrt{x-u}$ and $v = -u/2 \pm \sqrt{-w^2(4g(u) + w^2h(u))}/(2w^2)$ (four solutions).
+
+### 3.2 Avoiding computing all inverses
+
+The *ElligatorSwift* algorithm as stated in Section 1 requires the computation of $L = F_u^{-1}(x)$ (the
+set of all $t$ such that $(u, t)$ decode to $x$) in full. This is unnecessary.
+
+Observe that the procedure of restarting with probability $(1 - \frac{\\#L}{8})$ and otherwise returning a
+uniformly random element from $L$ is actually equivalent to always padding $L$ with $\bot$ values up to length 8,
+picking a uniformly random element from that, restarting whenever $\bot$ is picked:
+
+**Define** *ElligatorSwift(x)* as:
+* Loop:
+  * Pick a uniformly random field element $u.$
+  * Compute the set $L = F_u^{-1}(x).$
+  * Let $T$ be the 8-element vector consisting of the elements of $L$, plus $8 - \\#L$ times $\\{\bot\\}.$
+  * Select a uniformly random $t \in T.$
+  * If $t \neq \bot$, return $(u, t)$; restart loop otherwise.
+
+Now notice that the order of elements in $T$ does not matter, as all we do is pick a uniformly
+random element in it, so we do not need to have all $\bot$ values at the end.
+As we have 8 distinct formulas for finding $(v, w)$ (taking the variants due to $\pm$ into account),
+we can associate every index in $T$ with exactly one of those formulas, making sure that:
+* Formulas that yield no solutions (due to division by zero or non-existing square roots) or invalid solutions are made to return $\bot.$
+* For the $x_1$ and $x_2$ cases, if $g(-u-x)$ is a square, $\bot$ is returned instead (the round-trip check).
+* In case multiple formulas would return the same non- $\bot$ result, all but one of those must be turned into $\bot$ to avoid biasing those.
+
+The last condition above only occurs with negligible probability for cryptographically-sized curves, but is interesting
+to take into account as it allows exhaustive testing in small groups. See [Section 3.4](#34-dealing-with-special-cases)
+for an analysis of all the negligible cases.
+
+If we define $T = (G_{0,u}(x), G_{1,u}(x), \ldots, G_{7,u}(x))$, with each $G_{i,u}$ matching one of the formulas,
+the loop can be simplified to only compute one of the inverses instead of all of them:
+
+**Define** *ElligatorSwift(x)* as:
+* Loop:
+  * Pick a uniformly random field element $u.$
+  * Pick a uniformly random integer $c$ in $[0,8).$
+  * Let $t = G_{c,u}(x).$
+  * If $t \neq \bot$, return $(u, t)$; restart loop otherwise.
+
+This is implemented in `secp256k1_ellswift_xelligatorswift_var`.
+
+### 3.3 Finding the inverse
+
+To implement $G_{c,u}$, we map $c=0$ to the $x_1$ formula, $c=1$ to the $x_2$ formula, and $c=2$ and $c=3$ to the $x_3$ formula.
+Those are then repeated as $c=4$ through $c=7$ for the other sign of $w$ (noting that in each formula, $w$ is a square root of some expression).
+Ignoring the negligible cases, we get:
+
+**Define** $G_{c,u}(x)$ as:
+* If $c \in \\{0, 1, 4, 5\\}$ (for $x_1$ and $x_2$ formulas):
+  * If $g(-u-x)$ is square, return $\bot$ (as $x_3$ would be valid and take precedence).
+  * If $c \in \\{0, 4\\}$ (the $x_1$ formula) let $v = x$, otherwise let $v = -u-x$ (the $x_2$ formula)
+  * Let $s = -g(u)/(u^2 + uv + v^2 + a)$ (using $s = w^2$ in what follows).
+* Otherwise, when $c \in \\{2, 3, 6, 7\\}$ (for $x_3$ formulas):
+  * Let $s = x-u.$
+  * Let $r = \sqrt{-s(4g(u) + sh(u))}.$
+  * Let $v = (r/s - u)/2$ if $c \in \\{3, 7\\}$; $(-r/s - u)/2$ otherwise.
+* Let $w = \sqrt{s}.$
+* Depending on $c:$
+  * If $c \in \\{0, 1, 2, 3\\}:$ return $P_u^{'-1}(v, w).$
+  * If $c \in \\{4, 5, 6, 7\\}:$ return $P_u^{'-1}(v, -w).$
+
+Whenever a square root of a non-square is taken, $\bot$ is returned; for both square roots this happens with roughly
+50% on random inputs. Similarly, when a division by 0 would occur, $\bot$ is returned as well; this will only happen
+with negligible probability. A division by 0 in the first branch in fact cannot occur at all, because $u^2 + uv + v^2 + a = 0$
+implies $g(-u-x) = g(x)$ which would mean the $g(-u-x)$ is square condition has triggered
+and $\bot$ would have been returned already.
+
+**Note**: In the paper, the $case$ variable corresponds roughly to the $c$ above, but only takes on 4 possible values (1 to 4).
+The conditional negation of $w$ at the end is done randomly, which is equivalent, but makes testing harder. We choose to
+have the $G_{c,u}$ be deterministic, and capture all choices in $c.$
+
+Now observe that the $c \in \\{1, 5\\}$ and $c \in \\{3, 7\\}$ conditions effectively perform the same $v \rightarrow -u-v$
+transformation. Furthermore, that transformation has no effect on $s$ in the first branch
+as $u^2 + ux + x^2 + a = u^2 + u(-u-x) + (-u-x)^2 + a.$ Thus we can extract it out and move it down:
+
+**Define** $G_{c,u}(x)$ as:
+* If $c \in \\{0, 1, 4, 5\\}:$
+  * If $g(-u-x)$ is square, return $\bot.$
+  * Let $s = -g(u)/(u^2 + ux + x^2 + a).$
+  * Let $v = x.$
+* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$
+  * Let $s = x-u.$
+  * Let $r = \sqrt{-s(4g(u) + sh(u))}.$
+  * Let $v = (r/s - u)/2.$
+* Let $w = \sqrt{s}.$
+* Depending on $c:$
+  * If $c \in \\{0, 2\\}:$ return $P_u^{'-1}(v, w).$
+  * If $c \in \\{1, 3\\}:$ return $P_u^{'-1}(-u-v, w).$
+  * If $c \in \\{4, 6\\}:$ return $P_u^{'-1}(v, -w).$
+  * If $c \in \\{5, 7\\}:$ return $P_u^{'-1}(-u-v, -w).$
+
+This shows there will always be exactly 0, 4, or 8 $t$ values for a given $(u, x)$ input.
+There can be 0, 1, or 2 $(v, w)$ pairs before invoking $P_u^{'-1}$, and each results in 4 distinct $t$ values.
+
+### 3.4 Dealing with special cases
+
+As mentioned before there are a few cases to deal with which only happen in a negligibly small subset of inputs.
+For cryptographically sized fields, if only random inputs are going to be considered, it is unnecessary to deal with these. Still, for completeness
+we analyse them here. They generally fall into two categories: cases in which the encoder would produce $t$ values that
+do not decode back to $x$ (or at least cannot guarantee that they do), and cases in which the encoder might produce the same
+$t$ value for multiple $c$ inputs (thereby biasing that encoding):
+
+* In the branch for $x_1$ and $x_2$ (where $c \in \\{0, 1, 4, 5\\}$):
+  * When $g(u) = 0$, we would have $s=w=Y=0$, which is not on $S_u.$ This is only possible on even-ordered curves.
+    Excluding this also removes the one condition under which the simplified check for $x_3$ on the curve
+    fails (namely when $g(x_1)=g(x_2)=0$ but $g(x_3)$ is not square).
+    This does exclude some valid encodings: when both $g(u)=0$ and $u^2+ux+x^2+a=0$ (also implying $g(x)=0$),
+    the $S_u'$ equation degenerates to $0 = 0$, and many valid $t$ values may exist. Yet, these cannot be targeted uniformly by the
+    encoder anyway as there will generally be more than 8.
+  * When $g(x) = 0$, the same $t$ would be produced as in the $x_3$ branch (where $c \in \\{2, 3, 6, 7\\}$) which we give precedence
+    as it can deal with $g(u)=0$.
+    This is again only possible on even-ordered curves.
+* In the branch for $x_3$ (where $c \in \\{2, 3, 6, 7\\}$):
+  * When $s=0$, a division by zero would occur.
+  * When $v = -u-v$ and $c \in \\{3, 7\\}$, the same $t$ would be returned as in the $c \in \\{2, 6\\}$ cases.
+    It is equivalent to checking whether $r=0$.
+    This cannot occur in the $x_1$ or $x_2$ branches, as it would trigger the $g(-u-x)$ is square condition.
+    A similar concern for $w = -w$ does not exist, as $w=0$ is already impossible in both branches: in the first
+    it requires $g(u)=0$ which is already outlawed on even-ordered curves and impossible on others; in the second it would trigger division by zero.
+* Curve-specific special cases also exist that need to be rejected, because they result in $(u,t)$ which is invalid to the decoder, or because of division by zero in the encoder:
+  * For $a=0$ curves, when $u=0$ or when $t=0$. The latter can only be reached by the encoder when $g(u)=0$, which requires an even-ordered curve.
+  * For $a \neq 0$ curves, when $X_0(u)=0$, when $h(u)t^2 = -1$, or when $w(u + 2v) = 2X_0(u)$ while also either $w \neq 2Y_0(u)$ or $h(u)=0$.
+
+**Define** a version of $G_{c,u}(x)$ which deals with all these cases:
+* If $a=0$ and $u=0$, return $\bot.$
+* If $a \neq 0$ and $X_0(u)=0$, return $\bot.$
+* If $c \in \\{0, 1, 4, 5\\}:$
+  * If $g(u) = 0$ or $g(x) = 0$, return $\bot$ (even curves only).
+  * If $g(-u-x)$ is square, return $\bot.$
+  * Let $s = -g(u)/(u^2 + ux + x^2 + a)$ (cannot cause division by zero).
+  * Let $v = x.$
+* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$
+  * Let $s = x-u.$
+  * Let $r = \sqrt{-s(4g(u) + sh(u))}$; return $\bot$ if not square.
+  * If $c \in \\{3, 7\\}$ and $r=0$, return $\bot.$
+  * If $s = 0$, return $\bot.$
+  * Let $v = (r/s - u)/2.$
+* Let $w = \sqrt{s}$; return $\bot$ if not square.
+* If $a \neq 0$ and $w(u+2v) = 2X_0(u)$ and either $w \neq 2Y_0(u)$ or $h(u) = 0$, return $\bot.$
+* Depending on $c:$
+  * If $c \in \\{0, 2\\}$, let $t = P_u^{'-1}(v, w).$
+  * If $c \in \\{1, 3\\}$, let $t = P_u^{'-1}(-u-v, w).$
+  * If $c \in \\{4, 6\\}$, let $t = P_u^{'-1}(v, -w).$
+  * If $c \in \\{5, 7\\}$, let $t = P_u^{'-1}(-u-v, -w).$
+* If $a=0$ and $t=0$, return $\bot$ (even curves only).
+* If $a \neq 0$ and $h(u)t^2 = -1$, return $\bot.$
+* Return $t.$
+
+Given any $u$, using this algorithm over all $x$ and $c$ values, every $t$ value will be reached exactly once,
+for an $x$ for which $F_u(t) = x$ holds, except for these cases that will not be reached:
+* All cases where $P_u(t)$ is not defined:
+  * For $a=0$ curves, when $u=0$, $t=0$, or $g(u) = -t^2.$
+  * For $a \neq 0$ curves, when $h(u)t^2 = -1$, $X_0(u) = 0$, or $Y_0(u) (1 - h(u) t^2) = 2X_0(u)t.$
+* When $g(u)=0$, the potentially many $t$ values that decode to an $x$ satisfying $g(x)=0$ using the $x_2$ formula. These were excluded by the $g(u)=0$ condition in the $c \in \\{0, 1, 4, 5\\}$ branch.
+
+These cases form a negligible subset of all $(u, t)$ for cryptographically sized curves.
+
+### 3.5 Encoding for `secp256k1`
+
+Specialized for odd-ordered $a=0$ curves:
+
+**Define** $G_{c,u}(x)$ as:
+* If $u=0$, return $\bot.$
+* If $c \in \\{0, 1, 4, 5\\}:$
+  * If $(-u-x)^3 + b$ is square, return $\bot$
+  * Let $s = -(u^3 + b)/(u^2 + ux + x^2)$ (cannot cause division by 0).
+  * Let $v = x.$
+* Otherwise, when $c \in \\{2, 3, 6, 7\\}:$
+  * Let $s = x-u.$
+  * Let $r = \sqrt{-s(4(u^3 + b) + 3su^2)}$; return $\bot$ if not square.
+  * If $c \in \\{3, 7\\}$ and $r=0$, return $\bot.$
+  * If $s = 0$, return $\bot.$
+  * Let $v = (r/s - u)/2.$
+* Let $w = \sqrt{s}$; return $\bot$ if not square.
+* Depending on $c:$
+  * If $c \in \\{0, 2\\}:$ return $w(\frac{\sqrt{-3}-1}{2}u - v).$
+  * If $c \in \\{1, 3\\}:$ return $w(\frac{\sqrt{-3}+1}{2}u + v).$
+  * If $c \in \\{4, 6\\}:$ return $w(\frac{-\sqrt{-3}+1}{2}u + v).$
+  * If $c \in \\{5, 7\\}:$ return $w(\frac{-\sqrt{-3}-1}{2}u - v).$
+
+This is implemented in `secp256k1_ellswift_xswiftec_inv_var`.
+
+And the x-only ElligatorSwift encoding algorithm is still:
+
+**Define** *ElligatorSwift(x)* as:
+* Loop:
+  * Pick a uniformly random field element $u.$
+  * Pick a uniformly random integer $c$ in $[0,8).$
+  * Let $t = G_{c,u}(x).$
+  * If $t \neq \bot$, return $(u, t)$; restart loop otherwise.
+
+Note that this logic does not take the remapped $u=0$, $t=0$, and $g(u) = -t^2$ cases into account; it just avoids them.
+While it is not impossible to make the encoder target them, this would increase the maximum number of $t$ values for a given $(u, x)$
+combination beyond 8, and thereby slow down the ElligatorSwift loop proportionally, for a negligible gain in uniformity.
+
+## 4. Encoding and decoding full *(x, y)* coordinates
+
+So far we have only addressed encoding and decoding x-coordinates, but in some cases an encoding
+for full points with $(x, y)$ coordinates is desirable. It is possible to encode this information
+in $t$ as well.
+
+Note that for any $(X, Y) \in S_u$, $(\pm X, \pm Y)$ are all on $S_u.$ Moreover, all of these are
+mapped to the same x-coordinate. Negating $X$ or negating $Y$ just results in $x_1$ and $x_2$
+being swapped, and does not affect $x_3.$ This will not change the outcome x-coordinate as the order
+of $x_1$ and $x_2$ only matters if both were to be valid, and in that case $x_3$ would be used instead.
+
+Still, these four $(X, Y)$ combinations all correspond to distinct $t$ values, so we can encode
+the sign of the y-coordinate in the sign of $X$ or the sign of $Y.$ They correspond to the
+four distinct $P_u^{'-1}$ calls in the definition of $G_{u,c}.$
+
+**Note**: In the paper, the sign of the y coordinate is encoded in a separately-coded bit.
+
+To encode the sign of $y$ in the sign of $Y:$
+
+**Define** *Decode(u, t)* for full $(x, y)$ as:
+* Let $(X, Y) = P_u(t).$
+* Let $x$ be the first value in $(u + 4Y^2, \frac{-X}{2Y} - \frac{u}{2}, \frac{X}{2Y} - \frac{u}{2})$ for which $g(x)$ is square.
+* Let $y = \sqrt{g(x)}.$
+* If $sign(y) = sign(Y)$, return $(x, y)$; otherwise return $(x, -y).$
+
+And encoding would be done using a $G_{c,u}(x, y)$ function defined as:
+
+**Define** $G_{c,u}(x, y)$ as:
+* If $c \in \\{0, 1\\}:$
+  * If $g(u) = 0$ or $g(x) = 0$, return $\bot$ (even curves only).
+  * If $g(-u-x)$ is square, return $\bot.$
+  * Let $s = -g(u)/(u^2 + ux + x^2 + a)$ (cannot cause division by zero).
+  * Let $v = x.$
+* Otherwise, when $c \in \\{2, 3\\}:$
+  * Let $s = x-u.$
+  * Let $r = \sqrt{-s(4g(u) + sh(u))}$; return $\bot$ if not square.
+  * If $c = 3$ and $r = 0$, return $\bot.$
+  * Let $v = (r/s - u)/2.$
+* Let $w = \sqrt{s}$; return $\bot$ if not square.
+* Let $w' = w$ if $sign(w/2) = sign(y)$; $-w$ otherwise.
+* Depending on $c:$
+  * If $c \in \\{0, 2\\}:$ return $P_u^{'-1}(v, w').$
+  * If $c \in \\{1, 3\\}:$ return $P_u^{'-1}(-u-v, w').$
+
+Note that $c$ now only ranges $[0,4)$, as the sign of $w'$ is decided based on that of $y$, rather than on $c.$
+This change makes some valid encodings unreachable: when $y = 0$ and $sign(Y) \neq sign(0)$.
+
+In the above logic, $sign$ can be implemented in several ways, such as parity of the integer representation
+of the input field element (for prime-sized fields) or the quadratic residuosity (for fields where
+$-1$ is not square). The choice does not matter, as long as it only takes on two possible values, and for $x \neq 0$ it holds that $sign(x) \neq sign(-x)$.
+
+### 4.1 Full *(x, y)* coordinates for `secp256k1`
+
+For $a=0$ curves, there is another option. Note that for those,
+the $P_u(t)$ function translates negations of $t$ to negations of (both) $X$ and $Y.$ Thus, we can use $sign(t)$ to
+encode the y-coordinate directly. Combined with the earlier remapping to guarantee all inputs land on the curve, we get
+as decoder:
+
+**Define** *Decode(u, t)* as:
+* Let $u'=u$ if $u \neq 0$; $1$ otherwise.
+* Let $t'=t$ if $t \neq 0$; $1$ otherwise.
+* Let $t''=t'$ if $u'^3 + b + t'^2 \neq 0$; $2t'$ otherwise.
+* Let $X = \dfrac{u'^3 + b - t''^2}{2t''}.$
+* Let $Y = \dfrac{X + t''}{u'\sqrt{-3}}.$
+* Let $x$ be the first element of $(u' + 4Y^2, \frac{-X}{2Y} - \frac{u'}{2}, \frac{X}{2Y} - \frac{u'}{2})$ for which $g(x)$ is square.
+* Let $y = \sqrt{g(x)}.$
+* Return $(x, y)$ if $sign(y) = sign(t)$; $(x, -y)$ otherwise.
+
+This is implemented in `secp256k1_ellswift_swiftec_var`. The used $sign(x)$ function is the parity of $x$ when represented as in integer in $[0,q).$
+
+The corresponding encoder would invoke the x-only one, but negating the output $t$ if $sign(t) \neq sign(y).$
+
+This is implemented in `secp256k1_ellswift_elligatorswift_var`.
+
+Note that this is only intended for encoding points where both the x-coordinate and y-coordinate are unpredictable. When encoding x-only points
+where the y-coordinate is implicitly even (or implicitly square, or implicitly in $[0,q/2]$), the encoder in
+[Section 3.5](#35-encoding-for-secp256k1) must be used, or a bias is reintroduced that undoes all the benefit of using ElligatorSwift
+in the first place.

external/secp256k1/doc/musig.md

@@ -0,0 +1,54 @@
+Notes on the musig module API
+===========================
+
+The following sections contain additional notes on the API of the musig module (`include/secp256k1_musig.h`).
+A usage example can be found in `examples/musig.c`.
+
+## API misuse
+
+The musig API is designed with a focus on misuse resistance.
+However, due to the interactive nature of the MuSig protocol, there are additional failure modes that are not present in regular (single-party) Schnorr signature creation.
+While the results can be catastrophic (e.g. leaking of the secret key), it is unfortunately not possible for the musig implementation to prevent all such failure modes.
+
+Therefore, users of the musig module must take great care to make sure of the following:
+
+1. A unique nonce per signing session is generated in `secp256k1_musig_nonce_gen`.
+   See the corresponding comment in `include/secp256k1_musig.h` for how to ensure that.
+2. The `secp256k1_musig_secnonce` structure is never copied or serialized.
+   See also the comment on `secp256k1_musig_secnonce` in `include/secp256k1_musig.h`.
+3. Opaque data structures are never written to or read from directly.
+   Instead, only the provided accessor functions are used.
+
+## Key Aggregation and (Taproot) Tweaking
+
+Given a set of public keys, the aggregate public key is computed with `secp256k1_musig_pubkey_agg`.
+A plain tweak can be added to the resulting public key with `secp256k1_ec_pubkey_tweak_add` by setting the `tweak32` argument to the hash defined in BIP 32. Similarly, a Taproot tweak can be added with `secp256k1_xonly_pubkey_tweak_add` by setting the `tweak32` argument to the TapTweak hash defined in BIP 341.
+Both types of tweaking can be combined and invoked multiple times if the specific application requires it.
+
+## Signing
+
+This is covered by `examples/musig.c`.
+Essentially, the protocol proceeds in the following steps:
+
+1. Generate a keypair with `secp256k1_keypair_create` and obtain the public key with `secp256k1_keypair_pub`.
+2. Call `secp256k1_musig_pubkey_agg` with the pubkeys of all participants.
+3. Optionally add a (Taproot) tweak with `secp256k1_musig_pubkey_xonly_tweak_add` and a plain tweak with `secp256k1_musig_pubkey_ec_tweak_add`.
+4. Generate a pair of secret and public nonce with `secp256k1_musig_nonce_gen` and send the public nonce to the other signers.
+5. Someone (not necessarily the signer) aggregates the public nonces with `secp256k1_musig_nonce_agg` and sends it to the signers.
+6. Process the aggregate nonce with `secp256k1_musig_nonce_process`.
+7. Create a partial signature with `secp256k1_musig_partial_sign`.
+8. Verify the partial signatures (optional in some scenarios) with `secp256k1_musig_partial_sig_verify`.
+9. Someone (not necessarily the signer) obtains all partial signatures and aggregates them into the final Schnorr signature using `secp256k1_musig_partial_sig_agg`.
+
+The aggregate signature can be verified with `secp256k1_schnorrsig_verify`.
+
+Steps 1 through 5 above can occur before or after the signers are aware of the message to be signed.
+Whenever possible, it is recommended to generate the nonces only after the message is known.
+This provides enhanced defense-in-depth measures, protecting against potential API misuse in certain scenarios.
+However, it does require two rounds of communication during the signing process.
+The alternative, generating the nonces in a pre-processing step before the message is known, eliminates these additional protective measures but allows for non-interactive signing.
+Similarly, the API supports an alternative protocol flow where generating the aggregate key (steps 1 to 3) is allowed to happen after exchanging nonces (steps 4 to 5).
+
+## Verification
+
+A participant who wants to verify the partial signatures, but does not sign itself may do so using the above instructions except that the verifier skips steps 1, 4 and 7.

external/secp256k1/doc/release-process.md

@@ -1,4 +1,4 @@
-# Release Process
+# Release process
 
 This document outlines the process for releasing versions of the form `$MAJOR.$MINOR.$PATCH`.
 
@@ -12,50 +12,83 @@ It is best if the maintainers are present during the release, so they can help e
 
 This process also assumes that there will be no minor releases for old major releases.
 
+We aim to cut a regular release every 3-4 months, approximately twice as frequent as major Bitcoin Core releases. Every second release should be published one month before the feature freeze of the next major Bitcoin Core release, allowing sufficient time to update the library in Core.
+
+## Sanity checks
+Perform these checks when reviewing the release PR (see below):
+
+1. Ensure `make distcheck` doesn't fail.
+   ```shell
+   ./autogen.sh && ./configure --enable-dev-mode && make distcheck
+   ```
+2. Check installation with autotools:
+   ```shell
+   dir=$(mktemp -d)
+   ./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir
+   gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa
+   ```
+3. Check installation with CMake:
+   ```shell
+   dir=$(mktemp -d)
+   build=$(mktemp -d)
+   cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build && cmake --install $build && ls -RlAh $dir
+   gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa
+   ```
+4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to verify that there are no unexpected ABI incompatibilities and that the version number and the release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required.
+   ```shell
+   tools/check-abi.sh
+   ```
+
 ## Regular release
 
 1. Open a PR to the master branch with a commit (using message `"release: prepare for $MAJOR.$MINOR.$PATCH"`, for example) that
-   * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) (make sure to include an entry for `### ABI Compatibility`),
-   * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and
-   * if this is not a patch release
-       * updates `_PKG_VERSION_*` and `_LIB_VERSION_*`  in `configure.ac` and
+   * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) by
+       * adding a section for the release (make sure that the version number is a link to a diff between the previous and new version),
+       * removing the `[Unreleased]` section header,
+       * ensuring that the release notes are not missing entries (check the `needs-changelog` label on github), and
+       * including an entry for `### ABI Compatibility` if it doesn't exist,
+   * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and,
+   * if this is not a patch release,
+       * updates `_PKG_VERSION_*` and `_LIB_VERSION_*`  in `configure.ac`, and
        * updates `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_*` in `CMakeLists.txt`.
-2. After the PR is merged, tag the commit and push it:
+2. Perform the [sanity checks](#sanity-checks) on the PR branch.
+3. After the PR is merged, tag the commit, and push the tag:
    ```
    RELEASE_COMMIT=<merge commit of step 1>
    git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH" $RELEASE_COMMIT
    git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH
    ```
-3. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that
-   * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`, and
-   * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`.
+4. Open a PR to the master branch with a commit (using message `"release cleanup: bump version after $MAJOR.$MINOR.$PATCH"`, for example) that
+   * sets `_PKG_VERSION_IS_RELEASE` to `false` and increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`,
+   * increments the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`, and
+   * adds an `[Unreleased]` section header to the [CHANGELOG.md](../CHANGELOG.md).
 
    If other maintainers are not present to approve the PR, it can be merged without ACKs.
-4. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md).
+5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md).
+6. Send an announcement email to the bitcoin-dev mailing list.
 
 ## Maintenance release
 
-Note that bugfixes only need to be backported to releases for which no compatible release without the bug exists.
+Note that bug fixes need to be backported only to releases for which no compatible release without the bug exists.
 
-1. If `$PATCH = 1`, create maintenance branch `$MAJOR.$MINOR`:
+1. If there's no maintenance branch `$MAJOR.$MINOR`, create one:
    ```
-   git checkout -b $MAJOR.$MINOR v$MAJOR.$MINOR.0
+   git checkout -b $MAJOR.$MINOR v$MAJOR.$MINOR.$((PATCH - 1))
    git push git@github.com:bitcoin-core/secp256k1.git $MAJOR.$MINOR
    ```
 2. Open a pull request to the `$MAJOR.$MINOR` branch that
-   * includes the bugfixes,
-   * finalizes the release notes,
+   * includes the bug fixes,
+   * finalizes the release notes similar to a regular release,
    * increments `_PKG_VERSION_PATCH` and `_LIB_VERSION_REVISION` in `configure.ac`
      and the `$PATCH` component of `project(libsecp256k1 VERSION ...)` and `${PROJECT_NAME}_LIB_VERSION_REVISION` in `CMakeLists.txt`
      (with commit message `"release: bump versions for $MAJOR.$MINOR.$PATCH"`, for example).
-3. After the PRs are merged, update the release branch and tag the commit:
+3. Perform the [sanity checks](#sanity-checks) on the PR branch.
+4. After the PRs are merged, update the release branch, tag the commit, and push the tag:
    ```
    git checkout $MAJOR.$MINOR && git pull
    git tag -s v$MAJOR.$MINOR.$PATCH -m "libsecp256k1 $MAJOR.$MINOR.$PATCH"
-   ```
-4. Push tag:
-   ```
    git push git@github.com:bitcoin-core/secp256k1.git v$MAJOR.$MINOR.$PATCH
    ```
-5. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md).
-6. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md).
+6. Create a new GitHub release with a link to the corresponding entry in [CHANGELOG.md](../CHANGELOG.md).
+7. Send an announcement email to the bitcoin-dev mailing list.
+8. Open PR to the master branch that includes a commit (with commit message `"release notes: add $MAJOR.$MINOR.$PATCH"`, for example) that adds release notes to [CHANGELOG.md](../CHANGELOG.md).

external/secp256k1/examples/CMakeLists.txt

@@ -1,27 +1,31 @@
-add_library(example INTERFACE)
-target_include_directories(example INTERFACE
-  ${PROJECT_SOURCE_DIR}/include
-)
-target_link_libraries(example INTERFACE
-  secp256k1
-  $<$<PLATFORM_ID:Windows>:bcrypt>
-)
-if(NOT BUILD_SHARED_LIBS AND MSVC)
-  target_link_options(example INTERFACE /IGNORE:4217)
-endif()
+function(add_example name)
+  set(target_name ${name}_example)
+  add_executable(${target_name} ${name}.c)
+  target_include_directories(${target_name} PRIVATE
+    ${PROJECT_SOURCE_DIR}/include
+  )
+  target_link_libraries(${target_name}
+    secp256k1
+    $<$<PLATFORM_ID:Windows>:bcrypt>
+  )
+  set(test_name ${name}_example)
+  add_test(NAME secp256k1_${test_name} COMMAND ${target_name})
+endfunction()
 
-add_executable(ecdsa_example ecdsa.c)
-target_link_libraries(ecdsa_example example)
-add_test(NAME ecdsa_example COMMAND ecdsa_example)
+add_example(ecdsa)
 
 if(SECP256K1_ENABLE_MODULE_ECDH)
-  add_executable(ecdh_example ecdh.c)
-  target_link_libraries(ecdh_example example)
-  add_test(NAME ecdh_example COMMAND ecdh_example)
+  add_example(ecdh)
 endif()
 
 if(SECP256K1_ENABLE_MODULE_SCHNORRSIG)
-  add_executable(schnorr_example schnorr.c)
-  target_link_libraries(schnorr_example example)
-  add_test(NAME schnorr_example COMMAND schnorr_example)
+  add_example(schnorr)
+endif()
+
+if(SECP256K1_ENABLE_MODULE_ELLSWIFT)
+  add_example(ellswift)
+endif()
+
+if(SECP256K1_ENABLE_MODULE_MUSIG)
+  add_example(musig)
 endif()

external/secp256k1/examples/ecdh.c

@@ -42,18 +42,16 @@ int main(void) {
     assert(return_val);
 
     /*** Key Generation ***/
-
-    /* If the secret key is zero or out of range (bigger than secp256k1's
-     * order), we try to sample a new key. Note that the probability of this
-     * happening is negligible. */
-    while (1) {
-        if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) {
-            printf("Failed to generate randomness\n");
-            return 1;
-        }
-        if (secp256k1_ec_seckey_verify(ctx, seckey1) && secp256k1_ec_seckey_verify(ctx, seckey2)) {
-            break;
-        }
+    if (!fill_random(seckey1, sizeof(seckey1)) || !fill_random(seckey2, sizeof(seckey2))) {
+        printf("Failed to generate randomness\n");
+        return 1;
+    }
+    /* If the secret key is zero or out of range (greater than secp256k1's
+    * order), we fail. Note that the probability of this occurring is negligible
+    * with a properly functioning random number generator. */
+    if (!secp256k1_ec_seckey_verify(ctx, seckey1) || !secp256k1_ec_seckey_verify(ctx, seckey2)) {
+        printf("Generated secret key is invalid. This indicates an issue with the random number generator.\n");
+        return 1;
     }
 
     /* Public key creation using a valid context with a verified secret key should never fail */
@@ -108,7 +106,7 @@ int main(void) {
 
     /* It's best practice to try to clear secrets from memory after using them.
      * This is done because some bugs can allow an attacker to leak memory, for
-     * example through "out of bounds" array access (see Heartbleed), Or the OS
+     * example through "out of bounds" array access (see Heartbleed), or the OS
      * swapping them to disk. Hence, we overwrite the secret key buffer with zeros.
      *
      * Here we are preventing these writes from being optimized out, as any good compiler