fix: take stoppingMutex_ in ApplicationImp::signalStop so the stop notify cannot be lost

.github/workflows/xahau-ga-nix.yml

@@ -97,6 +97,8 @@ jobs:
       contents: read
     container:
       image: ubuntu:24.04
+      # [DO NOT MERGE] diagnostic: allow gdb/eu-stack to attach to hung test children
+      options: --cap-add=SYS_PTRACE
       volumes:
         - /home/runner/.conan-cache:/.conan-cache
         - /home/runner/.ccache-cache:/github/home/.ccache-cache
@@ -303,9 +305,98 @@ jobs:
           # Coverage builds are slower due to instrumentation; use fewer parallel jobs to avoid flakiness.
           # Use *_FLAGS_DEBUG so the build action's stdlib flag (e.g. -stdlib=libc++) in CMAKE_CXX_FLAGS isn't clobbered.
           cmake-args: '-Dcoverage=ON -Dcoverage_tool=${{ matrix.coverage_tool }} -Dcoverage_format=${{ matrix.coverage_format }} -Dcoverage_test_parallelism=$(($(nproc)/2)) -DCODE_COVERAGE_VERBOSE=ON -DCMAKE_CXX_FLAGS_DEBUG="-g -O0" -DCMAKE_C_FLAGS_DEBUG="-g -O0"'
-          cmake-target: 'coverage'
+          cmake-target: 'rippled'
           ccache_max_size: '100G'
 
+      # [DO NOT MERGE] Diagnostic: run the coverage test pass directly (not via
+      # the ninja `coverage` target, which buffers all output until exit) so
+      # every line gets a live timestamp, sample host/process state every 60s,
+      # and dump thread stacks of every rippled process at 12/25 min and on a
+      # 90 min deadline instead of sitting for the 6h job timeout.
+      - name: Run tests (coverage, diagnostic)
+        if: matrix.job_type == 'coverage'
+        shell: bash
+        run: |
+          apt-get install -y procps elfutils gdb > /dev/null
+          cd ${{ env.build_dir }}
+          rm -rf coverage-profraw && mkdir coverage-profraw
+          JOBS=$(($(nproc)/2))
+          echo "nproc=$(nproc) jobs=$JOBS loadavg=$(cat /proc/loadavg)"
+          free -m; df -h . /dev/shm /tmp; ulimit -a; cat /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io 2>/dev/null || true
+
+          cat > /tmp/flag.py <<'EOF'
+          import gdb
+          for t in gdb.selected_inferior().threads():
+              t.switch()
+              f = gdb.newest_frame()
+              while f:
+                  if f.name() and 'ApplicationImp::run' in f.name():
+                      f.select()
+                      print("thread", t.num, "in ApplicationImp::run, isTimeToStop =", gdb.parse_and_eval("this->isTimeToStop"))
+                      break
+                  f = f.older()
+          EOF
+          dump_stacks() {
+            echo "===== STACK DUMP ($1) $(date -u +%T)"
+            for p in $(pgrep -x rippled); do
+              echo "--- pid $p state=$(awk '/^State/{print $2}' /proc/$p/status) wchan=$(cat /proc/$p/wchan 2>/dev/null) cmd=$(tr '\0' ' ' < /proc/$p/cmdline | cut -c1-100)"
+              timeout 120 eu-stack -p $p 2>&1 | head -300 || true
+              st=$(awk '/^State/{print $2}' /proc/$p/status 2>/dev/null)
+              if [ "$st" = "S" ] && [ "$p" != "$RP" ]; then
+                timeout 900 gdb -p $p -batch -ex 'set pagination off' -ex 'source /tmp/flag.py' -ex 'thread apply all bt 30' 2>&1 \
+                  | grep -vE '^\[New LWP|^\[Thread debugging|^Using host|^warning:|^Reading symbols|^Download' | head -800 || true
+              fi
+            done
+          }
+
+          # Stress the suite whose Env lifetimes are shortest: the hang sits in
+          # Env::~AppBundle (thread.join on ApplicationImp::run) and shows up
+          # within a few iterations when it is present.
+          for i in 1 2 3 4 5 6; do
+            st=$SECONDS
+            if LLVM_PROFILE_FILE=/tmp/stress-%p.profraw timeout -s KILL 240 ./rippled --unittest=ripple.app.HookAPI --unittest-jobs 1 > /tmp/stress_$i.log 2>&1; then
+              echo "stress iter $i ok $((SECONDS-st))s: $(grep -oE '[0-9]+ cases, [0-9]+ tests total, [0-9]+ failures' /tmp/stress_$i.log)"
+            else
+              echo "stress iter $i FAILED/HUNG rc=$? after $((SECONDS-st))s; last lines:"; tail -3 /tmp/stress_$i.log
+            fi
+          done
+          rm -f /tmp/stress-*.profraw
+
+          ( while true; do
+              echo "===== sample $(date -u +%T) loadavg=$(cut -d' ' -f1-3 /proc/loadavg) psi_cpu=$(grep -o 'avg60=[0-9.]*' /proc/pressure/cpu 2>/dev/null | head -1) psi_mem=$(grep -o 'avg60=[0-9.]*' /proc/pressure/memory 2>/dev/null | head -1) psi_io=$(grep -o 'avg60=[0-9.]*' /proc/pressure/io 2>/dev/null | head -1)"
+              free -m | sed -n 2,3p
+              ps -eo pid,ppid,stat,etimes,pcpu,rss,wchan:28,args --sort=-pcpu | grep -E 'PID|rippled' | grep -v grep | head -14 | cut -c1-150
+              sleep 60
+            done ) &
+          MON=$!
+
+          LLVM_PROFILE_FILE="$PWD/coverage-profraw/rippled-%m-%p.profraw" \
+            stdbuf -oL -eL ./rippled --unittest --unittest-jobs $JOBS --unittest-log &
+          RP=$!
+          START=$SECONDS; D20=0; D40=0; rc=0
+          while kill -0 $RP 2>/dev/null; do
+            el=$((SECONDS-START))
+            if [ $D20 = 0 ] && [ $el -ge 720 ]; then D20=1; dump_stacks "12min"; fi
+            if [ $D40 = 0 ] && [ $el -ge 1500 ]; then D40=1; dump_stacks "25min"; fi
+            if [ $el -ge 3600 ]; then
+              dump_stacks "60min deadline"
+              echo "===== DEADLINE: killing rippled processes"
+              pkill -KILL -x rippled || true
+              break
+            fi
+            sleep 10
+          done
+          wait $RP || rc=$?
+          kill $MON 2>/dev/null || true
+          echo "rippled unittest exit code: $rc after $((SECONDS-START))s"
+          ls -la coverage-profraw | head -20
+
+          llvm-profdata-${{ matrix.compiler_version }} merge -sparse -o coverage.profdata coverage-profraw/*.profraw
+          llvm-cov-${{ matrix.compiler_version }} export -instr-profile=coverage.profdata -format=lcov \
+            -ignore-filename-regex="src/test|include/xrpl/beast/test|include/xrpl/beast/unit_test|$PWD/pb-xrpl.libpb" \
+            ./rippled > coverage.lcov
+          exit $rc
+
       - name: Set artifact name
         if: matrix.job_type == 'build'
         id: set-artifact-name

src/xrpld/app/main/Application.cpp

@@ -1682,6 +1682,11 @@ ApplicationImp::signalStop(std::string msg)
         else
             JLOG(m_journal.warn()) << "Server stopping: " << msg;
 
+        // run() checks isTimeToStop and blocks on stoppingCondition_ while
+        // holding stoppingMutex_. Notifying without the lock can land in the
+        // gap between that check and the wait and be lost, leaving run()
+        // blocked forever (seen as a hung Env::~AppBundle thread.join()).
+        std::lock_guard lk{stoppingMutex_};
         stoppingCondition_.notify_all();
     }
 }