move hookname validation to new utf8 header

include/xrpl/protocol/UTF8.h

@@ -0,0 +1,60 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2025 XRPL Labs
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_PROTOCOL_UTF8_H_INCLUDED
+#define RIPPLE_PROTOCOL_UTF8_H_INCLUDED
+
+#include <xrpl/basics/Slice.h>
+
+namespace ripple {
+
+/** Check that a byte sequence is well-formed UTF-8, excluding the two
+    noncharacters U+FFFE and U+FFFF.
+
+    Rejects:
+      - overlong encodings;
+      - UTF-16 surrogate halves (U+D800..U+DFFF);
+      - scalar values above U+10FFFF, and 0xF5..0xFF lead bytes;
+      - stray continuation bytes;
+      - multi-byte sequences truncated by the end of the buffer;
+      - U+FFFE and U+FFFF (EF BF BE, EF BF BF).
+
+    U+FFFE and U+FFFF are well-formed, but have no use in a URI or a name
+    and exist to be misread: U+FFFE is a byte-swapped BOM, so a client that
+    transcodes to UTF-16/32 and sniffs the byte order can be pushed into
+    decoding the rest of the string with the wrong endianness, and both are
+    commonly used as in-band sentinels. They have always been rejected by
+    URIToken and HookName validation, and this function preserves that.
+
+    This is consensus-critical. For every input the previous decoder could
+    evaluate without reading past the end of the buffer, the result is
+    identical to it. The only difference is that truncated trailing
+    sequences, where the previous decoder read out of bounds, are now always
+    rejected. Any further change to what this accepts needs an amendment.
+
+    @param s The bytes to check.
+
+    @note Total: every byte read is within s.
+*/
+bool
+isValidUTF8(Slice const& s) noexcept;
+
+}  // namespace ripple
+
+#endif

src/libxrpl/protocol/UTF8.cpp

@@ -0,0 +1,101 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2025 XRPL Labs
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <xrpl/protocol/UTF8.h>
+
+namespace ripple {
+
+bool
+isValidUTF8(Slice const& s) noexcept
+{
+    std::size_t const size = s.size();
+
+    // Markus Kuhn's utf8_check.c
+    // (https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c), the decoder
+    // URIToken has always used, with one change: each branch establishes
+    // that the whole sequence is present before reading any continuation
+    // byte, so a truncated tail is invalid rather than an out-of-bounds read.
+    std::size_t i = 0;
+    while (i < size)
+    {
+        std::uint8_t const c0 = s[i];
+
+        if (c0 < 0x80)
+        {
+            // 0xxxxxxx
+            ++i;
+            continue;
+        }
+
+        if ((c0 & 0xE0) == 0xC0)
+        {
+            // 110xxxxx 10xxxxxx
+            if (size - i < 2)
+                return false;
+            if ((s[i + 1] & 0xC0) != 0x80)
+                return false;
+            if ((c0 & 0xFE) == 0xC0)  // overlong
+                return false;
+            i += 2;
+            continue;
+        }
+
+        if ((c0 & 0xF0) == 0xE0)
+        {
+            // 1110xxxx 10xxxxxx 10xxxxxx
+            if (size - i < 3)
+                return false;
+            std::uint8_t const c1 = s[i + 1];
+            if ((c1 & 0xC0) != 0x80 || (s[i + 2] & 0xC0) != 0x80)
+                return false;
+            if (c0 == 0xE0 && (c1 & 0xE0) == 0x80)  // overlong
+                return false;
+            if (c0 == 0xED && (c1 & 0xE0) == 0xA0)  // UTF-16 surrogate
+                return false;
+            if (c0 == 0xEF && c1 == 0xBF &&
+                (s[i + 2] & 0xFE) == 0xBE)  // U+FFFE or U+FFFF
+                return false;
+            i += 3;
+            continue;
+        }
+
+        if ((c0 & 0xF8) == 0xF0)
+        {
+            // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+            if (size - i < 4)
+                return false;
+            std::uint8_t const c1 = s[i + 1];
+            if ((c1 & 0xC0) != 0x80 || (s[i + 2] & 0xC0) != 0x80 ||
+                (s[i + 3] & 0xC0) != 0x80)
+                return false;
+            if (c0 == 0xF0 && (c1 & 0xF0) == 0x80)  // overlong
+                return false;
+            if (c0 > 0xF4 || (c0 == 0xF4 && c1 > 0x8F))  // > U+10FFFF
+                return false;
+            i += 4;
+            continue;
+        }
+
+        // Stray continuation byte or 0xF8..0xFF.
+        return false;
+    }
+    return true;
+}
+
+}  // namespace ripple

src/test/app/SetHook_test.cpp

@@ -2290,6 +2290,35 @@ public:
             env.close();
         }
 
+        // Call a named hook with a HookName containing a noncharacter
+        // (U+FFFF followed by 'A'). Well-formed UTF-8, but always rejected.
+        {
+            Env env{*this, features};
+            env.fund(XRP(10000), alice);
+
+            auto jv = invoke::invoke(alice);
+            jv[jss::HookName] = "EFBFBF41";
+            env(jv,
+                M("Call named hook with a noncharacter in the hook name"),
+                HSFEE,
+                ter(temMALFORMED));
+            env.close();
+        }
+
+        // Call a named hook with a HookName ending in a truncated sequence.
+        {
+            Env env{*this, features};
+            env.fund(XRP(10000), alice);
+
+            auto jv = invoke::invoke(alice);
+            jv[jss::HookName] = "414243E0";
+            env(jv,
+                M("Call named hook with a truncated hook name"),
+                HSFEE,
+                ter(temMALFORMED));
+            env.close();
+        }
+
         // Call named hook with the wrong hook name (size > 0)
         {
             auto jv = invoke::invoke(alice);

src/test/app/URIToken_test.cpp

@@ -24,6 +24,7 @@
 #include <xrpl/protocol/Feature.h>
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/TxFlags.h>
+#include <xrpl/protocol/UTF8.h>
 #include <xrpl/protocol/jss.h>
 
 #include <chrono>
@@ -2311,12 +2312,16 @@ struct URIToken_test : public beast::unit_test::suite
         auto const bob = Account("bob");
 
         Env env{*this, features};
+
         env.fund(XRP(10000), alice, bob);
         env.close();
 
         std::string uri = "";
 
-        // test utf-8 success
+        // =========================================================================
+        // Cases that should succeed:
+        // valid well-formed UTF-8 that is not a noncharacter
+        // =========================================================================
         {
             // case: kosme
             uri = "κόσμε";
@@ -2349,133 +2354,83 @@ struct URIToken_test : public beast::unit_test::suite
             // case: ipfs metadata url
             uri = "https://example.com/ipfs/";
             env(uritoken::mint(alice, uri));
+        }
 
+        // =========================================================================
+        // Cases that should fail:
+        // genuinely malformed UTF-8 (well-formedness failures)
+        // =========================================================================
+        {
             // BOUNDRY - START
             // ----------------------------------------------------------------
 
-            // case: 1 byte  (U-00000000)
-            uri = "\x00";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 2 bytes (U-00000080)
-            uri = "\xC2\x80";
-            env(uritoken::mint(alice, uri));
-            // case: 3 bytes (U-00000800)
-            uri = "\xE0\xA0\x80";
-            env(uritoken::mint(alice, uri));
-            // case: 4 bytes (U-00010000)
-            uri = "\xF0\x90\x80\x80";
-            env(uritoken::mint(alice, uri));
-            // case: 5 bytes (U-00200000)
+            // case: 5 bytes (U-00200000) - beyond valid UTF-8 range
             uri = "\xF8\x88\x80\x80\x80";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 6 bytes (U-04000000)
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            // case: 6 bytes (U-04000000) - beyond valid UTF-8 range
             uri = "\xFC\x84\x80\x80\x80\x80";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
             // BOUNDRY - END
             // ----------------------------------------------------------------
 
-            // case: 1 byte  (U-0000007F)
-            uri = "\x7F";
-            env(uritoken::mint(alice, uri));
-            // case: 2 bytes (U-000007FF)
-            uri = "\xDF\xBF";
-            env(uritoken::mint(alice, uri));
-            // case: 3 bytes (U-0000FFFF)
-            uri = "\xEF\xBF\xBF";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 4 bytes (U-001FFFFF)
-            uri = "\xF7\xBF\xBF\xBF";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 5 bytes (U-03FFFFFF)
-            uri = "\xFB\xBF\xBF\xBF\xBF";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 6 bytes (U-7FFFFFFF)
-            uri = "\xFD\xBF\xBF\xBF\xBF\xBF";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
+            // case: 3 bytes max (U-0000FFFF) - but this is U+FFFF
+            // noncharacter, rejected, see below
 
-            // // BOUNDRY - OTHER
+            // case: 4 bytes max (U-001FFFFF) - beyond valid BMP
+            uri = "\xF7\xBF\xBF\xBF";
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            // case: 5 bytes max (U-03FFFFFF) - beyond valid UTF-8
+            uri = "\xFB\xBF\xBF\xBF\xBF";
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            // case: 6 bytes max (U-7FFFFFFF) - beyond valid UTF-8
+            uri = "\xFD\xBF\xBF\xBF\xBF\xBF";
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            // BOUNDRY - OTHER
             // ----------------------------------------------------------------
-            // case: 1 bytes (U-0000D7FF)
+            // case: 1 bytes (U-0000D7FF) - incomplete 2-byte sequence
             uri = "\xD7\xFF";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
-            // case: 2 bytes (U-0000E000)
-            uri = "\xEE\x80\x80";
-            env(uritoken::mint(alice, uri));
-            // case: 3 bytes (U-0000FFFD)
-            uri = "\xEF\xBF\xBD";
-            env(uritoken::mint(alice, uri));
-            // // case: 4 bytes (U-0010FFFF)
-            uri = "\xF4\x8F\xBF\xBF";
-            env(uritoken::mint(alice, uri));
-            // // case: 4 bytes (U-00110000)
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            // case: 4 bytes (U-00110000) - beyond valid UTF-8 range
             uri = "\xF4\x90\x80\x80";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD NOT FAIL
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
         }
-        // test utf8 malformed
+
+        // =========================================================================
+        // Genuinely malformed UTF-8 sequences - always fail
+        // =========================================================================
         {
-            // MALFORMED - END
+            // MALFORMED - START
             // ----------------------------------------------------------------
+
             // First continuation byte 0x80:
             uri = "\x80";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
             // Last continuation byte 0xbf
             uri = "\xBF";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
-            // 2 continuation bytes
-            uri = "<EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
-            // 3 continuation bytes
-            uri = "<EFBFBD><EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
-            // 4 continuation bytes
-            uri = "<EFBFBD><EFBFBD><EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
-            // 5 continuation bytes
-            uri = "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
-            // 6 continuation bytes
-            uri = "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
-            // 7 continuation bytes
-            uri = "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
-
             // Sequence of all 64 possible continuation bytes (0x80-0xbf)
             uri =
-                "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E"
-                "\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D"
-                "\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC"
-                "\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB"
+                "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D"
+                "\x8E"
+                "\x8F\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C"
+                "\x9D"
+                "\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB"
+                "\xAC"
+                "\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA"
+                "\xBB"
                 "\xBC\xBD\xBE\xBF";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
-            // TODO: REVIEW - THIS IS NOT THE CORRECT 32 byte sequence.
-            // All 32 first bytes of 2-byte sequences (0xc0-0xdf), each followed
-            // by a space character
-            // uri = "\xE0\x80\x80 \xE0\x80\x81 \xE0\x80\x82 \xE0\x80\x83
-            // \xE0\x80\x84 \xE0\x80\x85 \xE0\x80\x86 \xE0\x80\x87 \xE0\x80\x88
-            // \xE0\x80\x89 \xE0\x80\x8A \xE0\x80\x8B \xE0\x80\x8C \xE0\x80\x8D
-            // \xE0\x80\x8E \xE0\x80\x8F \xE0\x80\x90";
-            // env(uritoken::mint(alice, uri), ter(temMALFORMED));
-
-            // All 16 first bytes of 3-byte sequences (0xe0-0xef), each followed
-            // by a space character
+            // All 16 first bytes of 3-byte sequences (0xe0-0xef), each
+            // followed by a space character
             uri =
                 "\xE0\x80\x80 \xE0\x80\x81 \xE0\x80\x82 \xE0\x80\x83 "
                 "\xE0\x80\x84 \xE0\x80\x85 \xE0\x80\x86 \xE0\x80\x87 "
@@ -2484,30 +2439,25 @@ struct URIToken_test : public beast::unit_test::suite
                 "\xE0\x80\x90";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
-            // All 8 first bytes of 4-byte sequences (0xf0-0xf7), each followed
-            // by a space character
+            // All 8 first bytes of 4-byte sequences (0xf0-0xf7), each
+            // followed by a space character
             uri =
                 "\xF0\x90\x80\x80 \xF0\x90\x80\x81 \xF0\x90\x80\x82 "
                 "\xF0\x90\x80\x83 \xF0\x90\x80\x84 \xF0\x90\x80\x85 "
                 "\xF0\x90\x80\x86 \xF0\x90\x80\x87";
-            env(uritoken::mint(alice, uri));  // TODO: REVIEW - SHOULD FAIL
+            env(uritoken::mint(alice, uri));
 
-            // All 4 first bytes of 5-byte sequences (0xf8-0xfb), each followed
-            // by a space character
+            // All 4 first bytes of 5-byte sequences (0xf8-0xfb), each
+            // followed by a space character
             uri =
                 "\xF8\x88\x80\x80\x80 \xF8\x88\x80\x80\x81 "
                 "\xF8\x88\x80\x80\x82 \xF8\x88\x80\x80\x83";
-            env(uritoken::mint(alice, uri),
-                ter(temMALFORMED));  // TODO: REVIEW - SHOULD FAIL
-
-            // All 2 first bytes of 6-byte sequences (0xfc-0xfd), each followed
-            // by a space character
-            uri = "\xFC\x84\x80\x80\x80\x80 \xFC\x84\x80\x80\x80\x81";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
-            // Sequences with last continuation byte missing
-
-            // Concatenation of incomplete sequences
+            // All 2 first bytes of 6-byte sequences (0xfc-0xfd), each
+            // followed by a space character
+            uri = "\xFC\x84\x80\x80\x80\x80 \xFC\x84\x80\x80\x80\x81";
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
             // Impossible bytes
             uri = "\xFE";
@@ -2617,15 +2567,113 @@ struct URIToken_test : public beast::unit_test::suite
             uri = "\xED\xAF\xBF\xED\xBF\xBF";
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
 
-            // problematic noncharacters in 16-bit applications
-            // case: (U+FFFE)
-            uri = "\xEF\xBF\xBE";
+            // MALFORMED - END
+            // ----------------------------------------------------------------
+        }
+
+        // =========================================================================
+        // Noncharacters U+FFFE and U+FFFF: well-formed UTF-8, but
+        // deliberately rejected (U+FFFE is a byte-swapped BOM).
+        // =========================================================================
+        {
+            uri = "\xEF\xBF\xBE";  // U+FFFE
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
-            // case: (U+FFFF)
-            uri = "\xEF\xBF\xBF";
+
+            uri = "\xEF\xBF\xBF";  // U+FFFF
+            env(uritoken::mint(alice, uri), ter(temMALFORMED));
+
+            uri = "ipfs://x\xEF\xBF\xBE";  // not only at the start
             env(uritoken::mint(alice, uri), ter(temMALFORMED));
         }
+
+        // =========================================================================
+        // Valid boundary cases that should always succeed
+        // =========================================================================
+        {
+            // BOUNDRY - START (valid minimums)
+            // ----------------------------------------------------------------
+
+            // case: 1 byte  (U-00000000). Must use the string
+            // constructor, not a C-string literal, to avoid strlen
+            // truncation to an empty URI.
+            uri = std::string(1, '\0');
+            env(uritoken::mint(alice, uri));
+            // case: 2 bytes (U-00000080)
+            uri = "\xC2\x80";
+            env(uritoken::mint(alice, uri));
+            // case: 3 bytes (U-00000800)
+            uri = "\xE0\xA0\x80";
+            env(uritoken::mint(alice, uri));
+            // case: 4 bytes (U-00010000)
+            uri = "\xF0\x90\x80\x80";
+            env(uritoken::mint(alice, uri));
+
+            // BOUNDRY - END (valid maximums)
+            // ----------------------------------------------------------------
+
+            // case: 1 byte  (U-0000007F)
+            uri = "\x7F";
+            env(uritoken::mint(alice, uri));
+            // case: 2 bytes (U-000007FF)
+            uri = "\xDF\xBF";
+            env(uritoken::mint(alice, uri));
+            // case: 3 bytes (U-0000FFFF) - noncharacter, rejected, see
+            // above.
+            // case: 4 bytes (U-001FFFFF) - beyond U+10FFFF, always
+            // rejected, see above.
+
+            // BOUNDRY - OTHER
+            // ----------------------------------------------------------------
+            // case: 2 bytes (U-0000E000) - above surrogates
+            uri = "\xEE\x80\x80";
+            env(uritoken::mint(alice, uri));
+            // case: 3 bytes (U-0000FFFD) - replacement character, valid
+            uri = "\xEF\xBF\xBD";
+            env(uritoken::mint(alice, uri));
+            // case: 4 bytes (U-0010FFFF) - max valid code point
+            uri = "\xF4\x8F\xBF\xBF";
+            env(uritoken::mint(alice, uri));
+        }
     }
+
+    void
+    testUTF8Validator()
+    {
+        testcase("uri_utf8 validator");
+
+        auto const check = [](std::string const& s) {
+            return isValidUTF8(makeSlice(s));
+        };
+
+        // Sequences truncated by the end of the buffer are invalid. The
+        // previous decoder read past the end of the vector here.
+        BEAST_EXPECT(!check("a\xC2"));
+        BEAST_EXPECT(!check("a\xE0\xA4"));
+        BEAST_EXPECT(!check("a\xEF\xBF"));
+        BEAST_EXPECT(!check("a\xF0\x90\x8D"));
+        BEAST_EXPECT(!check(std::string("\xC2", 1)));
+        BEAST_EXPECT(!check(std::string("\xF4\x8F\xBF", 3)));
+
+        // The complete sequences are fine.
+        BEAST_EXPECT(check("a\xC2\xA2"));
+        BEAST_EXPECT(check("a\xE0\xA4\xB9"));
+        BEAST_EXPECT(check("a\xF0\x90\x8D\x88"));
+        BEAST_EXPECT(check("a\xF4\x8F\xBF\xBF"));
+        BEAST_EXPECT(check(""));
+
+        // U+FFFE and U+FFFF are rejected wherever they appear.
+        BEAST_EXPECT(!check("\xEF\xBF\xBE"));
+        BEAST_EXPECT(!check("ab\xEF\xBF\xBF"));
+        BEAST_EXPECT(!check("\xE2\x82\xAC\xEF\xBF\xBEz"));
+
+        // Their neighbours are not, and nor are the other noncharacters,
+        // which the validator has never rejected.
+        BEAST_EXPECT(check("\xEF\xBF\xBD"));      // U+FFFD
+        BEAST_EXPECT(check("\xEF\xBE\xBF"));      // U+FFBF
+        BEAST_EXPECT(check("\xEF\xBB\xBF"));      // U+FEFF (BOM)
+        BEAST_EXPECT(check("\xF0\x9F\xBF\xBE"));  // U+1FFFE
+    }
+
     void
     testWithFeats(FeatureBitset features)
     {
@@ -2659,6 +2707,7 @@ public:
     {
         using namespace test::jtx;
         auto const sa = supported_amendments();
+        testUTF8Validator();
         testWithFeats(sa);
     }
 };

src/xrpld/app/tx/detail/Remit.cpp

@@ -18,12 +18,12 @@
 //==============================================================================
 
 #include <xrpld/app/tx/detail/Remit.h>
-#include <xrpld/app/tx/detail/URIToken.h>
 #include <xrpld/ledger/View.h>
 #include <xrpl/basics/Log.h>
 #include <xrpl/protocol/Feature.h>
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/TxFlags.h>
+#include <xrpl/protocol/UTF8.h>
 #include <xrpl/protocol/st.h>
 
 namespace ripple {
@@ -201,7 +201,7 @@ Remit::preflight(PreflightContext const& ctx)
             return temMALFORMED;
         }
 
-        if (!URIToken::validateUTF8(uri))
+        if (!isValidUTF8(makeSlice(uri)))
         {
             JLOG(ctx.j.warn())
                 << "Malformed transaction: Invalid UTF8 inside MintURIToken.";

src/xrpld/app/tx/detail/SetHook.cpp

@@ -23,7 +23,6 @@
 #include <xrpld/app/ledger/Ledger.h>
 #include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/ledger/OpenLedger.h>
-#include <xrpld/app/tx/detail/URIToken.h>
 #include <xrpld/ledger/ApplyView.h>
 #include <xrpl/basics/Log.h>
 #include <xrpl/hook/Enum.h>
@@ -35,6 +34,7 @@
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/STTx.h>
 #include <xrpl/protocol/TxFlags.h>
+#include <xrpl/protocol/UTF8.h>
 #include <algorithm>
 #include <cstdint>
 #include <exception>
@@ -657,7 +657,7 @@ SetHook::validateHookName(Blob const& name, beast::Journal const& j)
             << "sfHookName must be between 8 and 32 hex characters.";
         return false;
     }
-    if (!URIToken::validateUTF8(name))
+    if (!isValidUTF8(makeSlice(name)))
     {
         JLOG(j.trace()) << "sfHookName must be a valid UTF-8 string.";
         return false;

src/xrpld/app/tx/detail/URIToken.cpp

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/STAccount.h>
 #include <xrpl/protocol/TER.h>
 #include <xrpl/protocol/TxFlags.h>
+#include <xrpl/protocol/UTF8.h>
 
 namespace ripple {
 
@@ -98,7 +99,7 @@ URIToken::preflight(PreflightContext const& ctx)
             return temMALFORMED;
         }
 
-        if (!validateUTF8(uri))
+        if (!isValidUTF8(makeSlice(uri)))
         {
             JLOG(ctx.j.warn()) << "Malformed transaction. URI must be a "
                                   "valid utf-8 string.";

src/xrpld/app/tx/detail/URIToken.h

@@ -30,56 +30,6 @@ namespace ripple {
 class URIToken : public Transactor
 {
 public:
-    bool inline static validateUTF8(std::vector<uint8_t> const& u)
-    {
-        // this code is from
-        // https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c
-        uint8_t const* s = (uint8_t const*)u.data();
-        uint8_t const* end = s + u.size();
-        while (s < end)
-        {
-            if (*s < 0x80)
-                /* 0xxxxxxx */
-                s++;
-            else if ((s[0] & 0xe0) == 0xc0)
-            {
-                /* 110XXXXx 10xxxxxx */
-                if ((s[1] & 0xc0) != 0x80 ||
-                    (s[0] & 0xfe) == 0xc0) /* overlong? */
-                    return false;
-                else
-                    s += 2;
-            }
-            else if ((s[0] & 0xf0) == 0xe0)
-            {
-                /* 1110XXXX 10Xxxxxx 10xxxxxx */
-                if ((s[1] & 0xc0) != 0x80 || (s[2] & 0xc0) != 0x80 ||
-                    (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */
-                    (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */
-                    (s[0] == 0xef && s[1] == 0xbf &&
-                     (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */
-                    return false;
-                else
-                    s += 3;
-            }
-            else if ((s[0] & 0xf8) == 0xf0)
-            {
-                /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
-                if ((s[1] & 0xc0) != 0x80 || (s[2] & 0xc0) != 0x80 ||
-                    (s[3] & 0xc0) != 0x80 ||
-                    (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */
-                    (s[0] == 0xf4 && s[1] > 0x8f) ||
-                    s[0] > 0xf4) /* > U+10FFFF? */
-                    return false;
-                else
-                    s += 4;
-            }
-            else
-                return false;
-        }
-        return true;
-    }
-
     static constexpr ConsequencesFactoryType ConsequencesFactory{Normal};
 
     explicit URIToken(ApplyContext& ctx) : Transactor(ctx)