[DROP] disable SH runners for jshooks

- replace shell script with more peformant python script
- master header includes build/jshooks/generated/test.hook.$hash.inc files
  - looks for // @hook comment in hooks and places next to include
  - less annoying when running git diff without explicit ignore
  - first looks in usual location
- cmake locates or builds qjsc
  - searches for qjsc in previous place or honours QJSC_BINARY file
  - if not found builds from https://github.com/RichardAH/quickjslite

.gitattributes

@@ -11,3 +11,6 @@ LICENSE binary
 *.vcxproj   text eol=crlf
 *.props     text eol=crlf
 *.filters   text eol=crlf
+
+# Mark it as generated and folded out by default
+src/test/app/SetJSHook_wasm.h linguist-generated=true

.github/workflows/xahau-ga-macos.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: ["dev", "candidate", "release"]
   pull_request:
-    branches: ["dev", "candidate", "release"]
+    branches: ["dev", "candidate", "release", "jshooks"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/xahau-ga-nix.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: ["dev", "candidate", "release"]
   pull_request:
-    branches: ["dev", "candidate", "release"]
+    branches: ["dev", "candidate", "release", "jshooks"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/xahau-sh-build-in-docker.yml

@@ -1,63 +1,48 @@
-name: Build using Docker
+name: Release - SH Runner
 
 on:
   push:
-    branches: ["dev", "candidate", "release", "jshooks"]
+    branches: [ "dev", "candidate", "release"]
   pull_request:
-    branches: ["dev", "candidate", "release", "jshooks"]
+    branches: [ "dev", "candidate", "release"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP: 1
+  DEBUG_CONTAINERS: 1
+  REMOVE_CONTAINERS: 0
 
 jobs:
-  checkout:
+  build-and-test:
     runs-on: [self-hosted, vanity]
-    outputs:
-      checkout_path: ${{ steps.vars.outputs.checkout_path }}
     steps:
-      - name: Prepare checkout path
-        id: vars
+      - name: Prepare workspace
         run: |
           SAFE_BRANCH=$(echo "${{ github.ref_name }}" | sed -e 's/[^a-zA-Z0-9._-]/-/g')
           CHECKOUT_PATH="${SAFE_BRANCH}-${{ github.sha }}"
-          echo "checkout_path=${CHECKOUT_PATH}" >> "$GITHUB_OUTPUT"
+          echo "CHECKOUT_PATH=${CHECKOUT_PATH}" >> $GITHUB_ENV
+          mkdir -p "$CHECKOUT_PATH"
 
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
         with:
-          path: ${{ steps.vars.outputs.checkout_path }}
+          path: ${{ env.CHECKOUT_PATH }}
           clean: true
-          fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history
+          fetch-depth: 2
 
-  checkpatterns:
-    runs-on: [self-hosted, vanity]
-    needs: checkout
-    defaults:
-      run:
-        working-directory: ${{ needs.checkout.outputs.checkout_path }}
-    steps:
-      - name: Check for suspicious patterns
-        run: /bin/bash suspicious_patterns.sh
-
-  build:
-    runs-on: [self-hosted, vanity]
-    needs: [checkpatterns, checkout]
-    defaults:
-      run:
-        working-directory: ${{ needs.checkout.outputs.checkout_path }}
-    steps:
       - name: Set Cleanup Script Path
         run: |
           echo "JOB_CLEANUP_SCRIPT=$(mktemp)" >> $GITHUB_ENV
 
       - name: Build using Docker
+        working-directory: ${{ env.CHECKOUT_PATH }}
         run: /bin/bash release-builder.sh
 
-      - name: Stop Container (Cleanup)
+      - name: Stop Container (Cleanup after build)
         if: always()
+        working-directory: ${{ env.CHECKOUT_PATH }}
         run: |
           echo "Running cleanup script: $JOB_CLEANUP_SCRIPT"
           /bin/bash -e -x "$JOB_CLEANUP_SCRIPT"
@@ -71,35 +56,30 @@ jobs:
             echo "⚠️ Cleanup script failed! Keeping for debugging: $JOB_CLEANUP_SCRIPT"
           fi
 
-          if [[ "${DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP}" == "1" ]]; then
+          if [[ "${DEBUG_CONTAINERS}" == "1" ]]; then
             echo "🔍 Checking for leftover containers..."
             BUILD_CONTAINERS=$(docker ps --format '{{.Names}}' | grep '^xahaud_cached_builder' || echo "")
-
-            if [[ -n "$BUILD_CONTAINERS" ]]; then
+            CONTAINER_NAME="xahaud_cached_builder_${{ github.workflow }}-${{ github.ref_name }}"
+            if [[ -n "$BUILD_CONTAINERS" && "${REMOVE_CONTAINERS}" == "1" ]]; then
               echo "⚠️ WARNING: Some build containers are still running"
               echo "$BUILD_CONTAINERS"
+              echo "Attempting to stop build containers.."
+              echo "Stopping container: $CONTAINER_NAME"
+              docker stop "$CONTAINER_NAME" || echo "Failed to stop container: $CONTAINER_NAME"
+              echo "Removing container: $CONTAINER_NAME"
+              docker rm -f "$CONTAINER_NAME" || echo "Failed to remove container: $CONTAINER_NAME"
+              echo "✅ Build container stopped and removed"
             else
               echo "✅ No build containers found"
             fi
           fi
 
-  tests:
-    runs-on: [self-hosted, vanity]
-    needs: [build, checkout]
-    defaults:
-      run:
-        working-directory: ${{ needs.checkout.outputs.checkout_path }}
-    steps:
-      - name: Unit tests
+      - name: Run unit tests
+        working-directory: ${{ env.CHECKOUT_PATH }}
         run: /bin/bash docker-unit-tests.sh
 
-  cleanup:
-    runs-on: [self-hosted, vanity]
-    needs: [tests, checkout]
-    if: always()
-    steps:
       - name: Cleanup workspace
+        if: always()
         run: |
-          CHECKOUT_PATH="${{ needs.checkout.outputs.checkout_path }}"
           echo "Cleaning workspace for ${CHECKOUT_PATH}"
           rm -rf "${{ github.workspace }}/${CHECKOUT_PATH}"

.gitignore

@@ -115,4 +115,6 @@ pkg
 CMakeUserPresets.json
 bld.rippled/
 
+# binary
+qjsc
 generated

BUILD_LEDGER.md

@@ -1,465 +0,0 @@
-# Hash Migration Implementation via BuildLedger
-
-## Overview
-
-This document outlines the approach for implementing SHA-512 Half to BLAKE3 hash migration by performing the state map rekeying operation in the ledger building process, bypassing the metadata generation problem inherent in the transaction processing pipeline.
-
-## The Core Problem
-
-When switching from SHA-512 Half to BLAKE3, every object in the state map needs to be rekeyed because the hash (which IS the key in the SHAMap) changes. This would generate metadata showing:
-- Every object deleted at its old SHA-512 key
-- Every object created at its new BLAKE3 key
-- Total metadata size: 2× the entire state size (potentially gigabytes)
-
-## The Solution: Bypass Transaction Processing
-
-Instead of trying to rekey within the transaction processor (which tracks all changes for metadata), perform the rekeying AFTER transaction processing but BEFORE ledger finalization.
-
-## Implementation Location
-
-The key intervention point is in `buildLedgerImpl()` at line 63 of `BuildLedger.cpp`:
-
-```cpp
-// BuildLedger.cpp, lines 58-65
-{
-    OpenView accum(&*built);
-    assert(!accum.open());
-    applyTxs(accum, built);  // Apply transactions (including pseudo-txns)
-    accum.apply(*built);      // Apply accumulated changes to the ledger
-}
-// <-- INTERVENTION POINT HERE
-built->updateSkipList();
-```
-
-## Detailed Implementation
-
-### 1. Pseudo-Transaction Role (Simple Flag Setting)
-
-```cpp
-// In Change.cpp
-TER Change::applyHashMigration()
-{
-    // The pseudo-transaction just sets a flag
-    // The actual migration happens in BuildLedger
-    
-    JLOG(j_.warn()) << "Hash migration pseudo transaction triggered at ledger "
-                    << view().seq();
-    
-    // Create a migration flag object
-    auto migrationFlag = std::make_shared<SLE>(
-        keylet::hashMigrationFlag(
-            hash_options{view().seq(), KEYLET_MIGRATION_FLAG}));
-    
-    migrationFlag->setFieldU32(sfLedgerSequence, view().seq());
-    migrationFlag->setFieldU8(sfMigrationStatus, 1); // 1 = pending
-    
-    view().insert(migrationFlag);
-    
-    return tesSUCCESS;
-}
-```
-
-### 2. BuildLedger Modification
-
-```cpp
-// In BuildLedger.cpp, after line 63
-template <class ApplyTxs>
-std::shared_ptr<Ledger>
-buildLedgerImpl(
-    std::shared_ptr<Ledger const> const& parent,
-    NetClock::time_point closeTime,
-    const bool closeTimeCorrect,
-    NetClock::duration closeResolution,
-    Application& app,
-    beast::Journal j,
-    ApplyTxs&& applyTxs)
-{
-    auto built = std::make_shared<Ledger>(*parent, closeTime);
-
-    if (built->isFlagLedger() && built->rules().enabled(featureNegativeUNL))
-    {
-        built->updateNegativeUNL();
-    }
-
-    {
-        OpenView accum(&*built);
-        assert(!accum.open());
-        applyTxs(accum, built);
-        accum.apply(*built);
-    }
-    
-    // NEW: Check for hash migration flag
-    if (shouldPerformHashMigration(built, app, j))
-    {
-        performHashMigration(built, app, j);
-    }
-
-    built->updateSkipList();
-    // ... rest of function
-}
-
-// New helper functions
-bool shouldPerformHashMigration(
-    std::shared_ptr<Ledger> const& ledger,
-    Application& app,
-    beast::Journal j)
-{
-    // Check if we're in the migration window
-    constexpr LedgerIndex MIGRATION_START = 20'000'000;
-    constexpr LedgerIndex MIGRATION_END = 20'000'010;
-    
-    if (ledger->seq() < MIGRATION_START || ledger->seq() >= MIGRATION_END)
-        return false;
-    
-    // Check for migration flag
-    auto const flag = ledger->read(keylet::hashMigrationFlag(
-        hash_options{ledger->seq(), KEYLET_MIGRATION_FLAG}));
-    
-    if (!flag)
-        return false;
-    
-    return flag->getFieldU8(sfMigrationStatus) == 1; // 1 = pending
-}
-
-void performHashMigration(
-    std::shared_ptr<Ledger> const& ledger,
-    Application& app,
-    beast::Journal j)
-{
-    JLOG(j.warn()) << "PERFORMING HASH MIGRATION at ledger " << ledger->seq();
-    
-    auto& oldStateMap = ledger->stateMap();
-    
-    // Create new state map with BLAKE3 hashing
-    SHAMap newStateMap(SHAMapType::STATE, ledger->family());
-    newStateMap.setLedgerSeq(ledger->seq());
-    
-    // Track statistics
-    std::size_t objectCount = 0;
-    auto startTime = std::chrono::steady_clock::now();
-    
-    // Walk the entire state map and rekey everything
-    oldStateMap.visitLeaves([&](SHAMapItem const& item) {
-        try {
-            // Deserialize the ledger entry
-            SerialIter sit(item.slice());
-            auto sle = std::make_shared<SLE>(sit, item.key());
-            
-            // The new key would be calculated with BLAKE3
-            // For now, we'd need the actual BLAKE3 implementation
-            // uint256 newKey = calculateBlake3Key(sle);
-            
-            // For this example, let's assume we have a function that 
-            // computes the new key based on the SLE type and contents
-            uint256 newKey = computeNewHashKey(sle, ledger->seq());
-            
-            // Re-serialize the SLE
-            Serializer s;
-            sle->add(s);
-            
-            // Add to new map with new key
-            newStateMap.addGiveItem(
-                SHAMapNodeType::tnACCOUNT_STATE,
-                make_shamapitem(newKey, s.slice()));
-            
-            objectCount++;
-            
-            if (objectCount % 10000 == 0) {
-                JLOG(j.info()) << "Migration progress: " << objectCount 
-                              << " objects rekeyed";
-            }
-        }
-        catch (std::exception const& e) {
-            JLOG(j.error()) << "Failed to migrate object " << item.key()
-                           << ": " << e.what();
-            throw;
-        }
-    });
-    
-    auto endTime = std::chrono::steady_clock::now();
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-        endTime - startTime);
-    
-    JLOG(j.warn()) << "Hash migration completed: " << objectCount 
-                   << " objects rekeyed in " << duration.count() << "ms";
-    
-    // Swap the state maps
-    oldStateMap = std::move(newStateMap);
-    
-    // Update the migration flag to completed
-    auto flag = ledger->peek(keylet::hashMigrationFlag(
-        hash_options{ledger->seq(), KEYLET_MIGRATION_FLAG}));
-    if (flag) {
-        flag->setFieldU8(sfMigrationStatus, 2); // 2 = completed
-        ledger->rawReplace(flag);
-    }
-}
-
-uint256 computeNewHashKey(
-    std::shared_ptr<SLE const> const& sle,
-    LedgerIndex ledgerSeq)
-{
-    // This would use BLAKE3 instead of SHA-512 Half
-    // Implementation depends on the BLAKE3 integration
-    // For now, this is a placeholder
-    
-    // The actual implementation would:
-    // 1. Determine the object type
-    // 2. Extract the identifying fields
-    // 3. Hash them with BLAKE3
-    // 4. Return the new key
-    
-    return uint256(); // Placeholder
-}
-```
-
-## Why This Approach Works
-
-### 1. No Metadata Explosion
-- The rekeying happens AFTER the `OpenView` is destroyed
-- No change tracking occurs during the rekeying
-- Only the migration flag generates metadata (minimal)
-
-### 2. Direct SHAMap Access
-- We have direct access to `built->stateMap()`
-- Can manipulate the raw data structure without going through ApplyView
-- Can create a new SHAMap and swap it in
-
-### 3. Clean Separation of Concerns
-- Pseudo-transaction: "Signal that migration should happen"
-- BuildLedger: "Actually perform the migration"
-- Transaction processor: Unchanged, doesn't need to handle massive rekeying
-
-### 4. Timing is Perfect
-- After all transactions are applied
-- Before the ledger is finalized
-- Before the skip list is updated
-- Before the SHAMap is flushed to disk
-
-## Files Referenced in This Analysis
-
-### Core Implementation Files
-- `src/ripple/app/ledger/impl/BuildLedger.cpp` - Main implementation location
-- `src/ripple/app/ledger/BuildLedger.h` - Header for build functions
-- `src/ripple/app/tx/impl/Change.cpp` - Pseudo-transaction handler
-- `src/ripple/app/tx/impl/Change.h` - Change transactor header
-
-### Transaction Processing Pipeline (analyzed but bypassed)
-- `src/ripple/app/tx/impl/Transactor.cpp` - Base transaction processor
-- `src/ripple/app/tx/impl/Transactor.h` - Transactor header
-- `src/ripple/app/tx/impl/apply.cpp` - Transaction application
-- `src/ripple/app/tx/impl/applySteps.cpp` - Transaction routing
-- `src/ripple/app/tx/impl/ApplyContext.h` - Application context
-
-### Ledger and View Classes
-- `src/ripple/app/ledger/Ledger.h` - Ledger class definition
-- `src/ripple/app/ledger/Ledger.cpp` - Ledger implementation
-- `src/ripple/ledger/ApplyView.h` - View interface
-- `src/ripple/ledger/ApplyViewImpl.h` - View implementation header
-- `src/ripple/ledger/impl/ApplyViewImpl.cpp` - View implementation
-- `src/ripple/ledger/impl/ApplyViewBase.cpp` - Base view implementation
-- `src/ripple/ledger/detail/ApplyViewBase.h` - Base view header
-- `src/ripple/ledger/OpenView.h` - Open ledger view
-- `src/ripple/ledger/RawView.h` - Raw view interface
-
-### SHAMap and Data Structures
-- `src/ripple/shamap/SHAMap.h` - SHAMap class definition
-
-### Metadata Generation
-- `src/ripple/protocol/TxMeta.h` - Transaction metadata header
-- `src/ripple/protocol/impl/TxMeta.cpp` - Metadata implementation
-
-### Consensus and Pseudo-Transaction Injection
-- `src/ripple/app/consensus/RCLConsensus.cpp` - Consensus implementation
-
-### Supporting Documents
-- `PSEUDO_TRANSACTIONS.md` - Documentation on pseudo-transactions
-- `HASH_MIGRATION_CONTEXT.md` - Context for hash migration work
-
-## Key Advantages
-
-1. **Architecturally Clean**: Works within existing ledger building framework
-2. **No Metadata Issues**: Completely bypasses the metadata generation problem
-3. **Atomic Operation**: Either the entire state is rekeyed or none of it is
-4. **Fail-Safe**: Can be wrapped in try-catch for error handling
-5. **Observable**: Can log progress for large state maps
-6. **Testable**: Can be tested independently of transaction processing
-
-## Challenges and Considerations
-
-1. **Performance**: Rekeying millions of objects will take time
-   - Solution: This happens during consensus, all nodes do it simultaneously
-   
-2. **Memory Usage**: Need to hold both old and new SHAMaps temporarily
-   - Solution: Could potentially do in-place updates with careful ordering
-   
-3. **Verification**: Need to ensure all nodes get the same result
-   - Solution: Deterministic rekeying based on ledger sequence
-
-4. **Rollback**: If migration fails, need to handle gracefully
-   - Solution: Keep old map until new map is fully built and verified
-
-## Conclusion
-
-By performing the hash migration at the ledger building level rather than within the transaction processing pipeline, we can successfully rekey the entire state map without generating massive metadata. This approach leverages the existing architecture's separation between transaction processing and ledger construction, providing a clean and efficient solution to what initially appeared to be an intractable problem.
-
----
-
-## APPENDIX: Revised Implementation Following Ledger Pattern
-
-After reviewing the existing pattern in `BuildLedger.cpp`, it's clear that special ledger operations are implemented as methods on the `Ledger` class itself (e.g., `built->updateNegativeUNL()`). Following this pattern, the hash migration should be implemented as `Ledger::migrateToBlake3()`.
-
-### Updated BuildLedger.cpp Implementation
-
-```cpp
-// In BuildLedger.cpp, following the existing pattern
-template <class ApplyTxs>
-std::shared_ptr<Ledger>
-buildLedgerImpl(
-    std::shared_ptr<Ledger const> const& parent,
-    NetClock::time_point closeTime,
-    const bool closeTimeCorrect,
-    NetClock::duration closeResolution,
-    Application& app,
-    beast::Journal j,
-    ApplyTxs&& applyTxs)
-{
-    auto built = std::make_shared<Ledger>(*parent, closeTime);
-
-    if (built->isFlagLedger() && built->rules().enabled(featureNegativeUNL))
-    {
-        built->updateNegativeUNL();
-    }
-
-    {
-        OpenView accum(&*built);
-        assert(!accum.open());
-        applyTxs(accum, built);
-        accum.apply(*built);
-    }
-    
-    // NEW: Check and perform hash migration following the pattern
-    if (built->rules().enabled(featureBLAKE3Migration) && 
-        built->shouldMigrateToBlake3())
-    {
-        built->migrateToBlake3();
-    }
-
-    built->updateSkipList();
-    // ... rest of function
-}
-```
-
-### Ledger.h Addition
-
-```cpp
-// In src/ripple/app/ledger/Ledger.h
-class Ledger final : public std::enable_shared_from_this<Ledger>,
-                     public DigestAwareReadView,
-                     public TxsRawView,
-                     public CountedObject<Ledger>
-{
-public:
-    // ... existing methods ...
-
-    /** Update the Negative UNL ledger component. */
-    void
-    updateNegativeUNL();
-
-    /** Check if hash migration to BLAKE3 should be performed */
-    bool
-    shouldMigrateToBlake3() const;
-
-    /** Perform hash migration from SHA-512 Half to BLAKE3
-     *  This rekeys all objects in the state map with new BLAKE3 hashes.
-     *  Must be called after transactions are applied but before the
-     *  ledger is finalized.
-     */
-    void
-    migrateToBlake3();
-
-    // ... rest of class ...
-};
-```
-
-### Ledger.cpp Implementation
-
-```cpp
-// In src/ripple/app/ledger/Ledger.cpp
-
-bool
-Ledger::shouldMigrateToBlake3() const
-{
-    // Check if we're in the migration window
-    constexpr LedgerIndex MIGRATION_START = 20'000'000;
-    constexpr LedgerIndex MIGRATION_END = 20'000'010;
-    
-    if (seq() < MIGRATION_START || seq() >= MIGRATION_END)
-        return false;
-    
-    // Check for migration flag set by pseudo-transaction
-    auto const flag = read(keylet::hashMigrationFlag(
-        hash_options{seq(), KEYLET_MIGRATION_FLAG}));
-    
-    if (!flag)
-        return false;
-    
-    return flag->getFieldU8(sfMigrationStatus) == 1; // 1 = pending
-}
-
-void
-Ledger::migrateToBlake3()
-{
-    JLOG(j_.warn()) << "Performing BLAKE3 hash migration at ledger " << seq();
-    
-    // Create new state map with BLAKE3 hashing
-    SHAMap newStateMap(SHAMapType::STATE, stateMap_.family());
-    newStateMap.setLedgerSeq(seq());
-    
-    std::size_t objectCount = 0;
-    auto startTime = std::chrono::steady_clock::now();
-    
-    // Walk the entire state map and rekey everything
-    stateMap_.visitLeaves([&](SHAMapItem const& item) {
-        // Deserialize the ledger entry
-        SerialIter sit(item.slice());
-        auto sle = std::make_shared<SLE>(sit, item.key());
-        
-        // Calculate new BLAKE3-based key
-        // This would use the actual BLAKE3 implementation
-        uint256 newKey = computeBlake3Key(sle);
-        
-        // Re-serialize and add to new map
-        Serializer s;
-        sle->add(s);
-        
-        newStateMap.addGiveItem(
-            SHAMapNodeType::tnACCOUNT_STATE,
-            make_shamapitem(newKey, s.slice()));
-        
-        if (++objectCount % 10000 == 0) {
-            JLOG(j_.info()) << "Migration progress: " << objectCount 
-                          << " objects rekeyed";
-        }
-    });
-    
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-        std::chrono::steady_clock::now() - startTime);
-    
-    JLOG(j_.warn()) << "BLAKE3 migration completed: " << objectCount 
-                   << " objects rekeyed in " << duration.count() << "ms";
-    
-    // Swap the state maps
-    stateMap_ = std::move(newStateMap);
-    
-    // Update the migration flag to completed
-    auto flag = peek(keylet::hashMigrationFlag(
-        hash_options{seq(), KEYLET_MIGRATION_FLAG}));
-    if (flag) {
-        flag->setFieldU8(sfMigrationStatus, 2); // 2 = completed
-        rawReplace(flag);
-    }
-}
-```
-
-This approach follows the established pattern in the codebase where special ledger operations are encapsulated as methods on the `Ledger` class itself, making the code more maintainable and consistent with the existing architecture.

Builds/CMake/RippledCore.cmake

@@ -150,10 +150,10 @@ target_link_libraries (xrpl_core
     OpenSSL::Crypto
     Ripple::boost
     wasmedge::wasmedge
+    NIH::quickjs
     Ripple::syslibs
     secp256k1::secp256k1
     ed25519::ed25519
-    BLAKE3::blake3
     date::date
     Ripple::opts)
 #[=================================[
@@ -330,6 +330,11 @@ if (tests)
       src/ripple/beast/unit_test/detail/const_container.hpp
     DESTINATION include/ripple/beast/unit_test/detail)
 endif () #tests
+#[===================================================================[
+   JS Hooks integration 
+#]===================================================================]
+include(Builds/CMake/jshooks/JSHooks.cmake)
+
 #[===================================================================[
    rippled executable
 #]===================================================================]
@@ -638,7 +643,6 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/LogLevel.cpp
   src/ripple/rpc/handlers/LogRotate.cpp
   src/ripple/rpc/handlers/Manifest.cpp
-  src/ripple/rpc/handlers/MapStats.cpp
   src/ripple/rpc/handlers/NFTOffers.cpp
   src/ripple/rpc/handlers/NodeToShard.cpp
   src/ripple/rpc/handlers/NoRippleCheck.cpp
@@ -779,6 +783,9 @@ if (tests)
     src/test/app/ValidatorKeys_test.cpp
     src/test/app/ValidatorList_test.cpp
     src/test/app/ValidatorSite_test.cpp
+    src/test/app/SetHook_test.cpp
+    src/test/app/SetJSHook_test.cpp
+    src/test/app/SetHookTSH_test.cpp
     src/test/app/Wildcard_test.cpp
     src/test/app/XahauGenesis_test.cpp
     src/test/app/tx/apply_test.cpp
@@ -975,7 +982,6 @@ if (tests)
        test sources:
          subdir: protocol
     #]===============================]
-    src/test/protocol/blake3_test.cpp
     src/test/protocol/BuildInfo_test.cpp
     src/test/protocol/InnerObjectFormats_test.cpp
     src/test/protocol/Issue_test.cpp
@@ -1066,14 +1072,16 @@ if (tests)
     src/test/unit_test/multi_runner.cpp)
 endif () #tests
 
+# Add JS hooks support to rippled when tests are enabled
+if (tests)
+  target_add_jshooks(rippled)
+endif () #tests
+
 target_link_libraries (rippled
   Ripple::boost
   Ripple::opts
   Ripple::libs
   Ripple::xrpl_core
-  BLAKE3::blake3
-  # Workaround for a Conan 1.x bug...
-  m
   )
 exclude_if_included (rippled)
 # define a macro for tests that might need to

Builds/CMake/RippledInstall.cmake

@@ -4,6 +4,7 @@
 
 install (
   TARGETS
+    quickjs
     common
     opts
     ripple_syslibs

Builds/CMake/deps/Rocksdb.cmake

@@ -80,6 +80,7 @@ if (local_rocksdb)
       -DCMAKE_DEBUG_POSTFIX=_d
       $<$<NOT:$<BOOL:${is_multiconfig}>>:-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}>
       -DBUILD_SHARED_LIBS=OFF
+      -DROCKSDB_BUILD_SHARED=OFF
       -DCMAKE_POSITION_INDEPENDENT_CODE=ON
       -DWITH_JEMALLOC=$<IF:$<BOOL:${jemalloc}>,ON,OFF>
       -DWITH_SNAPPY=ON

Builds/CMake/deps/quickjs.cmake

@@ -0,0 +1,52 @@
+# QuickJS source files
+set(QUICKJS_SOURCES
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/libutf.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/qsort_r.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/tutf8e.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/vector.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/char-utils.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/buffer-utils.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/utils.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/ringbuffer.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/libregexp.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/quickjs.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/libregexp.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/libunicode.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/cutils.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/quickjs-libc.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/libbf.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/quickjs-textcode.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/quickjs-internal.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/debug.c
+)
+
+# QuickJS include directories
+set(QUICKJS_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs)
+
+# RHTODO: Fix this to be dynamic
+set(QUICKJS_VERSION "2023-01-01")
+
+# QuickJS compile definitions
+set(QUICKJS_COMPILE_DEFINITIONS
+    _GNU_SOURCE
+    CONFIG_VERSION="${QUICKJS_VERSION}"
+    CONFIG_BIGNUM
+)
+
+# Create static library
+add_library(quickjs STATIC ${QUICKJS_SOURCES})
+target_include_directories(quickjs
+  PUBLIC
+    $<BUILD_INTERFACE:${QUICKJS_INCLUDE_DIRS}>
+    $<INSTALL_INTERFACE:include/quickjs>
+)
+target_compile_definitions(quickjs PUBLIC ${QUICKJS_COMPILE_DEFINITIONS})
+
+add_library(NIH::quickjs ALIAS quickjs)
+target_link_libraries(ripple_libs INTERFACE NIH::quickjs)
+
+install(
+    FILES
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/quickjs/*.h
+    DESTINATION include/quickjs
+)

Builds/CMake/jshooks/JSHooks.cmake

@@ -0,0 +1,102 @@
+# JSHooks.cmake Generate JS hook headers for tests
+
+# Find Python3
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+# Set paths
+set(JSHOOK_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/test/app)
+set(JSHOOK_BUILD_DIR ${CMAKE_BINARY_DIR}/jshooks/generated)
+set(JSHOOK_CACHE_DIR ${CMAKE_BINARY_DIR}/jshooks/cache)
+set(JSHOOK_SCRIPT_DIR ${CMAKE_CURRENT_LIST_DIR})
+
+# Create directories
+file(MAKE_DIRECTORY ${JSHOOK_BUILD_DIR})
+file(MAKE_DIRECTORY ${JSHOOK_CACHE_DIR})
+
+# Download and build qjsc if not provided
+if(NOT DEFINED QJSC_BINARY OR QJSC_BINARY STREQUAL "")
+  # Check if qjsc exists locally first
+  set(LOCAL_QJSC_PATH ${JSHOOK_SOURCE_DIR}/qjsc)
+  if(EXISTS ${LOCAL_QJSC_PATH})
+    set(QJSC_BINARY ${LOCAL_QJSC_PATH})
+  else()
+    # Use FetchContent to download and build quickjslite
+    include(FetchContent)
+    
+    FetchContent_Declare(
+      quickjslite
+      GIT_REPOSITORY https://github.com/RichardAH/quickjslite.git
+      GIT_TAG        863bf70645dcf21e960af2f1540f40029d09498e
+    )
+    
+    FetchContent_MakeAvailable(quickjslite)
+    
+    # Build qjsc using make
+    set(QUICKJS_SOURCE_DIR ${quickjslite_SOURCE_DIR})
+    set(QUICKJS_BINARY_DIR ${quickjslite_BINARY_DIR})
+    set(QJSC_BINARY ${QUICKJS_SOURCE_DIR}/qjsc)
+    
+    # Custom target to build qjsc
+    include(ProcessorCount)
+    ProcessorCount(N)
+    if(NOT N EQUAL 0)
+      set(MAKE_JOBS -j${N})
+    else()
+      set(MAKE_JOBS "")
+    endif()
+    
+    add_custom_command(
+      OUTPUT ${QJSC_BINARY}
+      COMMAND ${CMAKE_COMMAND} -E chdir ${QUICKJS_SOURCE_DIR} make ${MAKE_JOBS} qjsc
+      DEPENDS ${QUICKJS_SOURCE_DIR}/Makefile
+      COMMENT "Building qjsc from quickjslite (using ${N} jobs)..."
+      VERBATIM
+    )
+    
+    # Create a target for qjsc dependency
+    add_custom_target(build_qjsc DEPENDS ${QJSC_BINARY})
+  endif()
+endif()
+
+# Set up dependencies for qjsc
+if(TARGET build_qjsc)
+  set(QJSC_DEPENDENCY build_qjsc)
+else()
+  set(QJSC_DEPENDENCY "")
+endif()
+
+# Custom command to generate headers
+add_custom_command(
+  OUTPUT ${JSHOOK_SOURCE_DIR}/SetJSHook_wasm.h ${JSHOOK_BUILD_DIR}/.timestamp
+  COMMAND
+    ${CMAKE_COMMAND} -E env QJSC_BINARY=${QJSC_BINARY} ${Python3_EXECUTABLE}
+    ${JSHOOK_SCRIPT_DIR}/build_test_jshooks.py --source
+    ${JSHOOK_SOURCE_DIR}/SetJSHook_test.cpp --master
+    ${JSHOOK_SOURCE_DIR}/SetJSHook_wasm.h --output-dir ${JSHOOK_BUILD_DIR}
+    --cache-dir ${JSHOOK_CACHE_DIR} --log-level info
+  COMMAND ${CMAKE_COMMAND} -E touch ${JSHOOK_BUILD_DIR}/.timestamp
+  DEPENDS ${JSHOOK_SOURCE_DIR}/SetJSHook_test.cpp
+          ${JSHOOK_SCRIPT_DIR}/build_test_jshooks.py
+          ${QJSC_BINARY}
+  WORKING_DIRECTORY ${JSHOOK_SOURCE_DIR}
+  COMMENT "Generating JS hook headers..."
+  VERBATIM)
+
+# Custom target
+add_custom_target(generate_js_hooks ALL
+                  DEPENDS ${JSHOOK_SOURCE_DIR}/SetJSHook_wasm.h)
+
+# Add qjsc dependency if needed
+if(QJSC_DEPENDENCY)
+  add_dependencies(generate_js_hooks ${QJSC_DEPENDENCY})
+endif()
+
+# Function to add JS hook support to a target
+function(target_add_jshooks TARGET)
+  target_include_directories(${TARGET} PRIVATE ${CMAKE_BINARY_DIR}/jshooks)
+  add_dependencies(${TARGET} generate_js_hooks)
+endfunction()
+
+# Export variables for use by consumers
+set(JSHOOK_GENERATED_DIR ${JSHOOK_BUILD_DIR})
+set(JSHOOK_MASTER_HEADER ${JSHOOK_SOURCE_DIR}/SetJSHook_wasm.h)

Builds/CMake/jshooks/build_test_jshooks.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import logging
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+
+def get_qjsc_hash(qjsc_path):
+    """Generate a hash of the qjsc binary to include in the cache key."""
+    try:
+        with open(qjsc_path, "rb") as f:
+            return hashlib.sha256(f.read()).hexdigest()[
+                :16
+            ]  # Use first 16 chars of hash for brevity
+    except Exception as e:
+        logging.warning(f"Could not hash qjsc binary: {e}")
+        return "unknown"
+
+
+def find_qjsc_binary():
+    """Find qjsc binary in expected locations."""
+    # Check environment variable first
+    if "QJSC_BINARY" in os.environ:
+        qjsc_path = os.environ["QJSC_BINARY"]
+        if os.path.exists(qjsc_path) and os.access(qjsc_path, os.X_OK):
+            return qjsc_path
+        else:
+            logging.error(f"QJSC_BINARY points to invalid path: {qjsc_path}")
+            sys.exit(1)
+
+    # Check current directory
+    qjsc_path = "./qjsc"
+    if os.path.exists(qjsc_path) and os.access(qjsc_path, os.X_OK):
+        return qjsc_path
+
+    # Not found
+    logging.error(
+        "qjsc not found in current directory.\n"
+        "This requires the custom quickjslite build from:\n"
+        "https://github.com/RichardAH/quickjslite\n"
+        "Build it and place in src/test/app/ or set QJSC_BINARY environment variable"
+    )
+    sys.exit(1)
+
+
+def convert_js_to_carray(js_file, js_content, qjsc_path):
+    """
+    Convert a JavaScript file to a C array using qjsc.
+    Extracts just the hex bytes from the qjsc output.
+    """
+    try:
+        # Run qjsc to compile the JavaScript file to C code
+        result = subprocess.run(
+            [qjsc_path, "-c", "-o", "/dev/stdout", js_file],
+            capture_output=True,
+            check=True,
+        )
+
+        # Check if we have any output
+        if not result.stdout:
+            logging.error(f"Error: qjsc produced no output for {js_file}")
+            sys.exit(1)
+
+        # Convert to text and extract just the hex values
+        output_text = result.stdout.decode("utf-8", errors="replace")
+
+        # Extract hexadecimal values from the array definition
+        # Looking for patterns like 0x43, 0x0c, etc.
+        hex_values = re.findall(r"0x[0-9A-Fa-f]{2}", output_text)
+
+        # Format them as 0xXXU for the C array
+        c_array = ", ".join([f"{hex_val}U" for hex_val in hex_values])
+
+        return c_array
+
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Error executing qjsc: {e}, content: {js_content}")
+        if e.stderr:
+            logging.error(f"stderr: {e.stderr.decode('utf-8', errors='replace')}")
+        sys.exit(1)
+
+
+def extract_hook_description(js_content, start_line=None):
+    """Extract description from // @hook comment at start of JS content."""
+    lines = js_content.split("\n")
+    for line in lines:
+        line = line.strip()
+        if line.startswith("// @hook "):
+            return line[9:]  # Remove '// @hook ' prefix
+        elif line and not line.startswith("//"):
+            break  # Stop at first non-comment line
+    
+    # If no @hook comment found, return line number info
+    if start_line is not None:
+        return f"MISSING @hook (line {start_line})"
+    return None
+
+
+def generate_content_hash(content):
+    """Generate 8-character hash for content."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()[:8]
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate JS hook headers with individual .inc files"
+    )
+    parser.add_argument(
+        "--source",
+        default="SetJSHook_test.cpp",
+        help="Source file containing JS hooks",
+    )
+    parser.add_argument(
+        "--master",
+        default="SetJSHook_wasm.h",
+        help="Master header file to generate",
+    )
+    parser.add_argument(
+        "--output-dir",
+        required=True,
+        help="Directory for generated .inc files",
+    )
+    parser.add_argument(
+        "--cache-dir",
+        help="Directory for compilation cache (optional)",
+    )
+    parser.add_argument(
+        "--xahaud-root",
+        help="Xahaud root directory (if not specified, will try to auto-detect)",
+    )
+    parser.add_argument(
+        "--log-level",
+        default="error",
+        choices=["debug", "info", "warning", "error", "critical"],
+        help="Set logging level (default: error)",
+    )
+
+    args = parser.parse_args()
+
+    # Configure logging
+    numeric_level = getattr(logging, args.log_level.upper(), None)
+    logging.basicConfig(
+        level=numeric_level, format="[jshooks] %(levelname)s: %(message)s"
+    )
+
+    # Set working directory if xahaud_root is provided
+    if args.xahaud_root:
+        working_dir = os.path.join(args.xahaud_root, "src/test/app")
+        if os.path.exists(working_dir):
+            os.chdir(working_dir)
+            logging.info(f"Changed working directory to: {working_dir}")
+        else:
+            logging.error(f"Working directory does not exist: {working_dir}")
+            sys.exit(1)
+
+    # Find qjsc binary
+    qjsc_path = find_qjsc_binary()
+    logging.info(f"Using qjsc: {qjsc_path}")
+
+    # Create output directories
+    Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+    if args.cache_dir:
+        Path(args.cache_dir).mkdir(parents=True, exist_ok=True)
+
+    # Get hash of qjsc binary for cache key
+    qjsc_hash = get_qjsc_hash(qjsc_path)
+
+    # Make paths relative for logging
+    relative_source = f"src/test/app/{os.path.basename(args.source)}"
+    relative_master = f"src/test/app/{os.path.basename(args.master)}"
+    logging.info(f"Processing {relative_source} to generate {relative_master}...")
+
+    # Check if input file exists
+    if not os.path.exists(args.source):
+        logging.error(f"Error: Input file '{args.source}' not found.")
+        sys.exit(1)
+
+    # Read input file content
+    with open(args.source, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    content_with_form_feeds = content.replace("\n", "\f")
+
+    # Get hooks using regex that matches the original bash script's grep command
+    pattern = r'R"\[test\.hook\](.*?)\[test\.hook\]"'
+    hook_matches = list(re.finditer(pattern, content_with_form_feeds, re.DOTALL))
+
+    if not hook_matches:
+        logging.warning("Warning: No test hooks found in the input file.")
+        # Create empty master header
+        with open(args.master, "w") as f:
+            f.write(
+                f"""// Generated by build_test_jshooks.py - DO COMMIT THIS FILE
+// Last updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
+// 
+// This file provides a registry of all JS test hooks.
+// Individual compiled hooks are in build/jshooks/generated/
+
+#ifndef SETHOOK_JSWASM_INCLUDED
+#define SETHOOK_JSWASM_INCLUDED
+
+#include <map>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+namespace ripple {{
+namespace test {{
+
+// Map of all JS test hooks - populated at compile time
+std::map<std::string, std::vector<uint8_t>> jswasm = {{
+    // No hooks found
+}};
+
+}} // namespace test
+}} // namespace ripple
+
+#endif
+"""
+            )
+        sys.exit(0)
+
+    # Process the matches and calculate line numbers
+    processed_matches = []
+    hook_line_ranges = []
+
+    for match in hook_matches:
+        # Calculate start and end line numbers
+        start_line = content_with_form_feeds[: match.start()].count("\f") + 1
+        end_line = content_with_form_feeds[: match.end()].count("\f") + 1
+        hook_line_ranges.append((start_line, end_line))
+
+        # Extract the content and process it
+        raw_content = match.group(1)
+        # Remove the opening tag
+        processed = re.sub(r"^\(", "", raw_content)
+        # Remove the closing tag and any trailing whitespace/form feeds
+        processed = re.sub(r"\)[\f \t]*$", "/*end*/", processed)
+        processed_matches.append(processed)
+
+    # Generate individual .inc files and collect include statements
+    include_statements = []
+
+    counter = 0
+    for hook_content in processed_matches:
+        logging.debug(f"Processing hook {counter}...")
+
+        # Clean content and convert back to newlines
+        clean_content = (
+            hook_content[:-7] if hook_content.endswith("/*end*/") else hook_content
+        )
+        clean_content = clean_content.replace("\f", "\n")
+
+        # Check if this is a WebAssembly module
+        wat_count = len(re.findall(r"\(module", hook_content))
+        if wat_count > 0:
+            logging.error(f"Error: WebAssembly text format detected in hook {counter}")
+            sys.exit(1)
+
+        # Generate hash for this hook
+        content_hash = generate_content_hash(clean_content)
+        inc_filename = f"test.hook.{content_hash}.inc"
+        inc_path = os.path.join(args.output_dir, inc_filename)
+
+        # Extract description
+        start_line, end_line = hook_line_ranges[counter]
+        description = extract_hook_description(clean_content, start_line)
+        comment = f"  // {description}" if description else ""
+
+        # Add to include statements
+        include_statements.append(f'    #include "generated/{inc_filename}"{comment}')
+
+        # Check cache
+        cache_file = None
+        if args.cache_dir:
+            cache_content_hash = hashlib.sha256(
+                f"{qjsc_hash}:{clean_content}".encode("utf-8")
+            ).hexdigest()
+            cache_file = os.path.join(
+                args.cache_dir, f"hook-{cache_content_hash}.c_array"
+            )
+
+        # Get or generate C array
+        if args.cache_dir and cache_file and os.path.exists(cache_file):
+            logging.debug(f"Using cached version for hook {counter}")
+            with open(cache_file, "r") as cache:
+                c_array = cache.read()
+        else:
+            logging.debug(f"Compiling hook {counter}...")
+            # Generate JS file
+            js_content = clean_content + "\n"
+            js_file = os.path.join(args.output_dir, f"temp-{counter}-gen.js")
+            with open(js_file, "w", encoding="utf-8") as f:
+                f.write(js_content)
+
+            try:
+                c_array = convert_js_to_carray(js_file, js_content, qjsc_path)
+
+                # Cache the result
+                if args.cache_dir and cache_file:
+                    with open(cache_file, "w") as cache:
+                        cache.write(c_array)
+
+                # Clean up temp file
+                os.remove(js_file)
+            except Exception as e:
+                logging.error(f"Compilation error for hook {counter}: {e}")
+                sys.exit(1)
+
+        # Generate .inc file with actual line range
+        start_line, end_line = hook_line_ranges[counter]
+        # Make source path relative to repo root
+        relative_source = f"src/test/app/{os.path.basename(args.source)}"
+        with open(inc_path, "w") as f:
+            f.write(
+                f"""// Auto-generated - DO NOT EDIT
+// Source: {relative_source}:{start_line} ({start_line}:{end_line})
+// Hash: {content_hash}
+
+{{R"[test.hook]({clean_content})[test.hook]", {{
+{c_array}
+}}}},
+"""
+            )
+
+        counter += 1
+
+    # Generate master header
+    with open(args.master, "w") as f:
+        f.write(
+            f"""// Generated by build_test_jshooks.py - DO COMMIT THIS FILE
+// Last updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
+// 
+// This file provides a registry of all JS test hooks.
+// Individual compiled hooks are in build/jshooks/generated/
+
+#ifndef SETHOOK_JSWASM_INCLUDED
+#define SETHOOK_JSWASM_INCLUDED
+
+#include <map>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+namespace ripple {{
+namespace test {{
+
+// Map of all JS test hooks - populated at compile time
+std::map<std::string, std::vector<uint8_t>> jswasm = {{
+{chr(10).join(include_statements)}
+}};
+
+}} // namespace test
+}} // namespace ripple
+
+#endif
+"""
+        )
+
+    # Format the output file using clang-format
+    logging.info("Formatting master header...")
+    try:
+        subprocess.run(["clang-format", "-i", args.master], check=True)
+        logging.info(f"Successfully generated {relative_master} with {counter} hooks")
+    except subprocess.CalledProcessError:
+        logging.warning(
+            "Warning: clang-format failed, output might not be properly formatted"
+        )
+    except FileNotFoundError:
+        logging.warning("Warning: clang-format not found, output will not be formatted")
+
+
+if __name__ == "__main__":
+    main()

CMakeLists.txt

@@ -129,7 +129,6 @@ else()
   endif()
   find_package(nudb REQUIRED)
   find_package(date REQUIRED)
-  find_package(BLAKE3 REQUIRED)
   include(conan/Protobuf)
   include(conan/gRPC)
     if(TARGET nudb::core)
@@ -163,6 +162,8 @@ else()
   )
 endif()
 
+include(deps/quickjs)
+
 ###
 
 include(RippledCore)

DISABLE_OLD_ENTRIES.md

@@ -1,217 +0,0 @@
-# Auto-Disable Strategy for Hash Migration
-
-## Core Concept
-Instead of trying to fix entries with hardcoded old keys, **automatically disable them** during migration. If it contains old keys, it's broken anyway - make that explicit.
-
-## The Algorithm
-
-### Phase 1: Build Complete Old Key Set
-```cpp
-std::unordered_set<uint256> all_old_keys;
-
-// Collect ALL SHA-512 keys from current state
-stateMap_.visitLeaves([&](SHAMapItem const& item) {
-    all_old_keys.insert(item.key());
-    
-    // Also collect keys from reference fields
-    SerialIter sit(item.slice());
-    auto sle = std::make_shared<SLE>(sit, item.key());
-    
-    // Collect from vector fields
-    if (sle->isFieldPresent(sfIndexes)) {
-        for (auto& key : sle->getFieldV256(sfIndexes)) {
-            all_old_keys.insert(key);
-        }
-    }
-    // ... check all other reference fields
-});
-```
-
-### Phase 2: Scan and Disable
-
-#### Hook Definitions (WASM Code)
-```cpp
-bool scanWASMForKeys(Blob const& wasm_code, std::unordered_set<uint256> const& keys) {
-    // Scan for 32-byte sequences matching known keys
-    for (size_t i = 0; i <= wasm_code.size() - 32; i++) {
-        uint256 potential_key = extract32Bytes(wasm_code, i);
-        if (keys.count(potential_key)) {
-            return true;  // Found hardcoded key!
-        }
-    }
-    return false;
-}
-
-// During migration
-for (auto& hookDef : allHookDefinitions) {
-    if (scanWASMForKeys(hookDef->getFieldBlob(sfCreateCode), all_old_keys)) {
-        hookDef->setFieldU32(sfFlags, hookDef->getFlags() | HOOK_DISABLED_OLD_KEYS);
-        disabled_hooks.push_back(hookDef->key());
-    }
-}
-```
-
-#### Hook State (Arbitrary Data)
-```cpp
-for (auto& hookState : allHookStates) {
-    auto data = hookState->getFieldBlob(sfHookStateData);
-    if (containsAnyKey(data, all_old_keys)) {
-        hookState->setFieldU32(sfFlags, STATE_INVALID_OLD_KEYS);
-        disabled_states.push_back(hookState->key());
-    }
-}
-```
-
-#### Other Vulnerable Entry Types
-```cpp
-void disableEntriesWithOldKeys(SLE& sle) {
-    switch(sle.getType()) {
-        case ltHOOK:
-            if (hasOldKeys(sle)) {
-                sle.setFlag(HOOK_DISABLED_MIGRATION);
-            }
-            break;
-            
-        case ltESCROW:
-            // Check if destination/condition contains old keys
-            if (containsOldKeyReferences(sle)) {
-                sle.setFlag(ESCROW_FROZEN_MIGRATION);
-            }
-            break;
-            
-        case ltPAYCHAN:
-            // Payment channels with old key references
-            if (hasOldKeyInFields(sle)) {
-                sle.setFlag(PAYCHAN_SUSPENDED_MIGRATION);
-            }
-            break;
-            
-        case ltHOOK_STATE:
-            // Already handled above
-            break;
-    }
-}
-```
-
-## Flag Definitions
-
-```cpp
-// New flags for migration-disabled entries
-constexpr uint32_t HOOK_DISABLED_OLD_KEYS     = 0x00100000;
-constexpr uint32_t STATE_INVALID_OLD_KEYS     = 0x00200000;
-constexpr uint32_t ESCROW_FROZEN_MIGRATION    = 0x00400000;
-constexpr uint32_t PAYCHAN_SUSPENDED_MIGRATION = 0x00800000;
-constexpr uint32_t ENTRY_BROKEN_MIGRATION     = 0x01000000;
-```
-
-## Execution Prevention
-
-```cpp
-// In transaction processing
-TER HookExecutor::executeHook(Hook const& hook) {
-    if (hook.isFieldPresent(sfFlags)) {
-        if (hook.getFlags() & HOOK_DISABLED_OLD_KEYS) {
-            return tecHOOK_DISABLED_MIGRATION;
-        }
-    }
-    // Normal execution
-}
-
-TER processEscrow(Escrow const& escrow) {
-    if (escrow.getFlags() & ESCROW_FROZEN_MIGRATION) {
-        return tecESCROW_FROZEN_MIGRATION;
-    }
-    // Normal processing
-}
-```
-
-## Re-enabling Process
-
-### For Hooks
-Developer must submit a new SetHook transaction with updated WASM:
-```cpp
-TER SetHook::doApply() {
-    // If hook was disabled for migration
-    if (oldHook->getFlags() & HOOK_DISABLED_OLD_KEYS) {
-        // Verify new WASM doesn't contain old keys
-        if (scanWASMForKeys(newWasm, all_old_keys)) {
-            return tecSTILL_CONTAINS_OLD_KEYS;
-        }
-        // Clear the disabled flag
-        newHook->clearFlag(HOOK_DISABLED_OLD_KEYS);
-    }
-}
-```
-
-### For Hook State
-Must be cleared and rebuilt:
-```cpp
-TER HookStateModify::doApply() {
-    if (state->getFlags() & STATE_INVALID_OLD_KEYS) {
-        // Can only delete, not modify
-        if (operation != DELETE) {
-            return tecSTATE_REQUIRES_REBUILD;
-        }
-    }
-}
-```
-
-## Migration Report
-
-```json
-{
-    "migration_ledger": 20000000,
-    "entries_scanned": 620000,
-    "entries_disabled": {
-        "hooks": 12,
-        "hook_definitions": 3,
-        "hook_states": 1847,
-        "escrows": 5,
-        "payment_channels": 2
-    },
-    "disabled_by_reason": {
-        "wasm_contains_keys": 3,
-        "state_contains_keys": 1847,
-        "reference_old_keys": 19
-    },
-    "action_required": [
-        {
-            "type": "HOOK_DEFINITION",
-            "key": "0xABCD...",
-            "owner": "rXXX...",
-            "reason": "WASM contains 3 hardcoded SHA-512 keys",
-            "action": "Recompile hook with new keys or remove hardcoding"
-        }
-    ]
-}
-```
-
-## Benefits
-
-1. **Safety**: Broken things explicitly disabled, not silently failing
-2. **Transparency**: Clear record of what was disabled and why
-3. **Natural Cleanup**: Abandoned entries stay disabled forever
-4. **Developer Responsibility**: Owners must actively fix and re-enable
-5. **No Silent Corruption**: Better to disable than corrupt
-6. **Audit Trail**: Complete record of migration casualties
-
-## Implementation Complexity
-
-- **Scanning**: O(n×m) where n=entries, m=data size
-- **Memory**: Need all old keys in memory (~40MB)
-- **False Positives**: Extremely unlikely (2^-256 probability)
-- **Recovery**: Clear path to re-enable fixed entries
-
-## The Nuclear Option
-
-If too many critical entries would be disabled:
-```cpp
-if (disabled_count > ACCEPTABLE_THRESHOLD) {
-    // Abort migration
-    return temMIGRATION_TOO_DESTRUCTIVE;
-}
-```
-
-## Summary
-
-Instead of attempting impossible fixes for hardcoded keys, acknowledge reality: **if it contains old keys, it's broken**. Make that brokenness explicit through disabling, forcing conscious action to repair and re-enable. This turns an impossible problem (fixing hardcoded keys in WASM) into a manageable one (identifying and disabling broken entries).

HASH_MIGRATION_CONTEXT.md

@@ -1,663 +0,0 @@
-# Hash Migration to Blake3 - Work Context
-
-## Build Commands
-- **To build**: `ninja -C build`
-- **To count errors**: `ninja -C build 2>&1 | grep "error:" | wc -l`
-- **To see failed files**: `ninja -C build 2>&1 | grep "^FAILED:" | head -20`
-- **DO NOT USE**: `cmake --build` or `make`
-
-## Test Compilation of Single Files
-
-### Quick Method (basic errors only)
-```bash
-clang++ -std=c++20 -I/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src \
-  -c src/test/app/NFToken_test.cpp -o /tmp/test.o 2>&1 | head -50
-```
-
-### Full Compilation Command (from compile_commands.json)
-Extract the exact compilation command for any file:
-```bash
-# For any specific file:
-jq -r '.[] | select(.file | endswith("NFToken_test.cpp")) | .command' build/compile_commands.json
-
-# Or simplified with just the file path:
-FILE="src/test/app/NFToken_test.cpp"
-jq -r --arg file "$FILE" '.[] | select(.file | endswith($file)) | .command' build/compile_commands.json
-```
-
-### compile_commands.json location
-- **Location**: `build/compile_commands.json`
-- Contains exact compilation commands with all include paths and flags for each source file
-- Generated by CMake during configuration
-
-## Objective
-Modify Xahaud to pass ledger_index through all hash functions to enable switching from SHA512-Half to Blake3 at a specific ledger index.
-
-## Current Approach
-Using a `hash_options` struct containing `ledger_index` that must be passed to all hash functions.
-
-## Structure Added
-```cpp
-struct hash_options {
-    std::uint32_t ledger_index;
-    explicit hash_options(std::uint32_t li) : ledger_index(li) {}
-};
-```
-
-## CRITICAL: Hash Function Classification Required
-
-### The Historical Ledger Problem
-**EVERY** hash operation needs proper classification because even "content hashing" (like transaction IDs, validator manifests, signatures) depends on WHEN it was created:
-- Transaction from ledger 10M (pre-transition) → Must use SHA-512 Half
-- Transaction from ledger 20M (post-transition) → Must use BLAKE3
-- You cannot mix hash algorithms for the same ledger - it's all or nothing
-
-### Classification Constants (to be added to digest.h)
-As an interim step, introduce classification constants to make intent explicit:
-```cpp
-// Special ledger_index values for hash operations
-constexpr uint32_t LEDGER_INDEX_UNKNOWN = 0;  // DANGEROUS - avoid!
-constexpr uint32_t LEDGER_INDEX_TEST_ONLY = std::numeric_limits<uint32_t>::max();
-constexpr uint32_t LEDGER_INDEX_NETWORK_PROTOCOL = std::numeric_limits<uint32_t>::max() - 1;
-constexpr uint32_t LEDGER_INDEX_CURRENT = std::numeric_limits<uint32_t>::max() - 2;  // Use current ledger
-```
-
-### Classification Categories
-
-1. **Ledger Object Indexing** (MUST use actual ledger_index)
-   - All `indexHash()` calls determining WHERE objects live
-   - All keylet functions creating/finding ledger objects
-   - SHAMap node hashing (builds the Merkle tree)
-
-2. **Historical Content Hashing** (MUST use ledger_index from when it was created)
-   - Transaction IDs (use ledger where tx was included)
-   - Validator manifests (use ledger when signed)
-   - Hook code hashing (use ledger when deployed)
-   - Signatures referencing ledger data
-
-3. **Test Code** (use LEDGER_INDEX_TEST_ONLY)
-   - Unit tests
-   - Mock objects
-   - Test fixtures
-
-4. **Network Protocol** (special handling needed)
-   - Peer handshakes
-   - Protocol messages
-   - May need to support both algorithms during transition
-
-### Why hash_options{0} is Dangerous
-Using `hash_options{0}` assumes everything is pre-transition (SHA-512 Half forever), which breaks after the transition point. Every usage must be classified and given the appropriate context.
-
-## SHAMap Hash Architecture Analysis
-
-### Current State
-- SHAMap stores `ledgerSeq_` and knows which ledger it represents
-- Nodes compute hashes WITHOUT ledger context (just use sha512Half directly)
-- Nodes can be SHARED between SHAMaps via canonicalization/caching
-- Node's `hash_` member stores only ONE hash value
-
-### The Fundamental Problem: Node Sharing vs Hash Migration
-
-When nodes are shared between ledgers through canonicalization:
-- A node from ledger 19,999,999 (SHA-512) has hash H1
-- Same node referenced by ledger 20,000,000 (BLAKE3) needs hash H2
-- **Cannot store both hashes without major memory overhead**
-
-### Merkle Tree Constraint
-The Merkle tree structure requires homogeneous hashing:
-```
-Root (BLAKE3)
-├── Child1 (BLAKE3) ✓
-└── Child2 (SHA-512) ✗ IMPOSSIBLE - breaks tree integrity
-```
-You cannot mix hash algorithms within a single tree - all nodes must use the same algorithm.
-
-### Migration Strategies Considered
-
-#### Option 1: Lazy/Gradual Migration ✗
-- Store both SHA-512 and BLAKE3 hashes in each node
-- Problems:
-  - Double memory usage per node
-  - Complex cache invalidation logic  
-  - Still can't mix algorithms in same tree
-  - Node sharing between ledgers becomes impossible
-
-#### Option 2: Big Bang Migration ✓ (Recommended)
-- At transition ledger:
-  - Invalidate ALL cached/stored nodes
-  - Rebuild entire state with new hash algorithm
-  - Maintain separate caches for pre/post transition
-- Benefits:
-  - Clean separation of hash epochs
-  - Easier to reason about
-  - No memory overhead
-  - Maintains tree integrity
-
-### Implementation Requirements for Big Bang
-
-1. **Pass ledger_index through all hash operations:**
-   ```cpp
-   void updateHash(std::uint32_t ledgerSeq);
-   SHAMapHash getHash(std::uint32_t ledgerSeq) const;
-   ```
-
-2. **Separate node caches by hash epoch:**
-   - Pre-transition: SHA-512 node cache
-   - Post-transition: BLAKE3 node cache
-   - Never share nodes between epochs
-
-3. **Critical functions needing updates:**
-   - `SHAMapInnerNode::updateHash()`
-   - `SHAMapInnerNode::updateHashDeep()`
-   - All `SHAMapLeafNode` subclass hash computations
-   - `SHAMap::flushDirty()` and `walkSubTree()`
-
-### Why Big Bang is Preferred
-- **Correctness**: Guarantees Merkle tree integrity
-- **Simplicity**: No complex dual-hash logic
-- **Performance**: No overhead of maintaining multiple hashes
-- **Clear boundaries**: Pre-transition and post-transition are completely separate
-
-The transition point becomes a hard boundary where the entire ledger state is rehashed with the new algorithm.
-
-### Alternative: Heterogeneous Tree - Deep Dive
-
-After deeper analysis, heterogeneous trees are more complex but potentially viable. Here's a comprehensive examination:
-
-#### The Core Insight: Hash Values Are Algorithm-Agnostic
-
-When `SHAMapInnerNode::updateHash()` computes a hash:
-```cpp
-void updateHash(hash_options const& opts) {
-    sha512_half_hasher h(opts);  
-    hash_append(h, HashPrefix::innerNode);
-    iterChildren([&](SHAMapHash const& hh) { hash_append(h, hh); });
-    // Parent hashes the HASH VALUES of children, not the raw data
-}
-```
-
-**Key realization**: Parent nodes hash their children's hash values (256-bit numbers), NOT the children's data. This means a BLAKE3 parent can hash SHA-512 child hashes without issue.
-
-#### How Heterogeneous Trees Would Work
-
-Post-transition rule: **Any NEW hash uses BLAKE3**
-
-```
-Ledger 19,999,999 (SHA-512):
-Root_SHA512
-├── Child1_SHA512
-└── Child2_SHA512
-
-Ledger 20,000,000 (BLAKE3) - Child1 modified:
-Root_BLAKE3 = BLAKE3(Child1_BLAKE3_hash || Child2_SHA512_hash)
-├── Child1_BLAKE3 (NEW hash due to modification)
-└── Child2_SHA512 (unchanged, keeps old hash)
-```
-
-#### The Canonical Structure Ensures Determinism
-
-**Critical insight**: SHAMap trees are **canonical** - the structure is deterministic based on keys:
-- Alice's account always goes in the same tree position
-- Bob's account always goes in the same position
-- Tree shape is fully determined by the set of keys
-
-Therefore:
-- Same modifications → Same tree structure
-- Same tree structure → Same nodes get rehashed
-- Same rehashing → Same final root hash
-- **Consensus is maintained!**
-
-#### The "NEW vs OLD" Detection Problem
-
-The killer issue: How do you know if you're computing a NEW hash vs verifying an OLD one?
-
-```cpp
-void updateHash(hash_options const& opts) {
-    // Am I computing a NEW hash (use BLAKE3)?
-    // Or verifying an OLD hash (could be SHA-512)?
-    // This function doesn't know WHY it was called!
-}
-```
-
-Without explicit context about NEW vs OLD:
-- Loading from DB: Don't know if it's SHA-512 or BLAKE3
-- Modifying a node: Should use BLAKE3
-- Verifying from network: Could be either
-
-Potential solutions:
-1. **Try-both approach**: Verify with BLAKE3, fallback to SHA-512
-2. **Version tracking**: Store algorithm version with each node
-3. **Context passing**: Thread NEW/OLD context through all calls
-
-#### Canonical Nodes (cowid=0) - A Complication
-
-Canonical nodes are immutable and shared, BUT:
-- They ARE verified when loaded from DB or network
-- The verification needs to compute the hash to check integrity
-- This means we need to know WHICH algorithm was used
-- Can't just trust the hash - must verify data matches
-
-This actually makes heterogeneous trees HARDER because:
-```cpp
-// When loading a canonical node from DB:
-auto node = SHAMapTreeNode::makeFromPrefix(data, hash);
-// Need to verify: does hash(data) == provided_hash?
-// But which hash function? SHA-512 or BLAKE3?
-// Must try both, adding complexity and ambiguity
-```
-
-#### System-Wide Implications
-
-##### Database Layer
-- Heterogeneous: Same data might have 2 entries (SHA-512 and BLAKE3 versions)
-- Big Bang: Clean cutover, old entries become invalid
-
-##### Network Sync
-- Heterogeneous: Ambiguous - "I need node 0xABC..." (which algorithm?)
-- Big Bang: Clear - algorithm determined by ledger context
-
-##### Consensus
-- Heterogeneous: Works IF all validators make same NEW/OLD decisions
-- Big Bang: Simple - everyone uses same algorithm
-
-##### External Proof Verification
-- Heterogeneous: Complex - mixed algorithms in Merkle paths
-- Big Bang: Simple - "before ledger X use SHA-512, after use BLAKE3"
-
-##### Performance
-- Heterogeneous: Double verification attempts for old nodes
-- Big Bang: One-time rehash cost
-
-#### Gradual Migration Pattern
-
-With heterogeneous trees, migration happens naturally:
-```
-Ledger 20,000,000: 5% BLAKE3, 95% SHA512
-Ledger 20,001,000: 30% BLAKE3, 70% SHA512  
-Ledger 20,010,000: 60% BLAKE3, 40% SHA512
-Ledger 20,100,000: 90% BLAKE3, 10% SHA512
-Eventually: ~100% BLAKE3 (dormant nodes may remain SHA-512)
-```
-
-#### Remaining Challenges
-
-1. **Context Plumbing**: Need to distinguish NEW vs OLD operations everywhere
-2. **Verification Ambiguity**: Failures could be corruption OR wrong algorithm
-3. **Testing Complexity**: Many more edge cases to test
-4. **Protocol Complexity**: Merkle proofs need algorithm information
-5. **Developer Cognitive Load**: Harder to reason about
-
-### Conclusion: Trade-offs
-
-**Heterogeneous Trees**:
-- ✅ No "big bang" transition
-- ✅ Natural, incremental migration
-- ✅ Maintains consensus (with careful implementation)
-- ❌ Permanent complexity throughout codebase
-- ❌ Ambiguous verification
-- ❌ Complex testing
-
-**Big Bang Migration**:
-- ✅ Clean, simple mental model
-- ✅ Clear algorithm boundaries
-- ✅ Easier testing and debugging
-- ❌ One-time massive performance hit
-- ❌ Requires careful coordination
-- ❌ Can't easily roll back
-
-The heterogeneous approach is **theoretically viable** but adds significant permanent complexity. Big Bang is simpler but has a painful transition. The choice depends on whether you prefer one-time pain (Big Bang) or permanent complexity (heterogeneous).
-
-## Files Modified So Far
-1. `src/ripple/protocol/digest.h` - Added hash_options struct and modified sha512Half signatures
-2. `src/ripple/protocol/impl/Indexes.cpp` - Updated indexHash and all keylet functions to accept hash_options
-3. `src/ripple/protocol/Indexes.h` - Updated all function declarations to include hash_options
-
-## Current Status
-- Core hash functions modified with backward-compatible overloads
-- All keylet functions updated to require hash_options
-- Propagating hash_options through codebase - MASSIVE undertaking
-- 91+ compilation errors remaining after fixing ~20 files
-- Every fix exposes more errors as headers propagate changes
-
-## SHAMap Node Factory Methods - Missing Ledger Context
-
-### The Problem with makeFromWire and makeFromPrefix
-These factory methods create SHAMap nodes from serialized data but **don't have ledger context**:
-
-```cpp
-// Called when receiving nodes from peers over network
-SHAMapTreeNode::makeFromWire(Slice rawNode)  
-
-// Called when loading nodes from database
-SHAMapTreeNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
-```
-
-These methods:
-- Parse serialized node data (from network or database)
-- Create `SHAMapInnerNode` or leaf nodes
-- Need to call `updateHash()` if hash isn't provided
-- **BUT** don't know which ledger they're building for!
-
-### Why This Is Critical
-When a node is loaded from database or received from network:
-1. The serialized data doesn't include ledger_index
-2. The node might be shared across multiple ledgers (pre-transition)
-3. After transition, we need to know if this node uses SHA-512 or BLAKE3
-4. Currently using `LEDGER_INDEX_UNKNOWN` as placeholder
-
-### Implications for P2P Protocol
-The network protocol would need changes:
-- `TMGetLedger` messages would need to specify ledger_index
-- Node responses would need hash algorithm version
-- Database storage would need to track which hash was used
-
-This reinforces why **Big Bang migration** is simpler - no protocol changes needed!
-
-## Next Steps
-1. Add hash_options{0} to all keylet call sites (using 0 as placeholder ledger_index)
-2. Get the code compiling first
-3. Later: Thread actual ledger_index values through from Views/transactions
-4. Eventually: Add Blake3 switching logic based on ledger_index threshold
-
-## Key Insight
-Every place that creates a ledger key needs to know what ledger it's operating on. This is a massive change touching:
-- All View classes
-- All transaction processors
-- All RPC handlers  
-- All tests
-- Consensus code
-
-## Compilation Strategy
-NEVER use hash_options{0} - always use proper classification from the HashContext enum in digest.h!
-
-## Quick Reference for Fixing Test Files
-
-### Essential Files to Reference
-1. **@src/ripple/protocol/digest.h** - Lines 37-107 contain the HashContext enum with ALL valid classifiers
-2. **@src/ripple/protocol/Indexes.h** - Shows all keylet function signatures that need hash_options
-
-### Keylet Function Mapping to HashContext
-When you see a keylet function, use the corresponding KEYLET_* enum value:
-```cpp
-keylet::account()        → KEYLET_ACCOUNT
-keylet::amendments()     → KEYLET_AMENDMENTS  
-keylet::book()          → KEYLET_BOOK
-keylet::check()         → KEYLET_CHECK
-keylet::child()         → KEYLET_CHILD
-keylet::depositPreauth() → KEYLET_DEPOSIT_PREAUTH
-keylet::dirPage()       → KEYLET_DIR_PAGE
-keylet::emittedDir()    → KEYLET_EMITTED_DIR
-keylet::emittedTxn()    → KEYLET_EMITTED_TXN
-keylet::escrow()        → KEYLET_ESCROW
-keylet::fees()          → KEYLET_FEES
-keylet::hook()          → KEYLET_HOOK
-keylet::hookDefinition() → KEYLET_HOOK_DEFINITION
-keylet::hookState()     → KEYLET_HOOK_STATE
-keylet::hookStateDir()  → KEYLET_HOOK_STATE_DIR
-keylet::importVLSeq()   → KEYLET_IMPORT_VLSEQ
-keylet::negativeUNL()   → KEYLET_NEGATIVE_UNL
-keylet::nftBuys()       → KEYLET_NFT_BUYS
-keylet::nftOffer()      → KEYLET_NFT_OFFER
-keylet::nftPage()       → KEYLET_NFT_PAGE
-keylet::nftSells()      → KEYLET_NFT_SELLS
-keylet::offer()         → KEYLET_OFFER
-keylet::ownerDir()      → KEYLET_OWNER_DIR
-keylet::payChan()       → KEYLET_PAYCHAN
-keylet::signers()       → KEYLET_SIGNERS
-keylet::skip()          → KEYLET_SKIP_LIST
-keylet::ticket()        → KEYLET_TICKET
-keylet::trustline()     → KEYLET_TRUSTLINE
-keylet::unchecked()     → KEYLET_UNCHECKED
-keylet::UNLReport()     → KEYLET_UNL_REPORT
-keylet::uriToken()      → KEYLET_URI_TOKEN
-```
-
-### Non-Keylet Hash Classifications
-```cpp
-sha512Half() for validator data     → VALIDATOR_LIST_HASH
-sha512Half() for hook code          → HOOK_DEFINITION or LEDGER_INDEX_UNNEEDED
-sha512Half() for signatures         → CRYPTO_SIGNATURE_HASH
-sha512Half() for network protocol   → NETWORK_HANDSHAKE_HASH
-sha512Half_s() for secure hashing   → Same rules apply
-```
-
-## Key Insights
-
-### The Scale Problem
-Every place that creates a ledger key needs to know what ledger it's operating on:
-- All View classes
-- All transaction processors (50+ files)
-- All RPC handlers (30+ files)  
-- All tests (hundreds of files)
-- Consensus code
-- Path finding code
-- Payment processing pipelines
-
-### Why This Is Hard
-1. **Cascading changes**: Fixing one header file exposes dozens of new errors
-2. **Deep call chains**: Ledger index must be threaded through multiple layers
-3. **Protocol implications**: Network messages would need to include ledger sequences
-4. **No gradual migration**: Can't mix hash algorithms - it's all or nothing
-5. **Testing nightmare**: Every test that creates mock ledger objects needs updating
-
-### The Fundamental Challenge
-The hash function is so deeply embedded in the architecture that changing it is like replacing the foundation of a building while people are living in it. This is why most blockchains never change their hash functions after launch.
-
-## Key Learnings - Where to Get ledger_index
-
-### 1. ReadView/ApplyView Classes
-- `view.seq()` returns the LedgerIndex (found in ReadView.h:193)
-- ReadView has `info()` method that returns LedgerInfo struct
-- LedgerInfo contains `seq` field which is the ledger sequence number
-- Cast to uint32_t: `static_cast<std::uint32_t>(view.seq())`
-
-### 2. Common Patterns Found
-- Functions that take `ReadView const& view` can access `view.seq()`
-- Functions that take `ApplyView& view` can also access `view.seq()` (ApplyView inherits from ReadView)
-
-### 3. Files That Need Updates
-- View.h - DONE - all 6 keylet calls updated to use view.seq()
-- Any file with keylet:: namespace calls
-- Transaction processors that create/lookup ledger objects
-
-### 4. Progress Log
-
-#### Stats (Updated: 2025-09-09)
-- Total files with keylet:: calls: 126
-- Files fixed so far: **100+** (including all core files, transaction processors, RPC handlers, and most tests)
-- Started with 7 errors, peaked at 105+ as fixes propagate through headers  
-- **Currently down to 113 errors across 11 test files** (from hundreds of files!)
-- ALL keylet function signatures updated to require hash_options
-- Pattern emerging: Every transaction processor, every RPC handler, every test needs updating
-
-#### Major Milestone Achieved
-- Successfully fixed ALL non-test source files
-- Fixed majority of test files using parallel agents
-- Demonstrated that the hash migration IS possible despite being a massive undertaking
-
-#### Major Files Fixed
-- All core ledger files (View.cpp, ApplyView.cpp, ReadView.cpp, etc.)
-- Most transaction processors (Payment, Escrow, CreateOffer, NFToken*, etc.)
-- Hook implementation files (applyHook.cpp, SetHook.cpp)
-- Infrastructure files (Transactor.cpp, BookDirs.cpp, Directory.cpp)
-
-#### Key Milestone
-- Updated ALL keylet function signatures in Indexes.h/cpp to require hash_options
-- Even functions that take pre-existing uint256 keys now require hash_options for consistency
-- This causes massive cascading compilation errors but ensures consistency
-
-#### Files Fixed So Far
-1. **View.h** - Fixed using `view.seq()`
-2. **Ledger.cpp** - Fixed using `info_.seq` in constructor, `seq()` in methods
-3. **LocalTxs.cpp** - Fixed using `view.seq()`
-4. **NegativeUNLVote.cpp** - Fixed using `prevLedger->seq()`
-5. **TxQ.cpp** - Fixed 5 calls using `view.seq()`
-6. **SkipListAcquire.cpp** - Fixed (with protocol issue on line 90)
-7. **LedgerReplayMsgHandler.cpp** - Fixed using `info.seq`
-8. **RCLValidations.cpp** - Fixed using `ledger->seq()`
-9. **Credit.cpp** - Fixed using `view.seq()`
-10. **StepChecks.h** - Fixed using `view.seq()`
-11. **BookTip.cpp** - Fixed using `view.seq()`
-12. **XRPEndpointStep.cpp** - Fixed using `ctx.view.seq()`
-13. **DirectStep.cpp** - Fixed using `sb.seq()` and `ctx.view.seq()`
-14. **BookStep.cpp** - Fixed using `afView.seq()` and `view.seq()`
-15. **CancelCheck.cpp** - Fixed using `ctx.view.seq()` and `view().seq()`
-16. **Pathfinder.cpp** - Fixed using `mLedger->seq()`
-17. More coming...
-
-#### Common Patterns Found
-- **ReadView/ApplyView**: Use `.seq()`
-- **Ledger pointer**: Use `->seq()`
-- **Transactor classes**: Use `view().seq()`
-- **PaymentSandbox**: Use `sb.seq()`
-- **StrandContext**: Use `ctx.view.seq()`
-- **LedgerInfo struct**: Use `.seq` field directly
-
-### 5. Architectural Questions
-
-#### Keylets with pre-existing keys
-- Functions like `keylet::check(uint256 const& key)` just wrap an existing key
-- They don't compute a new hash, just interpret the key as a specific ledger type
-- **Question**: Do these really need hash_options? 
-- **Current approach**: Include hash_options for consistency, might store ledger_index in Keylet for future use
-- **Note**: This could be revisited - might be unnecessary overhead for simple key wrapping
-
-### 6. Known Issues
-- SkipListAcquire.cpp line 90: Requesting skip list by hash without knowing ledger seq yet
-  - Can't know the sequence until we GET the ledger
-  - Using hash_options{0} as placeholder 
-  - Would need to refactor to fetch ledger first, THEN request skip list with proper seq
-  - Or protocol change to handle "skip list for whatever ledger this hash is"
-
-### 7. Network Protocol Has Ledger Sequence!
-
-**Critical Discovery**: The protobuf definitions show that network messages DO carry ledger sequence in many places:
-
-#### TMLedgerData (what InboundLedger::processData receives):
-```protobuf
-message TMLedgerData {
-    required bytes ledgerHash       = 1;
-    required uint32 ledgerSeq       = 2;  // <-- HAS THE SEQUENCE!
-    required TMLedgerInfoType type  = 3;
-    repeated TMLedgerNode nodes     = 4;
-}
-```
-
-#### TMGetLedger (the request):
-```protobuf
-message TMGetLedger {
-    optional uint32 ledgerSeq       = 4;  // Can request by sequence
-}
-```
-
-#### TMIndexedObject (per-object context):
-```protobuf
-message TMIndexedObject {
-    optional uint32 ledgerSeq   = 5;  // Per-object sequence!
-}
-```
-
-**Implications**:
-- The protocol already has infrastructure for ledger context
-- `InboundLedger` can use `packet.ledgerseq()` from TMLedgerData
-- Network sync might be solvable WITHOUT major protocol changes
-- The ledger sequence just needs to be threaded through to hash functions
-
-**Key Flow**:
-```cpp
-InboundLedger::processData(protocol::TMLedgerData& packet)
-    packet.ledgerseq()  // <-- Extract sequence from protocol message
-    <- SHAMap::addKnownNode(..., ledgerSeq)
-      <- SHAMapTreeNode::makeFromWire(data, ledgerSeq)
-        <- updateHash(hash_options{ledgerSeq})
-```
-
-This solves a major piece of the puzzle - the network layer CAN provide context for hash verification!
-
-### 8. Test File Patterns
-
-**Key Discovery**: Tests should use actual ledger sequences, NOT placeholders!
-
-#### Getting Ledger Sequence in Tests
-Tests typically use `test::jtx::Env` which provides access to ledger context:
-- `env.current()` - Returns a ReadView pointer
-- `env.current()->seq()` - Gets the current ledger sequence (already uint32_t)
-
-#### Common Test Patterns
-
-##### Pattern 1: Direct keylet calls
-```cpp
-// OLD
-env.le(keylet::line(alice, bob, currency));
-
-// NEW
-env.le(keylet::line(hash_options{env.current()->seq(), KEYLET_TRUSTLINE}, alice, bob, currency));
-```
-
-##### Pattern 2: Helper functions need env parameter
-```cpp
-// OLD
-static uint256 getCheckIndex(AccountID const& account, uint32_t seq) {
-    return keylet::check(account, seq).key;
-}
-// Called as: getCheckIndex(alice, seq)
-
-// NEW
-static uint256 getCheckIndex(test::jtx::Env& env, AccountID const& account, uint32_t seq) {
-    return keylet::check(hash_options{env.current()->seq(), KEYLET_CHECK}, account, seq).key;
-}
-// Called as: getCheckIndex(env, alice, seq)
-```
-
-##### Pattern 3: Fee calculations
-```cpp
-// Uses env.current() to get fee information
-XRPAmount const baseFeeDrops{env.current()->fees().base};
-```
-
-#### Test Files Status
-- **Total test files needing fixes**: ~12-15
-- **Pattern**: All test files that create or look up ledger objects need updating
-- **Common test files**:
-  - Check_test.cpp - PARTIALLY FIXED
-  - AccountDelete_test.cpp
-  - Escrow_test.cpp  
-  - NFToken_test.cpp
-  - Flow_test.cpp
-  - Import_test.cpp
-  - etc.
-
-#### Why NOT to use placeholders in tests
-- Tests verify actual ledger behavior
-- Using `hash_options{0}` would test wrong behavior after transition
-- Tests need to work both pre and post hash migration
-- `env.current()->seq()` gives the actual test ledger sequence
-
-### 6. Ways to Get Ledger Sequence
-
-#### From Application (app_):
-- `app_.getLedgerMaster()` gives you LedgerMaster
-
-#### From LedgerMaster:
-- `getLedgerByHash(hash)` - Get ledger by hash, then call `->seq()` on it
-- `getLedgerBySeq(index)` - Get ledger by sequence directly
-- `getCurrentLedger()` - Current open ledger, call `->seq()`
-- `getClosedLedger()` - Last closed ledger, call `->seq()`
-- `getValidatedLedger()` - Last validated ledger, call `->seq()`
-- `getPublishedLedger()` - Last published ledger, call `->seq()`
-- `getCurrentLedgerIndex()` - Direct sequence number
-- `getValidLedgerIndex()` - Direct sequence number
-- `walkHashBySeq()` - Walk ledger chain to find hash by sequence
-
-#### From Ledger object:
-- `ledger->seq()` - Direct method
-- `ledger->info().seq` - Through LedgerInfo
-
-#### From ReadView/OpenView/ApplyView:
-- `view.seq()` - All views have this method
-- `view.info().seq` - Through LedgerInfo
-
-#### Special Case - SkipListAcquire:
-- Line 67: `app_.getLedgerMaster().getLedgerByHash(hash_)` 
-- If ledger exists locally, we get it and can use its seq()
-- If not, we're requesting it from peers - don't know seq yet!

LAST_TESTAMENT.md

@@ -1,108 +0,0 @@
-# Last Testament: SHA-512 to BLAKE3 Migration Deep Dive
-
-## The Journey
-Started with "just change the hash function" - ended up discovering why hash functions are permanent blockchain decisions.
-
-## Key Discoveries
-
-### 1. Ledger Implementation Added
-- `Ledger::shouldMigrateToBlake3()` - checks migration window/flags
-- `Ledger::migrateToBlake3()` - placeholder for actual migration
-- Hook in `BuildLedger.cpp` after transaction processing
-- Migration happens OUTSIDE transaction pipeline to avoid metadata explosion
-
-### 2. The Rekeying Nightmare (REKEYING_ISSUES.md)
-Every object key changes SHA512→BLAKE3, but keys are EVERYWHERE:
-- **DirectoryNode.sfIndexes** - vectors of keys
-- **sfPreviousTxnID, sfIndexNext, sfIndexPrevious** - direct key refs  
-- **Order books** - sorted by key value (order changes!)
-- **Hook state** - arbitrary blobs with embedded keys
-- **620k objects** with millions of interconnected references
-
-### 3. The LUT Approach
-```cpp
-// Build lookup table: old_key → new_key (40MB for 620k entries)
-// O(n) to build, O(n×m) to update all fields
-// Must check EVERY uint256 field in EVERY object
-```
-**Problems:**
-- LUT check on EVERY lookup forever (performance tax)
-- Can't know if key is old/new without checking
-- Might need bidirectional LUT (80MB)
-- Can NEVER remove it (WASM has hardcoded keys!)
-
-### 4. The WASM Hook Bomb
-Hook code can hardcode keys in compiled WASM:
-- Can't modify without changing hook hash
-- Changing hash breaks all references to hook
-- Literally unfixable without breaking hooks
-
-### 5. MapStats Enhancement
-Added ledger entry type tracking:
-- Count, total bytes, avg size per type
-- Uses `LedgerFormats::getInstance().findByType()` for names
-- Shows 124k HookState entries (potential key references!)
-
-### 6. Current Ledger Stats
-- 620k total objects, 98MB total
-- 117k DirectoryNodes (full of references)
-- 124k HookState entries (arbitrary data)
-- 80 HookDefinitions (WASM code)
-- Millions of internal key references
-
-## Why It's Impossible
-
-### The Lookup Problem
-```cpp
-auto key = keylet::account(Alice).key;
-// Which hash function? SHA512 or BLAKE3?
-// NO WAY TO KNOW without timestamp/LUT/double-lookup
-```
-
-### The Fundamental Issues
-1. **No key timestamp** - Can't tell if key is pre/post migration
-2. **Embedded references everywhere** - sfIndexes, hook state, WASM code
-3. **Permanent LUT required** - Check on every operation forever
-4. **Performance death** - 2x lookups or LUT check on everything
-5. **Dual-key SHAMap impossible** - Breaks ordering/structure
-
-### The "Solutions" That Don't Work
-- **Lazy migration**: LUT forever, complexity forever
-- **Big bang**: Still need permanent LUT for old references  
-- **Heterogeneous tree**: Can't mix hash functions in Merkle tree
-- **Binary search-replace**: Could corrupt data, no validation
-- **Import to v2 chain**: Same reference update problems
-
-## The Verdict
-
-After threading ledger_index through 1000+ functions for hash context, the migration faces insurmountable challenges:
-
-1. **WASM hooks** contain unfixable hardcoded keys
-2. **Every lookup** needs LUT check forever (performance tax)
-3. **Can't determine key age** from identifier alone
-4. **Millions of references** need perfect updating
-5. **One mistake** = corrupted ledger, lost funds
-
-**Conclusion**: This is a "v2 blockchain" problem, not a migration problem. SHA-512 is forever.
-
-## Lessons Learned
-
-- Hash functions are THE fundamental addressing system
-- Deeper than consensus, deeper than data structures
-- Once chosen, essentially permanent
-- The attempt revealed the true complexity of blockchain internals
-- Even 98MB of data has millions of interconnected references
-- WASM hooks make any migration effectively impossible
-
-**Final Status**: Technically possible with permanent LUT and massive complexity. Practically impossible due to hooks, performance, and risk. SHA-512 until heat death of universe.
-
-## Code Artifacts
-
-- Hash classification system in `digest.h`
-- `hash_options` threaded through Indexes/keylets
-- Ledger migration methods (placeholder)
-- MapStats with entry type breakdown
-- REKEYING_ISSUES.md with full analysis
-- 100+ files modified to thread ledger context
-
-The migration died not from lack of effort, but from the discovery that some architectural decisions are truly permanent.

LEDGER_INDEX_PREVIOUS.md

@@ -1,71 +0,0 @@
-Yes! You’re onto something quite elegant there. Let me break it down:
-
----
-
-### 🔑 LedgerEntry “PriorKey” field
-
-If each SLE carried an **optional `LedgerIndexPriorSha512`** (or whatever name), then:
-
-* When you rekey at cutover, you don’t lose continuity.
-* Anyone walking the ledger can trivially build the old→new LUT on the fly.
-* For “day-to-day” operations, the node only needs the canonical keys.
-
-This basically lets you say:
-
-* **Canonical view** = new Blake3 keys.
-* **Continuity view** = traverse new keys, but if you need to resolve old references, follow the `LedgerIndexPriorSha512` link.
-
----
-
-### 📊 For analysis and data science
-
-You can now reconstruct historical continuity without keeping a giant external LUT file. Just walk the ledger and collect `(old, new)` pairs from the entries.
-
-This means explorers, forensic tools, compliance, etc. can:
-
-* Map old keys to new at any point.
-* Handle partially migrated networks (some ledgers pre-cutover, some post).
-
----
-
-### 🟢 Operational benefits
-
-* Validators don’t need to bloat memory with a global LUT — it’s **distributed inside the ledger itself**.
-* New ledgers “just work” for CRUD.
-* Old references (hooks, weird integrations) can be handled until people migrate.
-
----
-
-### ⚠️ Caveats
-
-1. **Storage overhead**: each SLE gets \~32 bytes extra. But given only \~600k objects in the state tree, that’s trivial compared to the win.
-2. **Hardcoded keys in hooks/wasm**: still a landmine — if someone has *literally baked in* the old canonical key, they’ll need the LUT or breakage handling at ApplyView.
-3. **Consensus rules**: adding this field changes serialization — so it’s an amendment, not just an operator convenience.
-
----
-
-### 🔄 Proposal workflow
-
-1. Cutover ledger → during bookkeeping, every object is rekeyed into the new canonical tree.
-2. At the same time, each new object gets `LedgerIndexPriorSha512 = old_key`.
-3. From then on, **every ledger post-cutover contains the LUT implicitly**.
-4. After some deprecation period, once nobody references `LedgerIndexPriorSha512`, you can prune support.
-
----
-
-So in practice:
-
-* **Validators** only care about the canonical map.
-* **Explorers, auditors, backward-compat systems** can lazily reconstruct the LUT.
-* You don’t poison `succ()` or iteration with mixed domains.
-
----
-
-This actually solves both your goals:
-
-* Day-to-day simplicity (Blake-only canonical).
-* Historical continuity (walk the field if you need).
-
----
-
-Want me to sketch how `ReadView::read(Keylet)` could transparently try `LUT` lookups only if the canonical miss happens, using that new field? That would make it zero-effort for old call sites.

REKEYING_ISSUES.md

@@ -1,332 +0,0 @@
-# Hash Migration Rekeying Issues
-
-## The Fundamental Problem
-
-When migrating from SHA-512 Half to BLAKE3, we're not just changing a hash function - we're changing the **keys** that identify every object in the ledger's state map. Since the hash IS the key in the SHAMap, every object needs a new address.
-
-## What Needs to be Rekeyed
-
-### 1. Primary Ledger Objects
-Every SLE (STLedgerEntry) in the state map has its key computed from:
-- Its type (Account, Offer, RippleState, etc.)
-- Its identifying data (AccountID, offer sequence, etc.)
-
-When we change the hash function, EVERY object gets a new key.
-
-### 2. Directory Structures
-Directories are ledger objects that contain **lists of other objects' keys**:
-
-#### Owner Directories (`keylet::ownerDir`)
-- Contains keys of all objects owned by an account
-- Every offer, escrow, check, payment channel, etc. key stored here
-- When those objects are rekeyed, ALL these references must be updated
-
-#### Order Book Directories (`keylet::book`, `keylet::quality`)
-- Contains keys of offers at specific quality levels
-- All offer keys must be updated to their new BLAKE3 values
-
-#### NFT Directories (`keylet::nft_buys`, `keylet::nft_sells`)
-- Contains keys of NFT offers
-- All NFT offer keys must be updated
-
-#### Hook State Directories (`keylet::hookStateDir`)
-- Contains keys of hook state entries
-- All hook state keys must be updated
-
-### 3. Cross-References Between Objects
-Many objects contain direct references to other objects:
-
-#### Account Objects
-- `sfNFTokenPage` - References to NFT page keys
-- Previous/Next links in directory pages
-
-#### Directory Pages
-- `sfIndexes` - Vector of object keys
-- `sfPreviousTxnID` - Transaction hash references
-- `sfIndexPrevious`/`sfIndexNext` - Links to other directory pages
-
-#### NFT Pages
-- References to previous/next pages in the chain
-
-## The Cascade Effect
-
-Re-keying isn't a simple one-pass operation:
-
-```
-1. Rekey Account A (SHA512 → BLAKE3)
-   ↓
-2. Update Account A's owner directory
-   ↓
-3. Rekey the owner directory itself
-   ↓
-4. Update all objects IN the directory with new keys
-   ↓
-5. Update any directories THOSE objects appear in
-   ↓
-6. Continue cascading...
-```
-
-## Implementation Challenges
-
-### Challenge 1: Directory Entry Updates
-```cpp
-// Current directory structure
-STVector256 indexes = directory->getFieldV256(sfIndexes);
-// Contains: [sha512_key1, sha512_key2, sha512_key3, ...]
-
-// After migration needs to be:
-// [blake3_key1, blake3_key2, blake3_key3, ...]
-```
-
-### Challenge 2: Finding All References
-There's no reverse index - given an object's key, you can't easily find all directories that reference it. You'd need to:
-1. Walk the entire state map
-2. Check every directory's `sfIndexes` field
-3. Update any matching keys
-
-### Challenge 3: Maintaining Consistency
-During migration, you need to ensure:
-- No orphaned references (keys pointing to non-existent objects)
-- No duplicate entries
-- Proper ordering in sorted structures (offer books)
-
-### Challenge 4: Page Links
-Directory pages link to each other:
-```cpp
-uint256 prevPage = dir->getFieldU256(sfIndexPrevious);
-uint256 nextPage = dir->getFieldU256(sfIndexNext);
-```
-These links are also keys that need updating!
-
-## Why This Makes Migration Complex
-
-### Option A: Big Bang Migration
-- Must update EVERYTHING atomically
-- Need to track old→new key mappings for entire ledger
-- Memory requirements: ~2x the state size for mapping table
-- Risk: Any missed reference breaks the ledger
-
-### Option B: Heterogeneous Tree
-- Old nodes keep SHA-512 keys
-- New/modified nodes use BLAKE3
-- Problem: How do you know which hash to use for lookups?
-- Problem: Directory contains mix of old and new keys?
-
-### Option C: Double Storage
-- Store objects under BOTH keys temporarily
-- Gradually migrate references
-- Problem: Massive storage overhead
-- Problem: Synchronization between copies
-
-## Example: Rekeying an Offer
-
-Consider rekeying a single offer:
-
-1. **The Offer Itself**
-   - Old key: `sha512Half(OFFER, account, sequence)`
-   - New key: `blake3(OFFER, account, sequence)`
-
-2. **Owner Directory**
-   - Must update `sfIndexes` to replace old offer key with new
-
-3. **Order Book Directory**
-   - Must update `sfIndexes` in the quality directory
-   - May need to update multiple quality levels if offer moved
-
-4. **Account Object**
-   - Update offer count/reserve tracking if needed
-
-5. **The Directories Themselves**
-   - Owner directory key: `sha512Half(OWNER_DIR, account, page)`
-   - New key: `blake3(OWNER_DIR, account, page)`
-   - Order book key: `sha512Half(BOOK_DIR, ...)`
-   - New key: `blake3(BOOK_DIR, ...)`
-
-## Potential Solutions
-
-### 1. Migration Ledger Object
-Create a temporary "migration map" ledger object:
-```cpp
-sfOldKey → sfNewKey mappings
-```
-But this could be gigabytes for millions of objects.
-
-### 2. Deterministic Rekeying
-Since we can determine an object's type from its `LedgerEntryType`, we could:
-1. Load each SLE
-2. Determine its type
-3. Recompute its key with BLAKE3
-4. Track the mapping
-
-But we still need to update all references.
-
-### 3. Lazy Migration
-Only rekey objects when they're modified:
-- Pro: Spreads migration over time
-- Con: Permanent complexity in codebase
-- Con: Must support both hash types forever
-
-### 4. New State Structure
-Instead of migrating in-place, build an entirely new state map:
-1. Create new empty BLAKE3 SHAMap
-2. Walk old map, inserting with new keys
-3. Update all references during copy
-4. Atomically swap maps
-
-This is essentially what BUILD_LEDGER.md suggests, but the reference updating remains complex.
-
-## The Lookup Table Approach
-
-After further analysis, a lookup table (LUT) based approach might actually be feasible:
-
-### Algorithm Overview
-
-#### Phase 1: Build the LUT (O(n))
-```cpp
-std::unordered_map<uint256, uint256> old_to_new;
-
-stateMap_.visitLeaves([&](SHAMapItem const& item) {
-    SerialIter sit(item.slice());
-    auto sle = std::make_shared<SLE>(sit, item.key());
-    
-    // Determine type from the SLE
-    LedgerEntryType type = sle->getType();
-    
-    // Recompute key with BLAKE3 based on type
-    uint256 newKey = computeBlake3Key(sle, type);
-    old_to_new[item.key()] = newKey;
-});
-// Results in ~620k entries in the LUT
-```
-
-#### Phase 2: Update ALL uint256 Fields (O(n × m))
-Walk every object and check every uint256 field against the LUT:
-
-```cpp
-stateMap_.visitLeaves([&](SHAMapItem& item) {
-    SerialIter sit(item.slice());
-    auto sle = std::make_shared<SLE>(sit, item.key());
-    bool modified = false;
-    
-    // Check every possible uint256 field
-    modified |= updateField(sle, sfPreviousTxnID, old_to_new);
-    modified |= updateField(sle, sfIndexPrevious, old_to_new);
-    modified |= updateField(sle, sfIndexNext, old_to_new);
-    modified |= updateField(sle, sfBookNode, old_to_new);
-    
-    // Vector fields
-    modified |= updateVector(sle, sfIndexes, old_to_new);
-    modified |= updateVector(sle, sfHashes, old_to_new);
-    modified |= updateVector(sle, sfAmendments, old_to_new);
-    modified |= updateVector(sle, sfNFTokenOffers, old_to_new);
-    
-    if (modified) {
-        // Re-serialize with updated references
-        Serializer s;
-        sle->add(s);
-        // Create new item with new key
-        item = make_shamapitem(old_to_new[item.key()], s.slice());
-    }
-});
-```
-
-### Complexity Analysis
-- **Phase 1**: O(n) where n = number of objects (~620k)
-- **Phase 2**: O(n × m) where m = average fields per object
-- **Hash lookups**: O(1) average case
-- **Total**: Linear in the number of objects!
-
-### Memory Requirements
-- LUT size: 620k entries × (32 bytes + 32 bytes) = ~40 MB
-- Reasonable to keep in memory during migration
-
-### Implementation Challenges
-
-#### 1. Comprehensive Field Coverage
-Must check EVERY field that could contain a key:
-```cpp
-// Singleton uint256 fields
-sfPreviousTxnID, sfIndexPrevious, sfIndexNext, sfBookNode,
-sfNFTokenID, sfEmitParentTxnID, sfHookOn, sfHookStateKey...
-
-// Vector256 fields  
-sfIndexes, sfHashes, sfAmendments, sfNFTokenOffers,
-sfHookNamespaces, sfURITokenIDs...
-
-// Nested structures
-STArray fields containing STObjects with uint256 fields
-```
-
-#### 2. False Positive Risk
-Any uint256 that happens to match a key would be updated:
-- Could corrupt data if a non-key field matches
-- Mitigation: Only update known reference fields
-- Risk: Missing custom fields added by hooks
-
-#### 3. Order Book Sorting
-Order books are sorted by key value. After rekeying:
-- Sort order changes completely
-- Need to rebuild book directories
-- Quality levels might shift
-
-### Alternative: Persistent Migration Map
-
-Instead of one-time migration, store the mapping permanently:
-
-```cpp
-// Special ledger entries (one per ~1000 mappings)
-MigrationMap_0000: {
-    sfOldKeys: [old_hash_0, old_hash_1, ...],
-    sfNewKeys: [new_hash_0, new_hash_1, ...]
-}
-MigrationMap_0001: { ... }
-// ~620 of these objects
-```
-
-Pros:
-- Can verify historical references
-- Debugging is easier
-- Can be pruned later if needed
-
-Cons:
-- Permanent state bloat (~40MB)
-- Must be loaded on every node forever
-- Lookup overhead for historical operations
-
-### The Nuclear Option: Binary Search-Replace
-
-For maximum chaos (don't actually do this):
-```cpp
-// Build LUT
-std::map<std::array<uint8_t, 32>, std::array<uint8_t, 32>> binary_lut;
-
-// Scan serialized blobs and replace
-for (auto& node : shamap) {
-    auto data = node.getData();
-    for (size_t i = 0; i <= data.size() - 32; i++) {
-        if (binary_lut.count(data[i..i+31])) {
-            memcpy(&data[i], binary_lut[data[i..i+31]], 32);
-        }
-    }
-}
-```
-
-Why this is insane:
-- False positives would corrupt data
-- No validation of what you're replacing
-- Breaks all checksums and signatures
-- Impossible to debug when it goes wrong
-
-## Conclusion
-
-The rekeying problem is not just about changing hash functions - it's about maintaining referential integrity across millions of interlinked objects. Every key change cascades through the reference graph, making this one of the most complex migrations possible in a blockchain system.
-
-The lookup table approach makes it algorithmically feasible (linear time rather than quadratic), but the implementation complexity and risk remain enormous. You need to:
-1. Find every single field that could contain a key
-2. Update them all correctly
-3. Handle sorting changes in order books
-4. Avoid false positives
-5. Deal with custom fields from hooks
-6. Maintain consistency across the entire state
-
-This is likely why most blockchains never change their hash functions after genesis - even with an efficient algorithm, the complexity and risk are enormous.

TEST_FILES_TO_FIX.md

@@ -1,179 +0,0 @@
-# Test Files That Need hash_options Fixes
-
-## How to Check Compilation Errors
-
-Use the `compile_single_v2.py` script to check individual files:
-```bash
-# Check compilation errors for a specific file
-./compile_single_v2.py src/test/app/SomeFile_test.cpp -e 3 --errors-only
-
-# Get just the last few lines to see if it compiled successfully
-./compile_single_v2.py src/test/app/SomeFile_test.cpp 2>&1 | tail -5
-```
-
-## Originally Fixed Files (11 files)
-
-1. **src/test/app/Import_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-2. **src/test/app/LedgerReplay_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-3. **src/test/app/Offer_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-4. **src/test/app/SetHook_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-5. **src/test/app/SetHookTSH_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-6. **src/test/app/ValidatorList_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-7. **src/test/app/XahauGenesis_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options, keylet::fees() needs hash_options
-
-8. **src/test/consensus/NegativeUNL_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-9. **src/test/consensus/UNLReport_test.cpp**
-   - Status: Needs fixing
-   - Errors: keylet functions missing hash_options
-
-10. **src/test/jtx/impl/balance.cpp**
-    - Status: Needs fixing
-    - Errors: keylet functions missing hash_options
-
-11. **src/test/jtx/impl/Env.cpp**
-    - Status: Needs fixing
-    - Errors: keylet functions missing hash_options
-
-## Fix Strategy
-
-Each file needs:
-1. keylet function calls updated to include hash_options{ledger_seq, classifier}
-2. The ledger_seq typically comes from env.current()->seq() or view.seq()
-3. The classifier matches the keylet type (e.g., KEYLET_ACCOUNT, KEYLET_FEES, etc.)
-
-## Progress Tracking
-
-- [x] Import_test.cpp - FIXED
-- [x] LedgerReplay_test.cpp - FIXED
-- [x] Offer_test.cpp - FIXED
-- [x] SetHook_test.cpp - FIXED
-- [x] SetHookTSH_test.cpp - FIXED
-- [x] ValidatorList_test.cpp - FIXED (sha512Half calls updated with VALIDATOR_LIST_HASH classifier)
-- [x] XahauGenesis_test.cpp - FIXED (removed duplicate hash_options parameters)
-- [x] NegativeUNL_test.cpp - FIXED
-- [x] UNLReport_test.cpp - FIXED
-- [x] balance.cpp - FIXED
-- [x] Env.cpp - FIXED
-
-## All original 11 files have been successfully fixed!
-
-## Remaining Files Still Needing Fixes (9 files)
-
-### Status: NOT STARTED
-These files still have compilation errors and need hash_options fixes:
-
-1. **src/test/jtx/impl/uritoken.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/jtx/impl/uritoken.cpp -e 3 --errors-only`
-
-2. **src/test/jtx/impl/utility.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/jtx/impl/utility.cpp -e 3 --errors-only`
-
-3. **src/test/ledger/Directory_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/ledger/Directory_test.cpp -e 3 --errors-only`
-
-4. **src/test/ledger/Invariants_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/ledger/Invariants_test.cpp -e 3 --errors-only`
-
-5. **src/test/overlay/compression_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/overlay/compression_test.cpp -e 3 --errors-only`
-
-6. **src/test/rpc/AccountSet_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/rpc/AccountSet_test.cpp -e 3 --errors-only`
-
-7. **src/test/rpc/AccountTx_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/rpc/AccountTx_test.cpp -e 3 --errors-only`
-
-8. **src/test/rpc/Book_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/rpc/Book_test.cpp -e 3 --errors-only`
-
-9. **src/test/rpc/Catalogue_test.cpp**
-   - Status: Needs fixing
-   - Check errors: `./compile_single_v2.py src/test/rpc/Catalogue_test.cpp -e 3 --errors-only`
-
-## CRITICAL INSTRUCTIONS FOR FIXING
-
-### 1. Read the HashContext enum from digest.h
-**ALWAYS** check @src/ripple/protocol/digest.h lines 37-107 for the complete HashContext enum.
-This enum defines ALL the valid classifiers you can use in hash_options.
-
-### 2. Understanding hash_options constructor
-The hash_options struct (lines 110-126 in digest.h) has TWO constructors:
-- `hash_options(HashContext ctx)` - classifier only, no ledger index
-- `hash_options(std::uint32_t li, HashContext ctx)` - ledger index AND classifier
-
-### 3. How to classify each hash operation
-
-#### For keylet functions:
-- Match the keylet function name to the KEYLET_* enum value
-- Examples:
-  - `keylet::account()` → use `KEYLET_ACCOUNT`
-  - `keylet::fees()` → use `KEYLET_FEES`
-  - `keylet::trustline()` → use `KEYLET_TRUSTLINE`
-  - `keylet::negativeUNL()` → use `KEYLET_NEGATIVE_UNL`
-  - `keylet::UNLReport()` → use `KEYLET_UNL_REPORT`
-  - `keylet::hook()` → use `KEYLET_HOOK`
-  - `keylet::uriToken()` → use `KEYLET_URI_TOKEN`
-
-#### For sha512Half calls:
-- Validator manifests/lists → use `VALIDATOR_LIST_HASH`
-- Hook code hashing → use `HOOK_DEFINITION` or `LEDGER_INDEX_UNNEEDED`
-- Network protocol → use appropriate context from enum
-
-#### For test environments:
-- Use `env.current()->seq()` to get ledger sequence (it's already uint32_t, NO CAST NEEDED)
-- Use `ledger->seq()` for Ledger pointers
-- Use `view.seq()` for ReadView/ApplyView references
-
-### 4. IMPORTANT: Read the entire file first!
-When fixing a file, ALWAYS:
-1. Read the ENTIRE file first (or at least 500+ lines) to understand the context
-2. Look for patterns of how the test is structured
-3. Check what types of ledger objects are being tested
-4. Then fix ALL occurrences systematically
-
-### 5. Common patterns to fix:
-
-```cpp
-// OLD - missing hash_options
-env.le(keylet::account(alice));
-
-// NEW - with proper classification  
-env.le(keylet::account(hash_options{env.current()->seq(), KEYLET_ACCOUNT}, alice));
-
-// OLD - sha512Half without context
-auto hash = sha512Half(data);
-
-// NEW - with proper classification
-auto hash = sha512Half(hash_options{VALIDATOR_LIST_HASH}, data);
-```

analyze_keylet_calls.py

@@ -1,204 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-from pathlib import Path
-from collections import defaultdict
-from typing import Set, Dict, List, Tuple
-
-def find_keylet_calls(root_dir: str) -> Dict[str, List[Tuple[str, int, str]]]:
-    """
-    Find all keylet:: function calls with hash_options as first parameter.
-    Returns a dict mapping keylet function names to list of (file, line, full_match) tuples.
-    """
-    # Pattern to match keylet::<function>(hash_options{...}, ...) calls
-    # This captures:
-    # 1. The keylet function name
-    # 2. The entire first argument (hash_options{...})
-    # 3. The content inside hash_options{...}
-    pattern = re.compile(
-        r'keylet::(\w+)\s*\(\s*(hash_options\s*\{([^}]*)\})',
-        re.MULTILINE | re.DOTALL
-    )
-    
-    results = defaultdict(list)
-    unique_first_args = set()
-    
-    # Walk through all C++ source files
-    for root, dirs, files in os.walk(Path(root_dir) / "src" / "ripple"):
-        # Skip certain directories
-        dirs[:] = [d for d in dirs if d not in ['.git', 'build', '__pycache__']]
-        
-        for file in files:
-            if file.endswith(('.cpp', '.h', '.hpp')):
-                filepath = os.path.join(root, file)
-                try:
-                    with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
-                        content = f.read()
-                        
-                    # Find all matches in this file
-                    for match in pattern.finditer(content):
-                        func_name = match.group(1)
-                        full_first_arg = match.group(2)
-                        inner_content = match.group(3).strip()
-                        
-                        # Get line number
-                        line_num = content[:match.start()].count('\n') + 1
-                        
-                        # Store the result
-                        rel_path = os.path.relpath(filepath, root_dir)
-                        results[func_name].append((rel_path, line_num, full_first_arg))
-                        unique_first_args.add(inner_content)
-                        
-                except Exception as e:
-                    print(f"Error reading {filepath}: {e}")
-    
-    return results, unique_first_args
-
-def analyze_hash_options_content(unique_args: Set[str]) -> Dict[str, int]:
-    """Analyze the content of hash_options{...} arguments."""
-    categories = {
-        'literal_0': 0,
-        'literal_number': 0,
-        'view_seq': 0,
-        'ledger_seq': 0,
-        'info_seq': 0,
-        'ctx_view_seq': 0,
-        'sb_seq': 0,
-        'env_current_seq': 0,
-        'other': 0
-    }
-    
-    other_patterns = []
-    
-    for arg in unique_args:
-        arg_clean = arg.strip()
-        
-        if arg_clean == '0':
-            categories['literal_0'] += 1
-        elif arg_clean.isdigit():
-            categories['literal_number'] += 1
-        elif 'view.seq()' in arg_clean or 'view().seq()' in arg_clean:
-            categories['view_seq'] += 1
-        elif 'ledger->seq()' in arg_clean or 'ledger.seq()' in arg_clean:
-            categories['ledger_seq'] += 1
-        elif 'info.seq' in arg_clean or 'info_.seq' in arg_clean:
-            categories['info_seq'] += 1
-        elif 'ctx.view.seq()' in arg_clean:
-            categories['ctx_view_seq'] += 1
-        elif 'sb.seq()' in arg_clean:
-            categories['sb_seq'] += 1
-        elif 'env.current()->seq()' in arg_clean:
-            categories['env_current_seq'] += 1
-        else:
-            categories['other'] += 1
-            other_patterns.append(arg_clean)
-    
-    return categories, other_patterns
-
-def print_report(results: Dict[str, List], unique_args: Set[str]):
-    """Print a detailed report of findings."""
-    print("=" * 80)
-    print("KEYLET FUNCTION CALL ANALYSIS")
-    print("=" * 80)
-    
-    # Summary
-    total_calls = sum(len(calls) for calls in results.values())
-    print(f"\nTotal keylet calls found: {total_calls}")
-    print(f"Unique keylet functions: {len(results)}")
-    print(f"Unique hash_options arguments: {len(unique_args)}")
-    
-    # Function frequency
-    print("\n" + "=" * 80)
-    print("KEYLET FUNCTIONS BY FREQUENCY:")
-    print("=" * 80)
-    
-    sorted_funcs = sorted(results.items(), key=lambda x: len(x[1]), reverse=True)
-    for func_name, calls in sorted_funcs[:20]:  # Top 20
-        print(f"  {func_name:30} {len(calls):4} calls")
-    
-    if len(sorted_funcs) > 20:
-        print(f"  ... and {len(sorted_funcs) - 20} more functions")
-    
-    # Analyze hash_options content
-    print("\n" + "=" * 80)
-    print("HASH_OPTIONS ARGUMENT PATTERNS:")
-    print("=" * 80)
-    
-    categories, other_patterns = analyze_hash_options_content(unique_args)
-    
-    for category, count in sorted(categories.items(), key=lambda x: x[1], reverse=True):
-        if count > 0:
-            print(f"  {category:25} {count:4} occurrences")
-    
-    if other_patterns:
-        print("\n" + "=" * 80)
-        print("OTHER PATTERNS (need review):")
-        print("=" * 80)
-        for i, pattern in enumerate(sorted(set(other_patterns))[:10], 1):
-            # Truncate long patterns
-            display = pattern if len(pattern) <= 60 else pattern[:57] + "..."
-            print(f"  {i:2}. {display}")
-    
-    # Sample calls for most common functions
-    print("\n" + "=" * 80)
-    print("SAMPLE CALLS (top 5 functions):")
-    print("=" * 80)
-    
-    for func_name, calls in sorted_funcs[:5]:
-        print(f"\n{func_name}:")
-        for filepath, line_num, arg in calls[:3]:  # Show first 3 examples
-            print(f"  {filepath}:{line_num}")
-            print(f"    {arg}")
-        if len(calls) > 3:
-            print(f"  ... and {len(calls) - 3} more")
-
-def generate_replacement_script(results: Dict[str, List], unique_args: Set[str]):
-    """Generate a script to help with replacements."""
-    print("\n" + "=" * 80)
-    print("SUGGESTED MIGRATION STRATEGY:")
-    print("=" * 80)
-    
-    print("""
-The goal is to migrate from:
-  keylet::func(hash_options{ledger_seq})
-  
-To either:
-  keylet::func(hash_options{ledger_seq, KEYLET_CLASSIFIER})
-  
-Where KEYLET_CLASSIFIER would be a specific HashContext enum value
-based on the keylet function type.
-
-Suggested mappings:
-  - keylet::account()     -> LEDGER_HEADER_HASH (or new KEYLET_ACCOUNT)
-  - keylet::line()        -> LEDGER_HEADER_HASH (or new KEYLET_TRUSTLINE)
-  - keylet::offer()       -> LEDGER_HEADER_HASH (or new KEYLET_OFFER)
-  - keylet::ownerDir()    -> LEDGER_HEADER_HASH (or new KEYLET_OWNER_DIR)
-  - keylet::page()        -> LEDGER_HEADER_HASH (or new KEYLET_DIR_PAGE)
-  - keylet::fees()        -> LEDGER_HEADER_HASH (or new KEYLET_FEES)
-  - keylet::amendments()  -> LEDGER_HEADER_HASH (or new KEYLET_AMENDMENTS)
-  - keylet::check()       -> LEDGER_HEADER_HASH (or new KEYLET_CHECK)
-  - keylet::escrow()      -> LEDGER_HEADER_HASH (or new KEYLET_ESCROW)
-  - keylet::payChan()     -> LEDGER_HEADER_HASH (or new KEYLET_PAYCHAN)
-  - keylet::signers()     -> LEDGER_HEADER_HASH (or new KEYLET_SIGNERS)
-  - keylet::ticket()      -> LEDGER_HEADER_HASH (or new KEYLET_TICKET)
-  - keylet::nftpage_*()   -> LEDGER_HEADER_HASH (or new KEYLET_NFT_PAGE)
-  - keylet::nftoffer()    -> LEDGER_HEADER_HASH (or new KEYLET_NFT_OFFER)
-  - keylet::depositPreauth() -> LEDGER_HEADER_HASH (or new KEYLET_DEPOSIT_PREAUTH)
-""")
-
-if __name__ == "__main__":
-    # Get the project root directory
-    project_root = "/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc"
-    
-    print(f"Analyzing keylet calls in: {project_root}")
-    print("This may take a moment...\n")
-    
-    # Find all keylet calls
-    results, unique_args = find_keylet_calls(project_root)
-    
-    # Print the report
-    print_report(results, unique_args)
-    
-    # Generate replacement suggestions
-    generate_replacement_script(results, unique_args)

cfg/genesis.json

@@ -62,7 +62,8 @@
           "3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87",
           "0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3",
           "6E739F4F8B07BED29FC9FF440DA3C301CD14A180DF45819F658FEC2F7DE31427",
-          "36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6"
+          "36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6",
+          "DD4F86291F142A20761B32B4D0CE4291F86CA33F0B46F0D04171482FBA52E536"
         ],
         "Flags": 0,
         "LedgerEntryType": "Amendments",

cfg/rippled-standalone.cfg

@@ -1,4 +1,4 @@
-# standalone: ./rippled -a --ledgerfile config/genesis.json --conf config/rippled-standalone.cfg
+# standalone: ./rippled -a --ledgerfile cfg/genesis.json --conf cfg/rippled-standalone.cfg
 [server]
 port_rpc_admin_local
 port_ws_public
@@ -153,3 +153,4 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
 EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
 8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
 88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911
+DD4F86291F142A20761B32B4D0CE4291F86CA33F0B46F0D04171482FBA52E536 JsHooks

compile_single.py

@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-"""
-Compile a single file using commands from compile_commands.json
-"""
-
-import json
-import os
-import sys
-import subprocess
-import argparse
-from pathlib import Path
-
-
-def find_compile_command(compile_commands, file_path):
-    """Find the compile command for a given file path."""
-    # Normalize the input path
-    abs_path = os.path.abspath(file_path)
-    
-    for entry in compile_commands:
-        # Check if this entry matches our file
-        entry_file = os.path.abspath(entry['file'])
-        if entry_file == abs_path:
-            return entry
-    
-    # Try relative path matching as fallback
-    for entry in compile_commands:
-        if entry['file'].endswith(file_path) or file_path.endswith(entry['file']):
-            return entry
-    
-    return None
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Compile a single file using compile_commands.json'
-    )
-    parser.add_argument(
-        'file',
-        help='Path to the source file to compile'
-    )
-    parser.add_argument(
-        '--verbose', '-v',
-        action='store_true',
-        help='Show the compile command being executed'
-    )
-    parser.add_argument(
-        '--dump-output', '-d',
-        action='store_true',
-        help='Dump the full output from the compiler'
-    )
-    parser.add_argument(
-        '--compile-db',
-        default='build/compile_commands.json',
-        help='Path to compile_commands.json (default: build/compile_commands.json)'
-    )
-    
-    args = parser.parse_args()
-    
-    # Check if compile_commands.json exists
-    if not os.path.exists(args.compile_db):
-        print(f"Error: {args.compile_db} not found", file=sys.stderr)
-        print("Make sure you've run cmake with -DCMAKE_EXPORT_COMPILE_COMMANDS=ON", file=sys.stderr)
-        sys.exit(1)
-    
-    # Load compile commands
-    try:
-        with open(args.compile_db, 'r') as f:
-            compile_commands = json.load(f)
-    except json.JSONDecodeError as e:
-        print(f"Error parsing {args.compile_db}: {e}", file=sys.stderr)
-        sys.exit(1)
-    
-    # Find the compile command for the requested file
-    entry = find_compile_command(compile_commands, args.file)
-    
-    if not entry:
-        print(f"Error: No compile command found for {args.file}", file=sys.stderr)
-        print(f"Available files in {args.compile_db}:", file=sys.stderr)
-        # Show first 10 files as examples
-        for i, cmd in enumerate(compile_commands[:10]):
-            print(f"  {cmd['file']}", file=sys.stderr)
-        if len(compile_commands) > 10:
-            print(f"  ... and {len(compile_commands) - 10} more", file=sys.stderr)
-        sys.exit(1)
-    
-    # Extract the command and directory
-    command = entry['command']
-    directory = entry.get('directory', '.')
-    
-    if args.verbose:
-        print(f"Directory: {directory}", file=sys.stderr)
-        print(f"Command: {command}", file=sys.stderr)
-        print("-" * 80, file=sys.stderr)
-    
-    # Execute the compile command
-    try:
-        result = subprocess.run(
-            command,
-            shell=True,
-            cwd=directory,
-            capture_output=not args.dump_output,
-            text=True
-        )
-        
-        if args.dump_output:
-            # Output was already printed to stdout/stderr
-            pass
-        else:
-            # Only show output if there were errors or warnings
-            if result.stderr:
-                print(result.stderr, file=sys.stderr)
-            if result.stdout:
-                print(result.stdout)
-        
-        # Exit with the same code as the compiler
-        sys.exit(result.returncode)
-        
-    except subprocess.SubprocessError as e:
-        print(f"Error executing compile command: {e}", file=sys.stderr)
-        sys.exit(1)
-    except KeyboardInterrupt:
-        print("\nCompilation interrupted", file=sys.stderr)
-        sys.exit(130)
-
-
-if __name__ == '__main__':
-    main()

compile_single_v2.py

@@ -1,311 +0,0 @@
-#!/usr/bin/env python3
-"""
-Compile a single file using commands from compile_commands.json
-Enhanced version with error context display
-"""
-
-import json
-import os
-import sys
-import subprocess
-import argparse
-import re
-import logging
-from pathlib import Path
-
-
-def setup_logging(level):
-    """Setup logging configuration."""
-    numeric_level = getattr(logging, level.upper(), None)
-    if not isinstance(numeric_level, int):
-        raise ValueError(f'Invalid log level: {level}')
-    
-    logging.basicConfig(
-        level=numeric_level,
-        format='[%(levelname)s] %(message)s',
-        stream=sys.stderr
-    )
-
-
-def find_compile_command(compile_commands, file_path):
-    """Find the compile command for a given file path."""
-    # Normalize the input path
-    abs_path = os.path.abspath(file_path)
-    logging.debug(f"Looking for compile command for: {abs_path}")
-    
-    for entry in compile_commands:
-        # Check if this entry matches our file
-        entry_file = os.path.abspath(entry['file'])
-        if entry_file == abs_path:
-            logging.debug(f"Found exact match: {entry_file}")
-            return entry
-    
-    # Try relative path matching as fallback
-    for entry in compile_commands:
-        if entry['file'].endswith(file_path) or file_path.endswith(entry['file']):
-            logging.debug(f"Found relative match: {entry['file']}")
-            return entry
-    
-    logging.debug("No compile command found")
-    return None
-
-
-def extract_errors_with_context(output, file_path, context_lines=3):
-    """Extract error messages with context from compiler output."""
-    lines = output.split('\n')
-    errors = []
-    
-    logging.debug(f"Parsing {len(lines)} lines of compiler output")
-    logging.debug(f"Looking for errors in file: {file_path}")
-    
-    # Pattern to match error lines from clang/gcc
-    # Matches: filename:line:col: error: message
-    # Also handle color codes
-    error_pattern = re.compile(r'([^:]+):(\d+):(\d+):\s*(?:\x1b\[[0-9;]*m)?\s*(error|warning):\s*(?:\x1b\[[0-9;]*m)?\s*(.*?)(?:\x1b\[[0-9;]*m)?$')
-    
-    for i, line in enumerate(lines):
-        # Strip ANSI color codes for pattern matching
-        clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line)
-        match = error_pattern.search(clean_line)
-        
-        if match:
-            filename = match.group(1)
-            line_num = int(match.group(2))
-            col_num = int(match.group(3))
-            error_type = match.group(4)
-            message = match.group(5)
-            
-            logging.debug(f"Found {error_type} at {filename}:{line_num}:{col_num}")
-            
-            # Check if this error is from the file we're compiling
-            # Be more flexible with path matching
-            if (file_path in filename or 
-                filename.endswith(os.path.basename(file_path)) or
-                os.path.basename(filename) == os.path.basename(file_path)):
-                
-                logging.debug(f"  -> Including {error_type}: {message[:50]}...")
-                
-                error_info = {
-                    'line': line_num,
-                    'col': col_num,
-                    'type': error_type,
-                    'message': message,
-                    'full_line': line,  # Keep original line with colors
-                    'context_before': [],
-                    'context_after': []
-                }
-                
-                # Get context lines from compiler output
-                for j in range(max(0, i - context_lines), i):
-                    error_info['context_before'].append(lines[j])
-                
-                for j in range(i + 1, min(len(lines), i + context_lines + 1)):
-                    error_info['context_after'].append(lines[j])
-                
-                errors.append(error_info)
-            else:
-                logging.debug(f"  -> Skipping (different file: {filename})")
-    
-    logging.info(f"Found {len(errors)} errors/warnings")
-    return errors
-
-
-def read_source_context(file_path, line_num, context_lines=3):
-    """Read context from the source file around a specific line."""
-    try:
-        with open(file_path, 'r') as f:
-            lines = f.readlines()
-        
-        start = max(0, line_num - context_lines - 1)
-        end = min(len(lines), line_num + context_lines)
-        
-        context = []
-        for i in range(start, end):
-            line_marker = '>>> ' if i == line_num - 1 else '    '
-            context.append(f"{i+1:4d}:{line_marker}{lines[i].rstrip()}")
-        
-        return '\n'.join(context)
-    except Exception as e:
-        logging.warning(f"Could not read source context: {e}")
-        return None
-
-
-def format_error_with_context(error, file_path, show_source_context=False):
-    """Format an error with its context."""
-    output = []
-    output.append(f"\n{'='*80}")
-    output.append(f"Error at line {error['line']}, column {error['col']}:")
-    output.append(f"  {error['message']}")
-    
-    if show_source_context:
-        source_context = read_source_context(file_path, error['line'], 3)
-        if source_context:
-            output.append("\nSource context:")
-            output.append(source_context)
-    
-    if error['context_before'] or error['context_after']:
-        output.append("\nCompiler output context:")
-        for line in error['context_before']:
-            output.append(f"  {line}")
-        output.append(f">>> {error['full_line']}")
-        for line in error['context_after']:
-            output.append(f"  {line}")
-    
-    return '\n'.join(output)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Compile a single file using compile_commands.json with enhanced error display'
-    )
-    parser.add_argument(
-        'file',
-        help='Path to the source file to compile'
-    )
-    parser.add_argument(
-        '--verbose', '-v',
-        action='store_true',
-        help='Show the compile command being executed'
-    )
-    parser.add_argument(
-        '--dump-output', '-d',
-        action='store_true',
-        help='Dump the full output from the compiler'
-    )
-    parser.add_argument(
-        '--show-error-context', '-e',
-        type=int,
-        metavar='N',
-        help='Show N lines of context around each error (implies capturing output)'
-    )
-    parser.add_argument(
-        '--show-source-context', '-s',
-        action='store_true',
-        help='Show source file context around errors'
-    )
-    parser.add_argument(
-        '--errors-only',
-        action='store_true',
-        help='Only show errors, not warnings'
-    )
-    parser.add_argument(
-        '--compile-db',
-        default='build/compile_commands.json',
-        help='Path to compile_commands.json (default: build/compile_commands.json)'
-    )
-    parser.add_argument(
-        '--log-level', '-l',
-        default='WARNING',
-        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
-        help='Set logging level (default: WARNING)'
-    )
-    
-    args = parser.parse_args()
-    
-    # Setup logging
-    setup_logging(args.log_level)
-    
-    # Check if compile_commands.json exists
-    if not os.path.exists(args.compile_db):
-        print(f"Error: {args.compile_db} not found", file=sys.stderr)
-        print("Make sure you've run cmake with -DCMAKE_EXPORT_COMPILE_COMMANDS=ON", file=sys.stderr)
-        sys.exit(1)
-    
-    # Load compile commands
-    try:
-        with open(args.compile_db, 'r') as f:
-            compile_commands = json.load(f)
-        logging.info(f"Loaded {len(compile_commands)} compile commands")
-    except json.JSONDecodeError as e:
-        print(f"Error parsing {args.compile_db}: {e}", file=sys.stderr)
-        sys.exit(1)
-    
-    # Find the compile command for the requested file
-    entry = find_compile_command(compile_commands, args.file)
-    
-    if not entry:
-        print(f"Error: No compile command found for {args.file}", file=sys.stderr)
-        print(f"Available files in {args.compile_db}:", file=sys.stderr)
-        # Show first 10 files as examples
-        for i, cmd in enumerate(compile_commands[:10]):
-            print(f"  {cmd['file']}", file=sys.stderr)
-        if len(compile_commands) > 10:
-            print(f"  ... and {len(compile_commands) - 10} more", file=sys.stderr)
-        sys.exit(1)
-    
-    # Extract the command and directory
-    command = entry['command']
-    directory = entry.get('directory', '.')
-    source_file = entry['file']
-    
-    if args.verbose:
-        print(f"Directory: {directory}", file=sys.stderr)
-        print(f"Command: {command}", file=sys.stderr)
-        print("-" * 80, file=sys.stderr)
-    
-    logging.info(f"Compiling {source_file}")
-    logging.debug(f"Working directory: {directory}")
-    logging.debug(f"Command: {command}")
-    
-    # Execute the compile command
-    try:
-        # If we need to show error context, we must capture output
-        capture = not args.dump_output or args.show_error_context is not None
-        
-        logging.debug(f"Running compiler (capture={capture})")
-        result = subprocess.run(
-            command,
-            shell=True,
-            cwd=directory,
-            capture_output=capture,
-            text=True
-        )
-        
-        logging.info(f"Compiler returned code: {result.returncode}")
-        
-        if args.dump_output and not args.show_error_context:
-            # Output was already printed to stdout/stderr
-            pass
-        elif args.show_error_context is not None:
-            # Parse and display errors with context
-            all_output = result.stderr + "\n" + result.stdout
-            
-            # Log first few lines of output for debugging
-            output_lines = all_output.split('\n')[:10]
-            for line in output_lines:
-                logging.debug(f"Output: {line}")
-            
-            errors = extract_errors_with_context(all_output, args.file, args.show_error_context)
-            
-            if args.errors_only:
-                errors = [e for e in errors if e['type'] == 'error']
-                logging.info(f"Filtered to {len(errors)} errors only")
-            
-            print(f"\nFound {len(errors)} {'error' if args.errors_only else 'error/warning'}(s) in {args.file}:\n")
-            
-            for error in errors:
-                print(format_error_with_context(error, source_file, args.show_source_context))
-            
-            if errors:
-                print(f"\n{'='*80}")
-                print(f"Total: {len(errors)} {'error' if args.errors_only else 'error/warning'}(s)")
-        else:
-            # Default behavior - show output if there were errors or warnings
-            if result.stderr:
-                print(result.stderr, file=sys.stderr)
-            if result.stdout:
-                print(result.stdout)
-        
-        # Exit with the same code as the compiler
-        sys.exit(result.returncode)
-        
-    except subprocess.SubprocessError as e:
-        print(f"Error executing compile command: {e}", file=sys.stderr)
-        sys.exit(1)
-    except KeyboardInterrupt:
-        print("\nCompilation interrupted", file=sys.stderr)
-        sys.exit(130)
-
-
-if __name__ == '__main__':
-    main()

conanfile.py

@@ -24,19 +24,18 @@ class Xrpl(ConanFile):
     }
 
     requires = [
-        'blake3/1.5.0@xahaud/stable',
         'boost/1.86.0',
         'date/3.0.1',
         'libarchive/3.6.0',
         'lz4/1.9.3',
         'grpc/1.50.1',
         'nudb/2.0.8',
-        'openssl/3.3.2',
+        'openssl/1.1.1u',
         'protobuf/3.21.9',
         'snappy/1.1.10',
         'soci/4.0.3',
         'sqlite3/3.42.0',
-        'zlib/1.3.1',
+        'zlib/1.2.13',
         'wasmedge/0.11.2',
     ]
 
@@ -52,7 +51,6 @@ class Xrpl(ConanFile):
         'tests': True,
         'unity': False,
 
-        'blake3:simd': False,  # Disable SIMD for testing
         'cassandra-cpp-driver:shared': False,
         'date:header_only': True,
         'grpc:shared': False,

external/blake3/conandata.yml

@@ -1,10 +0,0 @@
-sources:
-  "1.5.0":
-    url: "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.5.0.tar.gz"
-    sha256: "f506140bc3af41d3432a4ce18b3b83b08eaa240e94ef161eb72b2e57cdc94c69"
-  "1.4.1":
-    url: "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.4.1.tar.gz"
-    sha256: "33020ac83a8169b2e847cc6fb1dd38806ffab6efe79fe6c320e322154a3bea2c"
-  "1.4.0":
-    url: "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.4.0.tar.gz"
-    sha256: "658e1c75e2d9bbed9f426385f02d2a188dc19978a39e067ba93e837861e5fe58"

external/blake3/conanfile.py

@@ -1,115 +0,0 @@
-from conan import ConanFile
-from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
-from conan.tools.files import copy, get
-from conan.tools.scm import Version
-import os
-
-required_conan_version = ">=1.54.0"
-
-
-class Blake3Conan(ConanFile):
-    name = "blake3"
-    version = "1.5.0"
-    description = "BLAKE3 cryptographic hash function"
-    topics = ("blake3", "hash", "cryptography")
-    url = "https://github.com/BLAKE3-team/BLAKE3"
-    homepage = "https://github.com/BLAKE3-team/BLAKE3"
-    license = "CC0-1.0 OR Apache-2.0"
-    
-    package_type = "library"
-    settings = "os", "arch", "compiler", "build_type"
-    options = {
-        "shared": [True, False],
-        "fPIC": [True, False],
-        "simd": [True, False],
-    }
-    default_options = {
-        "shared": False,
-        "fPIC": True,
-        "simd": False,  # Default to NO SIMD for testing
-    }
-
-    def config_options(self):
-        if self.settings.os == 'Windows':
-            del self.options.fPIC
-
-    def configure(self):
-        if self.options.shared:
-            self.options.rm_safe("fPIC")
-        # BLAKE3 is C code
-        self.settings.rm_safe("compiler.cppstd")
-        self.settings.rm_safe("compiler.libcxx")
-
-    def layout(self):
-        cmake_layout(self, src_folder="src")
-
-    def source(self):
-        get(self, **self.conan_data["sources"][self.version], strip_root=True)
-
-    def generate(self):
-        tc = CMakeToolchain(self)
-        # BLAKE3's CMake options
-        tc.variables["BUILD_SHARED_LIBS"] = self.options.shared
-        if not self.options.simd:
-            # For v1.5.0, we'll need to manually patch the CMakeLists.txt
-            # These flags don't work with the old CMake
-            tc.preprocessor_definitions["BLAKE3_USE_NEON"] = "0"
-        tc.generate()
-
-    def build(self):
-        # Patch CMakeLists.txt if SIMD is disabled
-        if not self.options.simd:
-            cmake_file = os.path.join(self.source_folder, "c", "CMakeLists.txt")
-            # Read the file
-            with open(cmake_file, 'r') as f:
-                content = f.read()
-            # Replace the ARM detection line to never match
-            content = content.replace(
-                'elseif(CMAKE_SYSTEM_PROCESSOR IN_LIST BLAKE3_ARMv8_NAMES',
-                'elseif(FALSE  # Disabled by conan simd=False'
-            )
-            # Write it back
-            with open(cmake_file, 'w') as f:
-                f.write(content)
-                
-        cmake = CMake(self)
-        # BLAKE3's C implementation has its CMakeLists.txt in the c/ subdirectory
-        cmake.configure(build_script_folder=os.path.join(self.source_folder, "c"))
-        cmake.build()
-
-    def package(self):
-        # Copy license files
-        copy(self, "LICENSE*", src=self.source_folder, 
-             dst=os.path.join(self.package_folder, "licenses"))
-        # Copy header
-        copy(self, "blake3.h", 
-             src=os.path.join(self.source_folder, "c"),
-             dst=os.path.join(self.package_folder, "include"))
-        # Copy library
-        copy(self, "*.a", src=self.build_folder, 
-             dst=os.path.join(self.package_folder, "lib"), keep_path=False)
-        copy(self, "*.lib", src=self.build_folder, 
-             dst=os.path.join(self.package_folder, "lib"), keep_path=False)
-        copy(self, "*.dylib", src=self.build_folder, 
-             dst=os.path.join(self.package_folder, "lib"), keep_path=False)
-        copy(self, "*.so*", src=self.build_folder, 
-             dst=os.path.join(self.package_folder, "lib"), keep_path=False)
-        copy(self, "*.dll", src=self.build_folder, 
-             dst=os.path.join(self.package_folder, "bin"), keep_path=False)
-
-    def package_info(self):
-        self.cpp_info.set_property("cmake_file_name", "BLAKE3")
-        self.cpp_info.set_property("cmake_target_name", "BLAKE3::blake3")
-        
-        # IMPORTANT: Explicitly set include directories to fix Conan CMakeDeps generation
-        self.cpp_info.includedirs = ["include"]
-        self.cpp_info.libs = ["blake3"]
-        
-        # System libraries
-        if self.settings.os in ["Linux", "FreeBSD"]:
-            self.cpp_info.system_libs.append("m")
-            self.cpp_info.system_libs.append("pthread")
-        
-        # TODO: to remove in conan v2 once cmake_find_package* generators removed
-        self.cpp_info.names["cmake_find_package"] = "BLAKE3"
-        self.cpp_info.names["cmake_find_package_multi"] = "BLAKE3"

fix_final_tests.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_keylet_without_hash_options(filepath: Path) -> int:
-    """Fix keylet calls without hash_options in test files."""
-    
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f:
-            content = f.read()
-    except Exception as e:
-        print(f"Error reading {filepath}: {e}")
-        return 0
-    
-    original_content = content
-    replacements = 0
-    
-    # Pattern to match keylet calls without hash_options
-    # E.g., keylet::ownerDir(acct.id()) or keylet::account(alice)
-    keylet_funcs = {
-        'ownerDir': 'KEYLET_OWNER_DIR',
-        'account': 'KEYLET_ACCOUNT', 
-        'signers': 'KEYLET_SIGNERS',
-        'offer': 'KEYLET_OFFER',
-        'line': 'KEYLET_TRUSTLINE',
-        'check': 'KEYLET_CHECK',
-        'escrow': 'KEYLET_ESCROW',
-        'payChan': 'KEYLET_PAYCHAN',
-        'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-        'ticket': 'KEYLET_TICKET',
-        'nftoffer': 'KEYLET_NFT_OFFER',
-        'fees': 'KEYLET_FEES',
-        'amendments': 'KEYLET_AMENDMENTS',
-        'negativeUNL': 'KEYLET_NEGATIVE_UNL',
-        'skip': 'KEYLET_SKIP_LIST',
-        'hook': 'KEYLET_HOOK',
-        'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-        'hookState': 'KEYLET_HOOK_STATE',
-        'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-        'emittedDir': 'KEYLET_EMITTED_DIR',
-        'emittedTxn': 'KEYLET_EMITTED_TXN',
-        'import_vlseq': 'KEYLET_IMPORT_VLSEQ',
-        'unchecked': 'KEYLET_UNCHECKED',
-        'uritoken': 'KEYLET_URI_TOKEN',
-        'nftpage': 'KEYLET_NFT_PAGE',
-        'nftpage_min': 'KEYLET_NFT_PAGE',
-        'nftpage_max': 'KEYLET_NFT_PAGE',
-        'nft_buys': 'KEYLET_NFT_BUYS',
-        'nft_sells': 'KEYLET_NFT_SELLS',
-        'child': 'KEYLET_CHILD',
-        'page': 'KEYLET_DIR_PAGE',
-        'UNLReport': 'KEYLET_UNL_REPORT',
-        'book': 'KEYLET_BOOK'
-    }
-    
-    for func, classifier in keylet_funcs.items():
-        # Pattern to match keylet::<func>(...) where the args don't start with hash_options
-        pattern = re.compile(
-            rf'\bkeylet::{re.escape(func)}\s*\(\s*(?!hash_options)',
-            re.MULTILINE
-        )
-        
-        matches = list(pattern.finditer(content))
-        
-        # Process matches in reverse order to maintain positions
-        for match in reversed(matches):
-            start = match.start()
-            # Find the matching closing parenthesis
-            paren_count = 1
-            pos = match.end()
-            while pos < len(content) and paren_count > 0:
-                if content[pos] == '(':
-                    paren_count += 1
-                elif content[pos] == ')':
-                    paren_count -= 1
-                pos += 1
-            
-            if paren_count == 0:
-                # Extract the full function call
-                full_call = content[start:pos]
-                args_start = match.end()
-                args_end = pos - 1
-                args = content[args_start:args_end]
-                
-                # Determine ledger sequence to use
-                ledger_seq = None
-                if 'view' in content[max(0, start-500):start]:
-                    if 'view.seq()' in content[max(0, start-500):start]:
-                        ledger_seq = '(view.seq())'
-                    else:
-                        ledger_seq = '0'
-                elif 'env' in content[max(0, start-500):start]:
-                    if 'env.current()' in content[max(0, start-500):start]:
-                        ledger_seq = '(env.current()->seq())'
-                    else:
-                        ledger_seq = '0'
-                else:
-                    ledger_seq = '0'
-                
-                # Build the new call
-                new_call = f'keylet::{func}(hash_options{{{ledger_seq}, {classifier}}}, {args})'
-                
-                # Replace in content
-                content = content[:start] + new_call + content[pos:]
-                replacements += 1
-    
-    if replacements > 0 and content != original_content:
-        with open(filepath, 'w', encoding='utf-8') as f:
-            f.write(content)
-        return replacements
-    
-    return 0
-
-def main():
-    project_root = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc")
-    
-    # Files to fix
-    test_files = [
-        "src/test/app/URIToken_test.cpp",
-        "src/test/app/Touch_test.cpp",
-        "src/test/app/SetRemarks_test.cpp",
-        "src/test/app/Remit_test.cpp",
-        "src/test/app/HookNegatives_test.cpp",
-        "src/test/app/Hook_test.cpp", 
-        "src/test/app/NFTokenBurn_test.cpp",
-        "src/test/app/NFToken_test.cpp",
-        "src/test/app/TxMeta_test.cpp",
-        "src/test/app/AccountTxPaging_test.cpp"
-    ]
-    
-    total_replacements = 0
-    
-    for rel_path in test_files:
-        filepath = project_root / rel_path
-        if filepath.exists():
-            replacements = fix_keylet_without_hash_options(filepath)
-            if replacements > 0:
-                print(f"Fixed {rel_path}: {replacements} replacements")
-                total_replacements += replacements
-    
-    print(f"\nTotal replacements: {total_replacements}")
-
-if __name__ == "__main__":
-    main()

fix_remaining_tests.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_file(filepath: Path) -> int:
-    """Fix various hash_options issues in a single file."""
-    
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f:
-            content = f.read()
-    except Exception as e:
-        print(f"Error reading {filepath}: {e}")
-        return 0
-    
-    original_content = content
-    replacements = 0
-    
-    # Fix duplicate keylet calls with hash_options
-    # Pattern: keylet::X(keylet::X(hash_options{...}, ...))
-    pattern1 = re.compile(
-        r'keylet::(\w+)\s*\(\s*keylet::\1\s*\(\s*hash_options\s*\{[^}]+\}[^)]*\)\s*\)',
-        re.MULTILINE
-    )
-    
-    def fix_duplicate(match):
-        nonlocal replacements
-        # Extract just the inner keylet call
-        inner = match.group(0)
-        # Find the position of the second keylet::
-        second_keylet_pos = inner.find('keylet::', 8)  # Skip first occurrence
-        if second_keylet_pos != -1:
-            # Extract everything after the second keylet::
-            fixed = 'keylet::' + inner[second_keylet_pos + 8:]
-            # Remove the extra closing paren at the end
-            if fixed.endswith('))'):
-                fixed = fixed[:-1]
-            replacements += 1
-            return fixed
-        return match.group(0)
-    
-    content = pattern1.sub(fix_duplicate, content)
-    
-    # Fix keylet calls without hash_options (like keylet::ownerDir(acc.id()))
-    # These need hash_options added
-    keylet_funcs = ['ownerDir', 'account', 'signers', 'offer']
-    for func in keylet_funcs:
-        # Pattern to match keylet::func(args) where args doesn't start with hash_options
-        pattern2 = re.compile(
-            rf'keylet::{func}\s*\(\s*(?!hash_options)([^)]+)\)',
-            re.MULTILINE
-        )
-        
-        def add_hash_options(match):
-            nonlocal replacements
-            args = match.group(1).strip()
-            # Determine the classifier based on function name
-            classifier_map = {
-                'ownerDir': 'KEYLET_OWNER_DIR',
-                'account': 'KEYLET_ACCOUNT',
-                'signers': 'KEYLET_SIGNERS',
-                'offer': 'KEYLET_OFFER'
-            }
-            classifier = classifier_map.get(func, 'LEDGER_INDEX_UNNEEDED')
-            
-            # Check if we're in a context where we can get env.current()->seq()
-            # Look back in the content to see if we're in a lambda or function with env
-            pos = match.start()
-            # Simple heuristic: if we see "env" within 500 chars before, use it
-            context = content[max(0, pos-500):pos]
-            if 'env.' in context or 'env)' in context or '&env' in context:
-                replacements += 1
-                return f'keylet::{func}(hash_options{{(env.current()->seq()), {classifier}}}, {args})'
-            else:
-                # Try view instead
-                if 'view' in context or 'ReadView' in context:
-                    replacements += 1
-                    return f'keylet::{func}(hash_options{{0, {classifier}}}, {args})'
-            return match.group(0)
-        
-        content = pattern2.sub(add_hash_options, content)
-    
-    # Fix missing closing parenthesis for keylet::account calls
-    pattern3 = re.compile(
-        r'(keylet::account\s*\(\s*hash_options\s*\{[^}]+\}\s*,\s*\w+(?:\.\w+\(\))?\s*)(\);)',
-        re.MULTILINE
-    )
-    
-    def fix_paren(match):
-        nonlocal replacements
-        replacements += 1
-        return match.group(1) + '));'
-    
-    content = pattern3.sub(fix_paren, content)
-    
-    if replacements > 0 and content != original_content:
-        with open(filepath, 'w', encoding='utf-8') as f:
-            f.write(content)
-        return replacements
-    
-    return 0
-
-def main():
-    project_root = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc")
-    
-    # Files to fix
-    test_files = [
-        "src/test/app/RCLValidations_test.cpp",
-        "src/test/app/PayStrand_test.cpp", 
-        "src/test/app/PayChan_test.cpp",
-        "src/test/app/ClaimReward_test.cpp",
-        "src/test/app/Import_test.cpp",
-        "src/test/app/LedgerReplay_test.cpp",
-        "src/test/app/Offer_test.cpp"
-    ]
-    
-    total_replacements = 0
-    
-    for rel_path in test_files:
-        filepath = project_root / rel_path
-        if filepath.exists():
-            replacements = fix_file(filepath)
-            if replacements > 0:
-                print(f"Fixed {rel_path}: {replacements} replacements")
-                total_replacements += replacements
-    
-    print(f"\nTotal replacements: {total_replacements}")
-
-if __name__ == "__main__":
-    main()

fix_sethook_ledger_seq.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_sethook_ledger_sequences():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/SetHook_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix keylet::hookState calls with hash_options{0, KEYLET_HOOK_STATE}
-    # These are inside test functions where env is available
-    content = re.sub(
-        r'keylet::hookState\(hash_options\{0, KEYLET_HOOK_STATE\}',
-        r'keylet::hookState(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE}',
-        content
-    )
-    
-    # Fix keylet::hookStateDir calls with hash_options{0, KEYLET_HOOK_STATE_DIR}
-    content = re.sub(
-        r'keylet::hookStateDir\(hash_options\{0, KEYLET_HOOK_STATE_DIR\}',
-        r'keylet::hookStateDir(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE_DIR}',
-        content
-    )
-    
-    # The sha512Half_s and sha512Half calls with LEDGER_INDEX_UNNEEDED are CORRECT
-    # because they're hashing non-ledger data (WASM bytecode, nonces, etc.)
-    # So we leave those alone.
-    
-    # The HASH_WASM macro uses are also CORRECT with LEDGER_INDEX_UNNEEDED
-    # because they're computing hashes of WASM bytecode at compile time,
-    # not ledger objects.
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-    print("Note: sha512Half* calls with LEDGER_INDEX_UNNEEDED are correct (hashing non-ledger data)")
-    print("Note: HASH_WASM macro uses are correct (compile-time WASM bytecode hashing)")
-
-if __name__ == "__main__":
-    fix_sethook_ledger_sequences()

fix_sethook_test.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_sethook_test():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/SetHook_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix keylet::hook(Account(...).id()) patterns
-    # These are looking up hooks on accounts in the ledger, so need env.current()->seq()
-    content = re.sub(
-        r'env\.le\(keylet::hook\(Account\(([^)]+)\)\.id\(\)\)\)',
-        r'env.le(keylet::hook(hash_options{(env.current()->seq()), KEYLET_HOOK}, Account(\1).id()))',
-        content
-    )
-    
-    # Fix other keylet::hook patterns with just an ID
-    content = re.sub(
-        r'env\.le\(keylet::hook\((\w+)\.id\(\)\)\)',
-        r'env.le(keylet::hook(hash_options{(env.current()->seq()), KEYLET_HOOK}, \1.id()))',
-        content
-    )
-    
-    # Fix patterns like keylet::hook(alice)
-    content = re.sub(
-        r'env\.le\(keylet::hook\((\w+)\)\)',
-        r'env.le(keylet::hook(hash_options{(env.current()->seq()), KEYLET_HOOK}, \1))',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_sethook_test()

fix_sethook_test_complete.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_sethook_test():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/SetHook_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix ALL keylet::hookState patterns including those with ripple:: prefix
-    # Pattern 1: ripple::keylet::hookState(id, key, ns)
-    content = re.sub(
-        r'ripple::keylet::hookState\(([^,]+),\s*([^,]+),\s*([^)]+)\)',
-        r'ripple::keylet::hookState(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE}, \1, \2, \3)',
-        content
-    )
-    
-    # Pattern 2: keylet::hookState without ripple:: prefix (if not already fixed)
-    content = re.sub(
-        r'(?<!ripple::)keylet::hookState\(([^h][^,]+),\s*([^,]+),\s*([^)]+)\)',
-        r'keylet::hookState(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE}, \1, \2, \3)',
-        content
-    )
-    
-    # Fix ripple::keylet::hookStateDir patterns
-    content = re.sub(
-        r'ripple::keylet::hookStateDir\(([^,]+),\s*([^)]+)\)',
-        r'ripple::keylet::hookStateDir(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE_DIR}, \1, \2)',
-        content
-    )
-    
-    # Fix ripple::keylet::hook patterns
-    content = re.sub(
-        r'ripple::keylet::hook\(([^)]+)\)(?!\))',
-        r'ripple::keylet::hook(hash_options{(env.current()->seq()), KEYLET_HOOK}, \1)',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_sethook_test()

fix_sethook_test_v2.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_sethook_test():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/SetHook_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix keylet::hookState - it takes 4 args now (hash_options + 3 original)
-    # Match patterns like: keylet::hookState(Account("alice").id(), key, ns)
-    content = re.sub(
-        r'keylet::hookState\(Account\(([^)]+)\)\.id\(\),\s*([^,]+),\s*([^)]+)\)',
-        r'keylet::hookState(hash_options{0, KEYLET_HOOK_STATE}, Account(\1).id(), \2, \3)',
-        content
-    )
-    
-    # Match patterns with variables like: keylet::hookState(alice.id(), key, ns)
-    content = re.sub(
-        r'keylet::hookState\((\w+)\.id\(\),\s*([^,]+),\s*([^)]+)\)',
-        r'keylet::hookState(hash_options{0, KEYLET_HOOK_STATE}, \1.id(), \2, \3)',
-        content
-    )
-    
-    # Match patterns with just IDs: keylet::hookState(accid, key, ns)
-    content = re.sub(
-        r'keylet::hookState\((\w+),\s*([^,]+),\s*([^)]+)\)(?!\))',
-        r'keylet::hookState(hash_options{0, KEYLET_HOOK_STATE}, \1, \2, \3)',
-        content
-    )
-    
-    # Fix keylet::hookStateDir patterns
-    content = re.sub(
-        r'keylet::hookStateDir\((\w+),\s*([^,]+),\s*([^)]+)\)',
-        r'keylet::hookStateDir(hash_options{0, KEYLET_HOOK_STATE_DIR}, \1, \2, \3)',
-        content
-    )
-    
-    # Fix sha512Half_s calls without hash_options  
-    content = re.sub(
-        r'sha512Half_s\(ripple::Slice\(',
-        r'sha512Half_s(hash_options{0, LEDGER_INDEX_UNNEEDED}, ripple::Slice(',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_sethook_test()

fix_sethook_test_v3.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_sethook_test():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/SetHook_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix keylet::hookStateDir - it takes 3 args now (hash_options + 2 original)
-    # Match patterns like: keylet::hookStateDir(Account("alice").id(), ns)
-    content = re.sub(
-        r'keylet::hookStateDir\(Account\(([^)]+)\)\.id\(\),\s*([^)]+)\)',
-        r'keylet::hookStateDir(hash_options{0, KEYLET_HOOK_STATE_DIR}, Account(\1).id(), \2)',
-        content
-    )
-    
-    # Match with variables
-    content = re.sub(
-        r'keylet::hookStateDir\((\w+)\.id\(\),\s*([^)]+)\)',
-        r'keylet::hookStateDir(hash_options{0, KEYLET_HOOK_STATE_DIR}, \1.id(), \2)',
-        content
-    )
-    
-    # Fix multiline hookStateDir patterns
-    content = re.sub(
-        r'keylet::hookStateDir\(\s*\n\s*Account\(([^)]+)\)\.id\(\),\s*([^)]+)\)',
-        r'keylet::hookStateDir(hash_options{0, KEYLET_HOOK_STATE_DIR},\n                Account(\1).id(), \2)',
-        content,
-        flags=re.MULTILINE
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_sethook_test()

fix_test_hash_options.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-from pathlib import Path
-
-def fix_test_files(root_dir: str):
-    """Fix hash_options in test files by adding appropriate classifiers."""
-    
-    # Pattern to match hash_options with only ledger sequence
-    # This will match things like hash_options{(env.current()->seq())}
-    pattern = re.compile(
-        r'(keylet::(\w+)\s*\([^)]*\s*)hash_options\s*\{\s*\(([^}]+)\)\s*\}',
-        re.MULTILINE
-    )
-    
-    # Mapping of keylet functions to their classifiers
-    keylet_classifiers = {
-        'account': 'KEYLET_ACCOUNT',
-        'ownerDir': 'KEYLET_OWNER_DIR',
-        'signers': 'KEYLET_SIGNERS',
-        'offer': 'KEYLET_OFFER',
-        'check': 'KEYLET_CHECK',
-        'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-        'escrow': 'KEYLET_ESCROW',
-        'payChan': 'KEYLET_PAYCHAN',
-        'line': 'KEYLET_TRUSTLINE',
-        'ticket': 'KEYLET_TICKET',
-        'hook': 'KEYLET_HOOK',
-        'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-        'hookState': 'KEYLET_HOOK_STATE',
-        'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-        'child': 'KEYLET_CHILD',
-        'page': 'KEYLET_DIR_PAGE',
-        'nftpage_min': 'KEYLET_NFT_PAGE',
-        'nftpage_max': 'KEYLET_NFT_PAGE',
-        'nftoffer': 'KEYLET_NFT_OFFER',
-        'nft_buys': 'KEYLET_NFT_BUYS',
-        'nft_sells': 'KEYLET_NFT_SELLS',
-        'uritoken': 'KEYLET_URI_TOKEN',
-    }
-    
-    files_fixed = 0
-    total_replacements = 0
-    
-    # Find all test files
-    test_dir = Path(root_dir) / "src" / "test"
-    
-    for filepath in test_dir.rglob("*.cpp"):
-        try:
-            with open(filepath, 'r', encoding='utf-8') as f:
-                original_content = f.read()
-            
-            content = original_content
-            replacements = 0
-            
-            def replacer(match):
-                nonlocal replacements
-                prefix = match.group(1)
-                keylet_func = match.group(2)
-                ledger_expr = match.group(3)
-                
-                # Get the classifier for this keylet function
-                classifier = keylet_classifiers.get(keylet_func)
-                if not classifier:
-                    print(f"WARNING: No classifier for keylet::{keylet_func} in {filepath}")
-                    # Default to a generic one
-                    classifier = 'KEYLET_UNCHECKED'
-                
-                replacements += 1
-                # Reconstruct with the classifier
-                return f'{prefix}hash_options{{({ledger_expr}), {classifier}}}'
-            
-            new_content = pattern.sub(replacer, content)
-            
-            # Also fix standalone hash_options calls (not in keylet context)
-            # These are likely in helper functions or direct usage
-            standalone_pattern = re.compile(
-                r'(?<!keylet::\w{1,50}\s{0,10}\([^)]*\s{0,10})hash_options\s*\{\s*\(([^}]+)\)\s*\}',
-                re.MULTILINE
-            )
-            
-            def standalone_replacer(match):
-                nonlocal replacements
-                ledger_expr = match.group(1)
-                replacements += 1
-                # For standalone ones in tests, use a test context
-                return f'hash_options{{({ledger_expr}), LEDGER_INDEX_UNNEEDED}}'
-            
-            new_content = standalone_pattern.sub(standalone_replacer, new_content)
-            
-            if replacements > 0:
-                with open(filepath, 'w', encoding='utf-8') as f:
-                    f.write(new_content)
-                
-                rel_path = filepath.relative_to(root_dir)
-                print(f"Fixed {rel_path}: {replacements} replacements")
-                files_fixed += 1
-                total_replacements += replacements
-                
-        except Exception as e:
-            print(f"Error processing {filepath}: {e}")
-    
-    print(f"\n{'='*60}")
-    print(f"Fixed {files_fixed} test files")
-    print(f"Total replacements: {total_replacements}")
-    
-    return files_fixed, total_replacements
-
-if __name__ == "__main__":
-    project_root = "/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc"
-    
-    print("Fixing hash_options in test files...")
-    files_fixed, total_replacements = fix_test_files(project_root)
-    
-    if files_fixed > 0:
-        print("\nDone! Now rebuild to see if there are any remaining issues.")
-    else:
-        print("\nNo test files needed fixing.")

fix_test_hash_options_simple.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-from pathlib import Path
-
-def fix_test_files(root_dir: str):
-    """Fix hash_options in test files by adding appropriate classifiers."""
-    
-    # Mapping of keylet functions to their classifiers  
-    keylet_classifiers = {
-        'account': 'KEYLET_ACCOUNT',
-        'ownerDir': 'KEYLET_OWNER_DIR',
-        'signers': 'KEYLET_SIGNERS',
-        'offer': 'KEYLET_OFFER',
-        'check': 'KEYLET_CHECK',
-        'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-        'escrow': 'KEYLET_ESCROW',
-        'payChan': 'KEYLET_PAYCHAN',
-        'line': 'KEYLET_TRUSTLINE',
-        'ticket': 'KEYLET_TICKET',
-        'hook': 'KEYLET_HOOK',
-        'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-        'hookState': 'KEYLET_HOOK_STATE',
-        'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-        'child': 'KEYLET_CHILD',
-        'page': 'KEYLET_DIR_PAGE',
-        'nftpage_min': 'KEYLET_NFT_PAGE',
-        'nftpage_max': 'KEYLET_NFT_PAGE',
-        'nftoffer': 'KEYLET_NFT_OFFER',
-        'nft_buys': 'KEYLET_NFT_BUYS',
-        'nft_sells': 'KEYLET_NFT_SELLS',
-        'uritoken': 'KEYLET_URI_TOKEN',
-        'fees': 'KEYLET_FEES',
-        'amendments': 'KEYLET_AMENDMENTS',
-        'negativeUNL': 'KEYLET_NEGATIVE_UNL',
-        'skip': 'KEYLET_SKIP_LIST',
-        'unchecked': 'KEYLET_UNCHECKED',
-        'import_vlseq': 'KEYLET_IMPORT_VLSEQ',
-        'UNLReport': 'KEYLET_UNL_REPORT',
-        'emittedDir': 'KEYLET_EMITTED_DIR',
-        'emittedTxn': 'KEYLET_EMITTED_TXN',
-        'book': 'KEYLET_BOOK',
-    }
-    
-    files_fixed = 0
-    total_replacements = 0
-    
-    # Find all test files
-    test_dir = Path(root_dir) / "src" / "test"
-    
-    for filepath in test_dir.rglob("*.cpp"):
-        try:
-            with open(filepath, 'r', encoding='utf-8') as f:
-                original_content = f.read()
-            
-            content = original_content
-            replacements = 0
-            
-            # Process line by line for better control
-            lines = content.split('\n')
-            new_lines = []
-            
-            for line in lines:
-                modified = False
-                # Look for keylet:: calls with hash_options that only have ledger seq
-                for func_name, classifier in keylet_classifiers.items():
-                    # Pattern for keylet::func(...hash_options{(ledger_seq)}...)
-                    pattern = f'keylet::{func_name}\\s*\\([^)]*hash_options\\s*\\{{\\s*\\(([^}}]+)\\)\\s*\\}}'
-                    
-                    matches = list(re.finditer(pattern, line))
-                    if matches:
-                        # Process from end to start to maintain positions
-                        for match in reversed(matches):
-                            ledger_expr = match.group(1)
-                            # Check if it already has a classifier (contains comma)
-                            if ',' not in ledger_expr:
-                                # Replace with classifier added
-                                new_text = f'keylet::{func_name}(' + line[match.start():match.end()].replace(
-                                    f'hash_options{{({ledger_expr})}}',
-                                    f'hash_options{{({ledger_expr}), {classifier}}}'
-                                )
-                                line = line[:match.start()] + new_text + line[match.end():]
-                                replacements += 1
-                                modified = True
-                
-                # Also look for standalone hash_options (not in keylet context)
-                if not modified and 'hash_options{(' in line and '),' not in line:
-                    # Simple pattern for standalone hash_options{(expr)}
-                    standalone_pattern = r'(?<!keylet::\w{1,30}\([^)]*\s{0,5})hash_options\s*\{\s*\(([^}]+)\)\s*\}'
-                    matches = list(re.finditer(standalone_pattern, line))
-                    for match in reversed(matches):
-                        ledger_expr = match.group(1)
-                        if ',' not in ledger_expr:
-                            # For standalone ones in tests, use LEDGER_INDEX_UNNEEDED
-                            line = line[:match.start()] + f'hash_options{{({ledger_expr}), LEDGER_INDEX_UNNEEDED}}' + line[match.end():]
-                            replacements += 1
-                
-                new_lines.append(line)
-            
-            if replacements > 0:
-                new_content = '\n'.join(new_lines)
-                with open(filepath, 'w', encoding='utf-8') as f:
-                    f.write(new_content)
-                
-                rel_path = filepath.relative_to(root_dir)
-                print(f"Fixed {rel_path}: {replacements} replacements")
-                files_fixed += 1
-                total_replacements += replacements
-                
-        except Exception as e:
-            print(f"Error processing {filepath}: {e}")
-    
-    print(f"\n{'='*60}")
-    print(f"Fixed {files_fixed} test files")
-    print(f"Total replacements: {total_replacements}")
-    
-    return files_fixed, total_replacements
-
-if __name__ == "__main__":
-    project_root = "/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc"
-    
-    print("Fixing hash_options in test files...")
-    files_fixed, total_replacements = fix_test_files(project_root)
-    
-    if files_fixed > 0:
-        print("\nDone! Now rebuild to see if there are any remaining issues.")
-    else:
-        print("\nNo test files needed fixing.")

fix_test_hash_options_v2.py

@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-from pathlib import Path
-
-# Mapping of keylet functions to their specific HashContext classifiers
-KEYLET_CLASSIFIERS = {
-    'account': 'KEYLET_ACCOUNT',
-    'amendments': 'KEYLET_AMENDMENTS',
-    'book': 'KEYLET_BOOK',
-    'check': 'KEYLET_CHECK',
-    'child': 'KEYLET_CHILD',
-    'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-    'emittedDir': 'KEYLET_EMITTED_DIR',
-    'emittedTxn': 'KEYLET_EMITTED_TXN',
-    'escrow': 'KEYLET_ESCROW',
-    'fees': 'KEYLET_FEES',
-    'hook': 'KEYLET_HOOK',
-    'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-    'hookState': 'KEYLET_HOOK_STATE',
-    'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-    'import_vlseq': 'KEYLET_IMPORT_VLSEQ',
-    'line': 'KEYLET_TRUSTLINE',
-    'negativeUNL': 'KEYLET_NEGATIVE_UNL',
-    'nft_buys': 'KEYLET_NFT_BUYS',
-    'nft_sells': 'KEYLET_NFT_SELLS',
-    'nftoffer': 'KEYLET_NFT_OFFER',
-    'nftpage': 'KEYLET_NFT_PAGE',
-    'nftpage_max': 'KEYLET_NFT_PAGE',
-    'nftpage_min': 'KEYLET_NFT_PAGE',
-    'offer': 'KEYLET_OFFER',
-    'ownerDir': 'KEYLET_OWNER_DIR',
-    'page': 'KEYLET_DIR_PAGE',
-    'payChan': 'KEYLET_PAYCHAN',
-    'signers': 'KEYLET_SIGNERS',
-    'skip': 'KEYLET_SKIP_LIST',
-    'ticket': 'KEYLET_TICKET',
-    'UNLReport': 'KEYLET_UNL_REPORT',
-    'unchecked': 'KEYLET_UNCHECKED',
-    'uritoken': 'KEYLET_URI_TOKEN',
-}
-
-def fix_keylet_calls_in_file(filepath: Path) -> int:
-    """Fix hash_options in a single file by adding appropriate classifiers."""
-    
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f:
-            content = f.read()
-    except Exception as e:
-        print(f"Error reading {filepath}: {e}")
-        return 0
-    
-    original_content = content
-    replacements = 0
-    
-    # Process each keylet function
-    for func_name, classifier in KEYLET_CLASSIFIERS.items():
-        # Pattern to match keylet::<func>(hash_options{<ledger_seq>}, ...)
-        # where ledger_seq doesn't already contain a comma (no classifier yet)
-        pattern = re.compile(
-            rf'keylet::{re.escape(func_name)}\s*\(\s*hash_options\s*\{{\s*\(([^,}}]+)\)\s*\}}',
-            re.MULTILINE
-        )
-        
-        def replacer(match):
-            nonlocal replacements
-            ledger_seq = match.group(1).strip()
-            replacements += 1
-            # Add the classifier
-            return f'keylet::{func_name}(hash_options{{({ledger_seq}), {classifier}}}'
-        
-        content = pattern.sub(replacer, content)
-    
-    # Also fix standalone hash_options that aren't in keylet calls
-    # These might be in test helper functions or other places
-    standalone_pattern = re.compile(
-        r'(?<!keylet::\w+\s*\(\s*)hash_options\s*\{\s*\(([^,}]+)\)\s*\}(?!\s*,)',
-        re.MULTILINE
-    )
-    
-    def standalone_replacer(match):
-        nonlocal replacements
-        ledger_seq = match.group(1).strip()
-        # Skip if it already has a classifier
-        if ',' in ledger_seq:
-            return match.group(0)
-        replacements += 1
-        # For standalone ones in tests, use LEDGER_INDEX_UNNEEDED
-        return f'hash_options{{({ledger_seq}), LEDGER_INDEX_UNNEEDED}}'
-    
-    # Apply standalone pattern only if we're in a test file
-    if '/test/' in str(filepath):
-        content = standalone_pattern.sub(standalone_replacer, content)
-    
-    if replacements > 0 and content != original_content:
-        with open(filepath, 'w', encoding='utf-8') as f:
-            f.write(content)
-        return replacements
-    
-    return 0
-
-def main():
-    project_root = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc")
-    
-    # Find all test cpp files that might have hash_options
-    test_files = list((project_root / "src" / "test").rglob("*.cpp"))
-    
-    print(f"Found {len(test_files)} test files to check...")
-    
-    total_files_fixed = 0
-    total_replacements = 0
-    
-    for filepath in test_files:
-        replacements = fix_keylet_calls_in_file(filepath)
-        if replacements > 0:
-            rel_path = filepath.relative_to(project_root)
-            print(f"Fixed {rel_path}: {replacements} replacements")
-            total_files_fixed += 1
-            total_replacements += replacements
-    
-    print(f"\n{'='*60}")
-    print(f"Fixed {total_files_fixed} test files")
-    print(f"Total replacements: {total_replacements}")
-    
-    if total_files_fixed > 0:
-        print("\nNow rebuild to see if there are any remaining issues.")
-    else:
-        print("\nNo test files needed fixing.")
-
-if __name__ == "__main__":
-    main()

fix_test_hash_options_v3.py

@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-from pathlib import Path
-
-# Mapping of keylet functions to their specific HashContext classifiers
-KEYLET_CLASSIFIERS = {
-    'account': 'KEYLET_ACCOUNT',
-    'amendments': 'KEYLET_AMENDMENTS',
-    'book': 'KEYLET_BOOK',
-    'check': 'KEYLET_CHECK',
-    'child': 'KEYLET_CHILD',
-    'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-    'emittedDir': 'KEYLET_EMITTED_DIR',
-    'emittedTxn': 'KEYLET_EMITTED_TXN',
-    'escrow': 'KEYLET_ESCROW',
-    'fees': 'KEYLET_FEES',
-    'hook': 'KEYLET_HOOK',
-    'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-    'hookState': 'KEYLET_HOOK_STATE',
-    'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-    'import_vlseq': 'KEYLET_IMPORT_VLSEQ',
-    'line': 'KEYLET_TRUSTLINE',
-    'negativeUNL': 'KEYLET_NEGATIVE_UNL',
-    'nft_buys': 'KEYLET_NFT_BUYS',
-    'nft_sells': 'KEYLET_NFT_SELLS',
-    'nftoffer': 'KEYLET_NFT_OFFER',
-    'nftpage': 'KEYLET_NFT_PAGE',
-    'nftpage_max': 'KEYLET_NFT_PAGE',
-    'nftpage_min': 'KEYLET_NFT_PAGE',
-    'offer': 'KEYLET_OFFER',
-    'ownerDir': 'KEYLET_OWNER_DIR',
-    'page': 'KEYLET_DIR_PAGE',
-    'payChan': 'KEYLET_PAYCHAN',
-    'signers': 'KEYLET_SIGNERS',
-    'skip': 'KEYLET_SKIP_LIST',
-    'ticket': 'KEYLET_TICKET',
-    'UNLReport': 'KEYLET_UNL_REPORT',
-    'unchecked': 'KEYLET_UNCHECKED',
-    'uritoken': 'KEYLET_URI_TOKEN',
-}
-
-def fix_keylet_calls_in_file(filepath: Path) -> int:
-    """Fix hash_options in a single file by adding appropriate classifiers."""
-    
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f:
-            content = f.read()
-    except Exception as e:
-        print(f"Error reading {filepath}: {e}")
-        return 0
-    
-    original_content = content
-    replacements = 0
-    
-    # Process each keylet function
-    for func_name, classifier in KEYLET_CLASSIFIERS.items():
-        # Pattern to match keylet::<func>(hash_options{<ledger_seq>}, ...)
-        # where ledger_seq doesn't already contain a comma (no classifier yet)
-        pattern = re.compile(
-            rf'keylet::{re.escape(func_name)}\s*\(\s*hash_options\s*\{{\s*\(([^}}]+)\)\s*\}}',
-            re.MULTILINE
-        )
-        
-        def replacer(match):
-            nonlocal replacements
-            ledger_seq = match.group(1).strip()
-            # Check if it already has a classifier (contains comma after the ledger expression)
-            # But be careful - the ledger expression itself might contain commas
-            # Look for a comma followed by a KEYLET_ or other classifier
-            if ', KEYLET_' in match.group(0) or ', LEDGER_' in match.group(0):
-                return match.group(0)
-            replacements += 1
-            # Add the classifier
-            return f'keylet::{func_name}(hash_options{{({ledger_seq}), {classifier}}}'
-        
-        content = pattern.sub(replacer, content)
-    
-    if replacements > 0 and content != original_content:
-        with open(filepath, 'w', encoding='utf-8') as f:
-            f.write(content)
-        return replacements
-    
-    return 0
-
-def main():
-    project_root = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc")
-    
-    # Find all test cpp files that might have hash_options
-    test_files = list((project_root / "src" / "test").rglob("*.cpp"))
-    
-    print(f"Found {len(test_files)} test files to check...")
-    
-    total_files_fixed = 0
-    total_replacements = 0
-    
-    for filepath in test_files:
-        replacements = fix_keylet_calls_in_file(filepath)
-        if replacements > 0:
-            rel_path = filepath.relative_to(project_root)
-            print(f"Fixed {rel_path}: {replacements} replacements")
-            total_files_fixed += 1
-            total_replacements += replacements
-    
-    print(f"\n{'='*60}")
-    print(f"Fixed {total_files_fixed} test files")
-    print(f"Total replacements: {total_replacements}")
-    
-    if total_files_fixed > 0:
-        print("\nNow rebuild to see if there are any remaining issues.")
-    else:
-        print("\nNo test files needed fixing.")
-
-if __name__ == "__main__":
-    main()

fix_xahau_genesis.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_xahau_genesis():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/XahauGenesis_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix sha512Half_s calls - they now need hash_options
-    content = re.sub(
-        r'ripple::sha512Half_s\(ripple::Slice\(',
-        r'ripple::sha512Half_s(hash_options{0, LEDGER_INDEX_UNNEEDED}, ripple::Slice(',
-        content
-    )
-    
-    # Fix keylet::amendments()
-    content = re.sub(
-        r'env\.le\(keylet::amendments\(\)\)',
-        r'env.le(keylet::amendments(hash_options{(env.current()->seq()), KEYLET_AMENDMENTS}))',
-        content
-    )
-    
-    # Fix keylet::account(id) calls
-    content = re.sub(
-        r'env\.le\(keylet::account\((\w+)\)\)',
-        r'env.le(keylet::account(hash_options{(env.current()->seq()), KEYLET_ACCOUNT}, \1))',
-        content
-    )
-    
-    # Fix keylet::hook(id) calls
-    content = re.sub(
-        r'env\.le\(keylet::hook\((\w+)\)\)',
-        r'env.le(keylet::hook(hash_options{(env.current()->seq()), KEYLET_HOOK}, \1))',
-        content
-    )
-    
-    # Fix keylet::hookDefinition calls
-    content = re.sub(
-        r'env\.le\(keylet::hookDefinition\(([^)]+)\)\)',
-        r'env.le(keylet::hookDefinition(hash_options{(env.current()->seq()), KEYLET_HOOK_DEFINITION}, \1))',
-        content
-    )
-    
-    # Fix keylet::fees() calls
-    content = re.sub(
-        r'env\.le\(keylet::fees\(\)\)',
-        r'env.le(keylet::fees(hash_options{(env.current()->seq()), KEYLET_FEES}))',
-        content
-    )
-    
-    # Fix standalone keylet::account assignments
-    content = re.sub(
-        r'(\s+auto\s+const\s+\w+Key\s*=\s*)keylet::account\((\w+)\);',
-        r'\1keylet::account(hash_options{(env.current()->seq()), KEYLET_ACCOUNT}, \2);',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_xahau_genesis()

fix_xahau_genesis_v2.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_xahau_genesis():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/XahauGenesis_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix ALL sha512Half_s calls - they now need hash_options
-    # Match multi-line patterns too
-    content = re.sub(
-        r'ripple::sha512Half_s\(',
-        r'ripple::sha512Half_s(hash_options{0, LEDGER_INDEX_UNNEEDED}, ',
-        content
-    )
-    
-    # Fix keylet::hookDefinition without hash_options
-    content = re.sub(
-        r'keylet::hookDefinition\(([^,)]+)\)(?!\.)',
-        r'keylet::hookDefinition(hash_options{0, KEYLET_HOOK_DEFINITION}, \1)',
-        content
-    )
-    
-    # Fix env.le(keylet::hookDefinition calls that might have been missed
-    content = re.sub(
-        r'env\.le\(keylet::hookDefinition\(hash_options\{0, KEYLET_HOOK_DEFINITION\}, ([^)]+)\)\)',
-        r'env.le(keylet::hookDefinition(hash_options{(env.current()->seq()), KEYLET_HOOK_DEFINITION}, \1))',
-        content
-    )
-    
-    # Fix keylet::account in view.read() calls
-    content = re.sub(
-        r'view->read\(keylet::account\((\w+)\)\)',
-        r'view->read(keylet::account(hash_options{(view->seq()), KEYLET_ACCOUNT}, \1))',
-        content
-    )
-    
-    # Fix env.current()->read(keylet::account calls
-    content = re.sub(
-        r'env\.current\(\)->read\(keylet::account\((\w+)\)\)',
-        r'env.current()->read(keylet::account(hash_options{(env.current()->seq()), KEYLET_ACCOUNT}, \1))',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_xahau_genesis()

fix_xahau_genesis_v3.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from pathlib import Path
-
-def fix_xahau_genesis():
-    filepath = Path("/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc/src/test/app/XahauGenesis_test.cpp")
-    
-    with open(filepath, 'r') as f:
-        content = f.read()
-    
-    # Fix keylet::signers without hash_options
-    content = re.sub(
-        r'env\.le\(keylet::signers\((\w+)\)\)',
-        r'env.le(keylet::signers(hash_options{(env.current()->seq()), KEYLET_SIGNERS}, \1))',
-        content
-    )
-    
-    # Fix keylet::hookState - it takes 4 arguments now (hash_options + 3 original)
-    content = re.sub(
-        r'keylet::hookState\(\s*([^,]+),\s*([^,]+),\s*([^)]+)\)',
-        r'keylet::hookState(hash_options{(env.current()->seq()), KEYLET_HOOK_STATE}, \1, \2, \3)',
-        content
-    )
-    
-    # Fix keylet::negativeUNL
-    content = re.sub(
-        r'env\.le\(keylet::negativeUNL\(\)\)',
-        r'env.le(keylet::negativeUNL(hash_options{(env.current()->seq()), KEYLET_NEGATIVE_UNL}))',
-        content
-    )
-    
-    with open(filepath, 'w') as f:
-        f.write(content)
-    
-    print(f"Fixed {filepath}")
-
-if __name__ == "__main__":
-    fix_xahau_genesis()

gemini-short-input-hash-deep-research.md

@@ -1,263 +0,0 @@
-
-# **A Performance and Security Analysis of Modern Hash Functions for Small-Input Payloads: Selecting a High-Speed Successor to SHA-512/256**
-
-## **Executive Summary & Introduction**
-
-### **The Challenge: The Need for Speed in Small-Payload Hashing**
-
-In modern computing systems, the performance of cryptographic hash functions is a critical design consideration. While functions from the SHA-2 family, such as SHA-512, are widely deployed and trusted for their robust security, they can represent a significant performance bottleneck in applications that process a high volume of small data payloads.1 Use cases such as the generation of authentication tokens, per-request key derivation, and the indexing of data in secure databases frequently involve hashing inputs of 128 bytes or less. In these scenarios, the computational overhead of legacy algorithms can impede system throughput and increase latency.
-
-This report addresses the specific challenge of selecting a high-performance, cryptographically secure replacement for sha512\_half, which is formally specified as SHA-512/256. The objective is to identify the fastest hash function that produces a 256-bit digest, thereby providing a 128-bit security level against collision attacks, while being optimized for inputs up to 128 bytes.3 The analysis is conducted within the context of modern 64-bit CPU architectures (x86-64 and ARMv8) and must account for the profound impact of hardware acceleration features, including both general-purpose Single Instruction, Multiple Data (SIMD) extensions and dedicated cryptographic instructions.
-
-### **The Contenders: Introducing the Candidates**
-
-To meet these requirements, this analysis will evaluate two leading-edge cryptographic hash functions against the established NIST standard, SHA-512/256, which serves as the performance and security baseline.
-
-* **The Incumbent (Baseline): SHA-512/256.** As a member of the venerable SHA-2 family, SHA-512/256 is a FIPS-standardized algorithm built upon the Merkle-Damgård construction.3 It leverages 64-bit arithmetic, which historically offered a performance advantage over its 32-bit counterpart, SHA-256, on 64-bit processors.6 A key feature of this truncated variant is its inherent resistance to length-extension attacks, a known vulnerability of SHA-512 and SHA-256.8 Its performance, particularly in the context of hardware acceleration, will serve as the primary benchmark for comparison.  
-* **The Modern Challengers: BLAKE3 and KangarooTwelve.** Two primary candidates have been identified based on their design goals, which explicitly target substantial performance improvements over legacy standards.  
-  * **BLAKE3:** Released in 2020, BLAKE3 represents the latest evolution of the BLAKE family of hash functions. It was engineered from the ground up for extreme speed and massive parallelism, utilizing a tree-based structure over a highly optimized compression function derived from ChaCha20.9 It is a single, unified algorithm designed to deliver exceptional performance across a wide array of platforms, from high-end servers to resource-constrained embedded systems.  
-  * **KangarooTwelve (K12):** KangarooTwelve is a high-speed eXtendable-Output Function (XOF) derived from the Keccak permutation, the same primitive that underpins the FIPS 202 SHA-3 standard.12 By significantly reducing the number of rounds from 24 (in SHA-3) to 12, K12 achieves a major speedup while leveraging the extensive security analysis of its parent algorithm.12
-
-### **Scope and Methodology**
-
-The scope of this report is strictly confined to cryptographic hash functions that provide a minimum 128-bit security level against all standard attack vectors, including collision, preimage, and second-preimage attacks. This focus necessitates the exclusion of non-cryptographic hash functions, despite their often-superior performance. Algorithms such as xxHash are explicitly designed for speed in non-adversarial contexts like hash tables and checksums, and they make no claims of cryptographic security.15
-
-The case of MeowHash serves as a potent cautionary tale. Designed for extreme speed on systems with AES hardware acceleration, it was initially promoted for certain security-adjacent use cases.18 However, subsequent public cryptanalysis revealed catastrophic vulnerabilities, including a practical key-recovery attack and the ability to generate collisions with probabilities far exceeding theoretical security bounds.19 These findings underscore the profound risks of employing algorithms outside their rigorously defined security context and firmly justify their exclusion from this analysis.
-
-The methodology employed herein is a multi-faceted evaluation that synthesizes empirical data with theoretical analysis. It comprises three core pillars:
-
-1. **Algorithmic Design Analysis:** An examination of the underlying construction (e.g., Merkle-Damgård, Sponge, Tree) and core cryptographic primitives of each candidate to understand their intrinsic performance characteristics and security properties.  
-2. **Security Posture Assessment:** A review of the stated security goals, the justification for design choices (such as reduced round counts), and the body of public cryptanalysis for each algorithm.  
-3. **Quantitative Performance Synthesis:** A comprehensive analysis of performance data from reputable, independent sources, including the eBACS/SUPERCOP benchmarking project, peer-reviewed academic papers, and official documentation from the algorithm designers. Performance will be normalized and compared across relevant architectures and input sizes to provide a clear, data-driven conclusion.
-
-## **Architectural Underpinnings of High-Speed Hashing**
-
-The performance of a hash function is not merely a product of its internal mathematics but is fundamentally dictated by its high-level construction and its interaction with the underlying CPU architecture. The evolution from serial, iterative designs to highly parallelizable tree structures, combined with the proliferation of hardware acceleration, has created a complex performance landscape.
-
-### **The Evolution of Hash Constructions: From Serial to Parallel**
-
-The way a hash function processes an input message is its most defining architectural characteristic, directly influencing its speed, security, and potential for parallelism.
-
-#### **Merkle-Damgård Construction (SHA-2)**
-
-The Merkle-Damgård construction is the foundational design of the most widely deployed hash functions, including the entire SHA-2 family.5 Its operation is inherently sequential. The input message is padded and divided into fixed-size blocks. A compression function,
-
-f, processes these blocks iteratively. The process begins with a fixed initialization vector (IV). For each message block Mi​, the compression function computes a new chaining value Hi​=f(Hi−1​,Mi​). The final hash output is derived from the last chaining value, Hn​.22
-
-This iterative dependency, where the input to one step is the output of the previous, makes the construction simple to implement but fundamentally limits parallelism for a single message. The processing of block Mi​ cannot begin until the processing of Mi−1​ is complete. Furthermore, the standard Merkle-Damgård construction is susceptible to length-extension attacks, where an attacker who knows the hash of a message M can compute the hash of M∥P∥Mnew​ for some padding P without knowing M. This vulnerability is a primary reason why truncated variants like SHA-512/256, which do not expose the full internal state in their output, are recommended for many security protocols.8
-
-#### **Sponge Construction (SHA-3 & KangarooTwelve)**
-
-The Sponge construction, standardized with SHA-3, represents a significant departure from the Merkle-Damgård paradigm.13 It operates on a fixed-size internal state,
-
-S, which is larger than the desired output size. The state is conceptually divided into two parts: an outer part, the *rate* (r), and an inner part, the *capacity* (c). The security of the function is determined by the size of the capacity.
-
-The process involves two phases 22:
-
-1. **Absorbing Phase:** The input message is padded and broken into blocks of size r. Each block is XORed into the rate portion of the state, after which a fixed, unkeyed permutation, f, is applied to the entire state. This process is repeated for all message blocks.  
-2. **Squeezing Phase:** Once all input has been absorbed, the output hash is generated. The rate portion of the state is extracted as the first block of output. If more output is required, the permutation f is applied again, and the new rate is extracted as the next block. This can be repeated to produce an output of arbitrary length, a capability known as an eXtendable-Output Function (XOF).24
-
-This design provides robust immunity to length-extension attacks because the capacity portion of the state is never directly modified by the message blocks nor directly outputted.25 This flexibility and security are central to KangarooTwelve's design.
-
-#### **Tree-Based Hashing (BLAKE3 & K12's Parallel Mode)**
-
-Tree-based hashing is the key innovation enabling the massive throughput of modern hash functions on large inputs.26 Instead of processing a message sequentially, the input is divided into a large number of independent chunks. These chunks form the leaves of a Merkle tree.27 Each chunk can be hashed in parallel, utilizing multiple CPU cores or the multiple "lanes" of a wide SIMD vector. The resulting intermediate hash values are then paired and hashed together to form parent nodes, continuing up the tree until a single root hash is produced.11
-
-This structure allows for a degree of parallelism limited only by the number of chunks, making it exceptionally well-suited to modern hardware. However, this parallelism comes with a crucial caveat for the use case in question. The tree hashing modes of both BLAKE3 and KangarooTwelve are only activated for inputs that exceed a certain threshold. For BLAKE3, this threshold is 1024 bytes 11; for KangarooTwelve, it is 8192 bytes.24 As the specified maximum input size is 128 bytes, it falls far below these thresholds. Consequently, the widely advertised parallelism advantage of these modern hashes, which is their primary performance driver for large file hashing, is
-
-**entirely irrelevant** to this specific analysis. The performance competition for small inputs is therefore not about parallelism but about the raw, single-threaded efficiency of the underlying compression function on a single block of data and the algorithm's initialization overhead. This reframes the entire performance evaluation, shifting the focus from architectural parallelism to the micro-architectural efficiency of the core cryptographic permutation.
-
-### **The Hardware Acceleration Landscape: SIMD and Dedicated Instructions**
-
-Modern CPUs are not simple scalar processors; they contain specialized hardware to accelerate common computational tasks, including cryptography. Understanding this landscape is critical, as the availability of acceleration for one algorithm but not another can create performance differences of an order of magnitude.
-
-#### **General-Purpose SIMD (Single Instruction, Multiple Data)**
-
-SIMD instruction sets allow a single instruction to operate on multiple data elements packed into a wide vector register. Key examples include SSE2, AVX2, and AVX-512 on x86-64 architectures, and NEON on ARMv8.9 Algorithms whose internal operations can be expressed as parallel, independent computations on smaller words (e.g., 32-bit or 64-bit) are ideal candidates for SIMD optimization. Both BLAKE3 and KangarooTwelve are designed to be highly friendly to SIMD implementation, which is the primary source of their speed in software on modern CPUs.32
-
-#### **Dedicated Cryptographic Extensions**
-
-In addition to general-purpose SIMD, many CPUs now include instructions specifically designed to accelerate standardized cryptographic algorithms.
-
-* **Intel SHA Extensions:** Introduced by Intel and adopted by AMD, these instructions provide hardware acceleration for SHA-1 and SHA-256.34 Their availability on a wide range of modern processors, from Intel Ice Lake and Rocket Lake onwards, and all AMD Zen processors, gives SHA-256 a formidable performance advantage over algorithms that must be implemented in software or with general-purpose SIMD.8 Critically, widespread hardware support for SHA-512 is a very recent development, only appearing in Intel's 2024 Arrow Lake and Lunar Lake architectures, and is not present in the vast majority of currently deployed systems.34  
-* **ARMv8 Cryptography Extensions:** The ARMv8 architecture includes optional cryptography extensions. The baseline extensions provide hardware support for AES, SHA-1, and SHA-256.35 Support for SHA-512 and SHA-3 (Keccak) was introduced as a further optional extension in the ARMv8.2-A revision.35 This means that on many ARMv8 devices, SHA-256 is hardware-accelerated while SHA-512 and Keccak-based functions are not. High-performance cores, such as Apple's M-series processors, do implement these advanced extensions, providing acceleration for all three families.12
-
-This disparity in hardware support creates a significant performance inversion. Historically, SHA-512 was often faster than SHA-256 on 64-bit CPUs because it processes larger 1024-bit blocks using 64-bit native operations, resulting in more data processed per round compared to SHA-256's 512-bit blocks and 32-bit operations.6 However, the introduction of dedicated SHA-256 hardware instructions provides a performance boost that far outweighs the architectural advantage of SHA-512's 64-bit design. On a modern CPU with SHA-256 extensions but no SHA-512 extensions, SHA-256 will be substantially faster.8 This elevates the performance bar for any proposed replacement for SHA-512/256; to be considered a truly "fast" alternative, a candidate must not only outperform software-based SHA-512 but also be competitive with hardware-accelerated SHA-256.
-
-## **Candidate Deep Dive: BLAKE3**
-
-BLAKE3 is a state-of-the-art cryptographic hash function designed with the explicit goal of being the fastest secure hash function available, leveraging parallelism at every level of modern CPU architecture.
-
-### **Algorithm and Design Rationale**
-
-BLAKE3 is a single, unified algorithm, avoiding the multiple variants of its predecessors (e.g., BLAKE2b, BLAKE2s).37 Its design is an elegant synthesis of two proven components: the BLAKE2s compression function and the Bao verified tree hashing mode.9
-
-* **Core Components:** The heart of BLAKE3 is its compression function, which is a modified version of the BLAKE2s compression function. BLAKE2s itself is based on the core permutation of the ChaCha stream cipher, an ARX (Add-Rotate-XOR) design known for its exceptional speed in software.11 BLAKE3 operates exclusively on 32-bit words, a deliberate choice that ensures high performance on both 64-bit and 32-bit architectures, from high-end x86 servers to low-power ARM cores.11  
-* **Reduced Round Count:** One of the most significant optimizations in BLAKE3 is the reduction of the number of rounds in its compression function from 10 (in BLAKE2s) to 7\.11 This 30% reduction in the core computational workload provides a direct and substantial increase in speed for processing each block of data.  
-* **Tree Structure:** As established, for the specified input range of up to 128 bytes, the tree structure is trivial. The input constitutes a single chunk, which is processed as the root node of the tree. This design ensures that for small inputs, there is no additional overhead from the tree mode; the performance is purely that of the highly optimized 7-round compression function.39
-
-### **Security Posture**
-
-Despite its focus on speed, BLAKE3 is designed to be a fully secure cryptographic hash function, suitable for a wide range of applications including digital signatures and message authentication codes.10
-
-* **Security Claims:** BLAKE3 targets a 128-bit security level for all standard goals, including collision resistance, preimage resistance, and differentiability.28 This security level is equivalent to that of SHA-256 and makes a 256-bit output appropriate and secure.  
-* **Justification for Reduced Rounds:** The decision to reduce the round count to 7 is grounded in the extensive public cryptanalysis of the BLAKE family. The original BLAKE was a finalist in the NIST SHA-3 competition, and both it and its successor BLAKE2 have been subjected to intense scrutiny.38 The best known attacks on BLAKE2 are only able to break a small fraction of its total rounds, indicating that the original 10 rounds of BLAKE2s already contained a very large security margin.33 The BLAKE3 designers concluded that 7 rounds still provides a comfortable margin of safety against known attack vectors while yielding a significant performance gain.  
-* **Inherent Security Features:** The tree-based mode of operation, even in its trivial form for small inputs, provides inherent immunity to length-extension attacks, a notable advantage over non-truncated members of the SHA-2 family like SHA-256 and SHA-512.9
-
-### **Performance Profile for Small Inputs**
-
-BLAKE3 was explicitly designed to excel not only on large, parallelizable inputs but also on the small inputs relevant to this analysis.
-
-* **Design Intent:** The official BLAKE3 paper and its authors state that performance for inputs of 64 bytes (the internal block size) or shorter is "best in class".28 The paper's benchmarks claim superior single-message throughput compared to SHA-256 for all input sizes.42  
-* **Benchmark Evidence:** While direct, cross-platform benchmarks for very small inputs are scarce, available data points consistently support BLAKE3's speed claims. In optimized Rust benchmarks on an x86-64 machine, hashing a single block with BLAKE3 (using AVX-512) took 43 ns, compared to 77 ns for BLAKE2s (using SSE4.1).43 This demonstrates the raw speed of the 7-round compression function. This is significant because BLAKE2s itself is already benchmarked as being faster than SHA-512 for most input sizes on modern CPUs.43 Therefore, by extension, BLAKE3's improved performance over BLAKE2s solidifies its position as a top contender for small-input speed.
-
-## **Candidate Deep Dive: KangarooTwelve**
-
-KangarooTwelve (K12) is a high-speed cryptographic hash function from the designers of Keccak/SHA-3. It aims to provide a much faster alternative to the official FIPS 202 standards while retaining the same underlying security principles and benefiting from the same extensive cryptanalysis.
-
-### **Algorithm and Design Rationale**
-
-K12 is best understood as a performance-tuned variant of the SHAKE eXtendable-Output Functions.
-
-* **Core Components:** The core primitive of K12 is the Keccak-p permutation.12 This is the same Keccak-p permutation used in all SHA-3 and SHAKE functions, but with the number of rounds reduced from 24 to 12\. For inputs up to its parallel threshold of 8192 bytes, K12's operation is a simple, flat sponge construction, functionally equivalent to a round-reduced version of SHAKE128.31 It uses a capacity of 256 bits, targeting a 128-bit security level.41  
-* **Reduced Round Count:** The primary source of K12's significant performance advantage over the standardized SHA-3 functions is the halving of the round count from 24 to 12\.13 This directly cuts the computational work of the core permutation in half, leading to a nearly 2x speedup for short messages compared to SHAKE128, the fastest of the FIPS 202 instances.12
-
-### **Security Posture**
-
-The security case for KangarooTwelve is directly inherited from the decade of intense international scrutiny applied to its parent, Keccak.
-
-* **Security Claims:** K12 targets a 128-bit security level against all standard attacks, including collision and preimage attacks, making it directly comparable to BLAKE3 and SHA-256.24  
-* **Justification for Reduced Rounds:** The decision to use 12 rounds is based on a conservative evaluation of the existing cryptanalysis of the Keccak permutation. At the time of K12's design, the best known practical collision attacks were only applicable up to 6 rounds of the permutation.49 The most powerful theoretical distinguishers could only reach 9 rounds.49 By selecting 12 rounds, the designers established a 100% security margin over the best known collision attacks and a 33% margin over the best theoretical distinguishers, a level they argue is comfortable and well-justified.49
-
-### **Performance Profile for Small Inputs**
-
-KangarooTwelve was designed to be fast for both long and short messages, addressing a perceived performance gap in the official SHA-3 standard.
-
-* **Design Intent:** The explicit goal for short messages was to be approximately twice as fast as SHAKE128.12 This makes it a compelling high-speed alternative for applications that require or prefer a Keccak-based construction.  
-* **Future-Proofing through Hardware Acceleration:** A key strategic advantage of K12 is its direct lineage from SHA-3. As CPU manufacturers increasingly adopt optional hardware acceleration for SHA-3 (as seen in ARMv8.2-A and later), K12 stands to benefit directly from these instructions.36 This provides a potential future performance pathway that is unavailable to algorithms like BLAKE3, which rely on general-purpose SIMD. On an Apple M1 processor, which includes these SHA-3 extensions, K12 is reported to be 1.7 times faster than hardware-accelerated SHA-256 and 3 times faster than hardware-accelerated SHA-512 for long messages, demonstrating the power of this dedicated hardware support.12
-
-## **Quantitative Performance Showdown**
-
-To provide a definitive recommendation, it is essential to move beyond theoretical designs and analyze empirical performance data. This section synthesizes results from multiple high-quality sources to build a comparative performance profile of the candidates across relevant architectures and the specified input range.
-
-### **Benchmarking Methodology and Caveats**
-
-Obtaining a single, perfectly consistent benchmark that compares all three candidates across all target architectures and input sizes is challenging. Therefore, this analysis relies on a synthesis of data from the eBACS/SUPERCOP project, which provides standardized performance metrics in cycles per byte (cpb) 53, supplemented by figures from the algorithms' design papers and other academic sources. The primary metric for comparison will be
-
-**single-message latency**, which measures the time required to hash one message from start to finish. This is the most relevant metric for general-purpose applications.
-
-It is important to distinguish this from multi-message throughput, which measures the aggregate performance when hashing many independent messages in parallel on a single core. As demonstrated in a high-throughput use case for the Solana platform, an optimized, batched implementation of hardware-accelerated SHA-256 can outperform BLAKE3 on small messages due to the simpler scheduling of the Merkle-Damgård construction into SIMD lanes.42 While this is a valid consideration for highly specialized, high-volume workloads, single-message latency remains the more universal measure of a hash function's "speed."
-
-### **Cross-Architectural Benchmark Synthesis**
-
-The following table presents a synthesized view of the performance of BLAKE3, KangarooTwelve, and the baseline SHA-512/256 for the specified input sizes. Performance is measured in median cycles per byte (cpb); lower values are better. The data represents estimates derived from a combination of official benchmarks and independent analyses on representative modern CPUs.
-
-**Comparative Performance of Hash Functions for Small Inputs (Median Cycles/Byte)**
-
-| Input Size (Bytes) | Intel Cascade Lake-SP (AVX-512) | Apple M1 (ARMv8 \+ Crypto Ext.) |
-| :---- | :---- | :---- |
-|  | **BLAKE3** | **KangarooTwelve** |
-| **16** | \~17 cpb | \~22 cpb |
-| **32** | \~10 cpb | \~14 cpb |
-| **64** | **\~5 cpb** | \~9 cpb |
-| **128** | **\~3 cpb** | \~6 cpb |
-| *Long Message (Ref.)* | *\~0.3 cpb* | *\~0.51 cpb* |
-
-Data synthesized from sources.12 SHA-512/256 values are based on software/SIMD performance for Intel and hardware-accelerated performance for Apple M1. The "Long Message" row is for reference to show peak throughput.
-
-### **Analysis of Performance Deltas and Architectural Nuances**
-
-The benchmark data reveals several critical trends that are essential for making an informed decision.
-
-* **Initialization Overhead:** For all algorithms, the cycles-per-byte metric is significantly higher for the smallest inputs (e.g., 16 bytes) and decreases as the input size grows. This reflects the fixed computational cost of initializing the hash state and performing finalization, which is amortized over a larger number of bytes for longer messages. The algorithm with the lowest fixed overhead will have an advantage on the smallest payloads.  
-* **x86-64 (AVX) Performance:** On the Intel Cascade Lake-SP platform, which lacks dedicated hardware acceleration for any of the candidates, **BLAKE3 demonstrates a clear and decisive performance advantage across the entire input range.** Its ARX-based design, inherited from ChaCha, is exceptionally well-suited to implementation with general-purpose SIMD instruction sets like AVX2 and AVX-512.9 As the input size approaches and fills its 64-byte block, BLAKE3's efficiency becomes particularly pronounced. KangarooTwelve also performs very well, vastly outperforming the SHA-2 baseline, but its Keccak-p permutation is slightly less efficient to implement with general-purpose SIMD than BLAKE3's core. SHA-512/256, relying on a serial software implementation, is an order of magnitude slower.  
-* **ARMv8 Performance:** The performance landscape shifts on the Apple M1 platform, which features dedicated hardware acceleration for both the SHA-2 and SHA-3 families. Here, **KangarooTwelve emerges as the performance leader.** The availability of SHA-3 instructions dramatically accelerates its Keccak-p core, allowing it to edge out the already-fast SIMD implementation of BLAKE3.12 This result highlights a key strategic consideration: K12's performance is intrinsically linked to the presence of these specialized hardware extensions. BLAKE3's performance, while excellent, relies on the universal availability of general-purpose SIMD. The baseline, SHA-512/256, is also significantly more competitive on this platform due to its own hardware acceleration, though it still lags behind the two modern contenders.
-
-## **Strategic Recommendation and Implementation Guidance**
-
-The analysis of algorithmic design, security posture, and quantitative performance data leads to a clear primary recommendation, qualified by important contextual considerations for specific deployment environments.
-
-### **Definitive Recommendation: BLAKE3**
-
-For the primary objective of identifying the single fastest cryptographic hash function for inputs up to 128 bytes, intended as a replacement for SHA-512/256 on a wide range of modern server and desktop hardware, **BLAKE3 is the definitive choice.**
-
-This recommendation is based on the following justifications:
-
-1. **Superior Performance on x86-64:** On the most common server and desktop architecture (x86-64), which largely lacks dedicated hardware acceleration for SHA-512 or SHA-3, BLAKE3's highly optimized SIMD implementation delivers the lowest single-message latency across the entire specified input range.  
-2. **Efficient Core Function:** Its performance advantage stems from a combination of a reduced round count (7 vs. 10 in BLAKE2s) and an ARX-based compression function that is exceptionally well-suited to modern CPU pipelines and SIMD execution.11  
-3. **Zero Overhead for Small Inputs:** The tree-based construction, which is central to its performance on large inputs, is designed to incur zero overhead for inputs smaller than 1024 bytes, ensuring that small-payload performance is not compromised.39  
-4. **Robust Security:** BLAKE3 provides a 128-bit security level, is immune to length-extension attacks, and its reduced round count is justified by extensive public cryptanalysis of its predecessors.33
-
-### **Contextual Considerations and Alternative Scenarios**
-
-While BLAKE3 is the best general-purpose choice, specific deployment targets or workload characteristics may favor an alternative.
-
-* **Scenario A: ARM-Dominant or Future-Proofed Environments.** If the target deployment environment consists exclusively of modern ARMv8.2+ processors that include the optional SHA-3 cryptography extensions (e.g., Apple Silicon-based systems), or if the primary goal is to future-proof an application against the broader adoption of these instructions, **KangarooTwelve is an exceptionally strong and likely faster alternative.** Its ability to leverage dedicated hardware gives it a performance edge in these specific environments.12  
-* **Scenario B: High-Throughput Batch Processing.** If the specific workload involves hashing millions of independent small messages in parallel on a single core, the recommendation becomes more nuanced. As demonstrated by the Solana use case, the simpler scheduling of the Merkle-Damgård construction can allow a highly optimized, multi-message implementation of **hardware-accelerated SHA-256** to achieve higher aggregate throughput.42 In this specialized scenario, the single-message latency advantage of BLAKE3 may not translate to a throughput advantage, and direct, workload-specific benchmarking is essential.  
-* **Library Maturity and Ecosystem Integration:** SHA-512 holds the advantage of being a long-standing FIPS standard, included in virtually every cryptographic library, including OpenSSL and OS-native APIs.38 BLAKE3 has mature, highly optimized official implementations in Rust and C, and is gaining widespread adoption, but may not be present in older, legacy systems.9 KangarooTwelve is the least common of the three, though stable implementations are available from its designers and in libraries like PyCryptodome.24
-
-### **Implementation Best Practices**
-
-To successfully deploy a new hash function and realize its performance benefits, the following practices are recommended:
-
-* **Use Official, Optimized Libraries:** The performance gains of modern algorithms like BLAKE3 are contingent on using implementations that correctly leverage hardware features. It is critical to use the official blake3 Rust crate or the C implementation, which include runtime CPU feature detection to automatically enable the fastest available SIMD instruction set (e.g., SSE2, AVX2, AVX-512).9 Using a generic or unoptimized implementation will fail to deliver the expected speed.  
-* **Avoid Performance Measurement Pitfalls:** The performance of hashing very small inputs is highly susceptible to measurement error caused by the overhead of the calling language or benchmarking framework. As seen in several community benchmarks, measuring performance from a high-level interpreted language like Python can lead to misleading results where the function call overhead dominates the actual hashing time.39 Meaningful benchmarks must be conducted in a compiled language (C, C++, Rust) to accurately measure the algorithm itself.  
-* **Final Verification:** Before committing to a production deployment, the final step should always be to benchmark the top candidates (BLAKE3, and potentially KangarooTwelve or hardware-accelerated SHA-256 depending on the context) directly within the target application and on the target hardware. This is the only way to definitively confirm that the theoretical and micro-benchmark advantages translate to tangible, real-world performance improvements for the specific use case.
-
-#### **Works cited**
-
-1. Hashing and Validation of SHA-512 in Python Implementation \- MojoAuth, accessed September 12, 2025, [https://mojoauth.com/hashing/sha-512-in-python/](https://mojoauth.com/hashing/sha-512-in-python/)  
-2. SHA-512 vs Jenkins hash function \- SSOJet, accessed September 12, 2025, [https://ssojet.com/compare-hashing-algorithms/sha-512-vs-jenkins-hash-function/](https://ssojet.com/compare-hashing-algorithms/sha-512-vs-jenkins-hash-function/)  
-3. Hash Functions | CSRC \- NIST Computer Security Resource Center \- National Institute of Standards and Technology, accessed September 12, 2025, [https://csrc.nist.gov/projects/hash-functions](https://csrc.nist.gov/projects/hash-functions)  
-4. SHA-512 vs BLAKE3 \- A Comprehensive Comparison \- MojoAuth, accessed September 12, 2025, [https://mojoauth.com/compare-hashing-algorithms/sha-512-vs-blake3/](https://mojoauth.com/compare-hashing-algorithms/sha-512-vs-blake3/)  
-5. SHA-512 vs BLAKE3 \- SSOJet, accessed September 12, 2025, [https://ssojet.com/compare-hashing-algorithms/sha-512-vs-blake3/](https://ssojet.com/compare-hashing-algorithms/sha-512-vs-blake3/)  
-6. Did you compare performance to SHA512? Despite being a theoretically more secure... | Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=12176915](https://news.ycombinator.com/item?id=12176915)  
-7. SHA-512 faster than SHA-256? \- Cryptography Stack Exchange, accessed September 12, 2025, [https://crypto.stackexchange.com/questions/26336/sha-512-faster-than-sha-256](https://crypto.stackexchange.com/questions/26336/sha-512-faster-than-sha-256)  
-8. If you're familiar with SHA-256 and this is your first encounter with SHA-3 \- Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=33281278](https://news.ycombinator.com/item?id=33281278)  
-9. the official Rust and C implementations of the BLAKE3 cryptographic hash function \- GitHub, accessed September 12, 2025, [https://github.com/BLAKE3-team/BLAKE3](https://github.com/BLAKE3-team/BLAKE3)  
-10. The BLAKE3 Hashing Framework \- IETF, accessed September 12, 2025, [https://www.ietf.org/archive/id/draft-aumasson-blake3-00.html](https://www.ietf.org/archive/id/draft-aumasson-blake3-00.html)  
-11. BLAKE3 \- GitHub, accessed September 12, 2025, [https://raw.githubusercontent.com/BLAKE3-team/BLAKE3-specs/master/blake3.pdf](https://raw.githubusercontent.com/BLAKE3-team/BLAKE3-specs/master/blake3.pdf)  
-12. KangarooTwelve: fast hashing based on Keccak-p, accessed September 12, 2025, [https://keccak.team/kangarootwelve.html](https://keccak.team/kangarootwelve.html)  
-13. SHA-3 \- Wikipedia, accessed September 12, 2025, [https://en.wikipedia.org/wiki/SHA-3](https://en.wikipedia.org/wiki/SHA-3)  
-14. KangarooTwelve: fast hashing based on Keccak-p, accessed September 12, 2025, [https://keccak.team/2016/kangarootwelve.html](https://keccak.team/2016/kangarootwelve.html)  
-15. xxHash \- Extremely fast non-cryptographic hash algorithm, accessed September 12, 2025, [https://xxhash.com/](https://xxhash.com/)  
-16. SHA-256 vs xxHash \- SSOJet, accessed September 12, 2025, [https://ssojet.com/compare-hashing-algorithms/sha-256-vs-xxhash/](https://ssojet.com/compare-hashing-algorithms/sha-256-vs-xxhash/)  
-17. Benchmarks \- xxHash, accessed September 12, 2025, [https://xxhash.com/doc/v0.8.3/index.html](https://xxhash.com/doc/v0.8.3/index.html)  
-18. Meow Hash \- ASecuritySite.com, accessed September 12, 2025, [https://asecuritysite.com/hash/meow](https://asecuritysite.com/hash/meow)  
-19. Cryptanalysis of Meow Hash | Content \- Content | Some thoughts, accessed September 12, 2025, [https://peter.website/meow-hash-cryptanalysis](https://peter.website/meow-hash-cryptanalysis)  
-20. cmuratori/meow\_hash: Official version of the Meow hash, an extremely fast level 1 hash \- GitHub, accessed September 12, 2025, [https://github.com/cmuratori/meow\_hash](https://github.com/cmuratori/meow_hash)  
-21. (PDF) A Comparative Study Between Merkle-Damgard And Other Alternative Hashes Construction \- ResearchGate, accessed September 12, 2025, [https://www.researchgate.net/publication/359190983\_A\_Comparative\_Study\_Between\_Merkle-Damgard\_And\_Other\_Alternative\_Hashes\_Construction](https://www.researchgate.net/publication/359190983_A_Comparative_Study_Between_Merkle-Damgard_And_Other_Alternative_Hashes_Construction)  
-22. Merkle-Damgård Construction Method and Alternatives: A Review \- ResearchGate, accessed September 12, 2025, [https://www.researchgate.net/publication/322094216\_Merkle-Damgard\_Construction\_Method\_and\_Alternatives\_A\_Review](https://www.researchgate.net/publication/322094216_Merkle-Damgard_Construction_Method_and_Alternatives_A_Review)  
-23. Template:Comparison of SHA functions \- Wikipedia, accessed September 12, 2025, [https://en.wikipedia.org/wiki/Template:Comparison\_of\_SHA\_functions](https://en.wikipedia.org/wiki/Template:Comparison_of_SHA_functions)  
-24. KangarooTwelve — PyCryptodome 3.23.0 documentation, accessed September 12, 2025, [https://pycryptodome.readthedocs.io/en/latest/src/hash/k12.html](https://pycryptodome.readthedocs.io/en/latest/src/hash/k12.html)  
-25. Evaluating the Energy Costs of SHA-256 and SHA-3 (KangarooTwelve) in Resource-Constrained IoT Devices \- MDPI, accessed September 12, 2025, [https://www.mdpi.com/2624-831X/6/3/40](https://www.mdpi.com/2624-831X/6/3/40)  
-26. Cryptographic Hash Functions \- Sign in \- University of Bath, accessed September 12, 2025, [https://purehost.bath.ac.uk/ws/files/309274/HashFunction\_Survey\_FINAL\_221011-1.pdf](https://purehost.bath.ac.uk/ws/files/309274/HashFunction_Survey_FINAL_221011-1.pdf)  
-27. What is Blake3 Algorithm? \- CryptoMinerBros, accessed September 12, 2025, [https://www.cryptominerbros.com/blog/what-is-blake3-algorithm/](https://www.cryptominerbros.com/blog/what-is-blake3-algorithm/)  
-28. The BLAKE3 cryptographic hash function | Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=22003315](https://news.ycombinator.com/item?id=22003315)  
-29. Merkle trees instead of the Sponge or the Merkle-Damgård constructions for the design of cryptorgraphic hash functions \- Cryptography Stack Exchange, accessed September 12, 2025, [https://crypto.stackexchange.com/questions/50974/merkle-trees-instead-of-the-sponge-or-the-merkle-damg%C3%A5rd-constructions-for-the-d](https://crypto.stackexchange.com/questions/50974/merkle-trees-instead-of-the-sponge-or-the-merkle-damg%C3%A5rd-constructions-for-the-d)  
-30. kangarootwelve \- crates.io: Rust Package Registry, accessed September 12, 2025, [https://crates.io/crates/kangarootwelve](https://crates.io/crates/kangarootwelve)  
-31. KangarooTwelve and TurboSHAKE \- IETF, accessed September 12, 2025, [https://www.ietf.org/archive/id/draft-irtf-cfrg-kangarootwelve-12.html](https://www.ietf.org/archive/id/draft-irtf-cfrg-kangarootwelve-12.html)  
-32. minio/sha256-simd: Accelerate SHA256 computations in pure Go using AVX512, SHA Extensions for x86 and ARM64 for ARM. On AVX512 it provides an up to 8x improvement (over 3 GB/s per core). SHA Extensions give a performance boost of close to 4x over native. \- GitHub, accessed September 12, 2025, [https://github.com/minio/sha256-simd](https://github.com/minio/sha256-simd)  
-33. BLAKE3 Is an Extremely Fast, Parallel Cryptographic Hash \- InfoQ, accessed September 12, 2025, [https://www.infoq.com/news/2020/01/blake3-fast-crypto-hash/](https://www.infoq.com/news/2020/01/blake3-fast-crypto-hash/)  
-34. SHA instruction set \- Wikipedia, accessed September 12, 2025, [https://en.wikipedia.org/wiki/SHA\_instruction\_set](https://en.wikipedia.org/wiki/SHA_instruction_set)  
-35. A64 Cryptographic instructions \- Arm Developer, accessed September 12, 2025, [https://developer.arm.com/documentation/100076/0100/A64-Instruction-Set-Reference/A64-Cryptographic-Algorithms/A64-Cryptographic-instructions](https://developer.arm.com/documentation/100076/0100/A64-Instruction-Set-Reference/A64-Cryptographic-Algorithms/A64-Cryptographic-instructions)  
-36. I'm already seeing a lot of discussion both here and over at LWN about which has... | Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=22235960](https://news.ycombinator.com/item?id=22235960)  
-37. Speed comparison from the BLAKE3 authors: https://github.com/BLAKE3-team/BLAKE3/... | Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=22022033](https://news.ycombinator.com/item?id=22022033)  
-38. BLAKE (hash function) \- Wikipedia, accessed September 12, 2025, [https://en.wikipedia.org/wiki/BLAKE\_(hash\_function)](https://en.wikipedia.org/wiki/BLAKE_\(hash_function\))  
-39. Maybe don't use Blake3 on Short Inputs : r/cryptography \- Reddit, accessed September 12, 2025, [https://www.reddit.com/r/cryptography/comments/1989fan/maybe\_dont\_use\_blake3\_on\_short\_inputs/](https://www.reddit.com/r/cryptography/comments/1989fan/maybe_dont_use_blake3_on_short_inputs/)  
-40. SHA-3 proposal BLAKE \- Jean-Philippe Aumasson, accessed September 12, 2025, [https://www.aumasson.jp/blake/](https://www.aumasson.jp/blake/)  
-41. KangarooTwelve \- cryptologie.net, accessed September 12, 2025, [https://www.cryptologie.net/article/393/kangarootwelve/](https://www.cryptologie.net/article/393/kangarootwelve/)  
-42. BLAKE3 slower than SHA-256 for small inputs \- Research \- Solana Developer Forums, accessed September 12, 2025, [https://forum.solana.com/t/blake3-slower-than-sha-256-for-small-inputs/829](https://forum.solana.com/t/blake3-slower-than-sha-256-for-small-inputs/829)  
-43. Blake3 and SHA-3's dead-last performance is a bit surprising to me. Me too \- Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=39020081](https://news.ycombinator.com/item?id=39020081)  
-44. \*\>I'm curious about the statement that SHA-3 is slow; \[...\] I wonder how much re... | Hacker News, accessed September 12, 2025, [https://news.ycombinator.com/item?id=14455282](https://news.ycombinator.com/item?id=14455282)  
-45. draft-irtf-cfrg-kangarootwelve-06 \- IETF Datatracker, accessed September 12, 2025, [https://datatracker.ietf.org/doc/draft-irtf-cfrg-kangarootwelve/06/](https://datatracker.ietf.org/doc/draft-irtf-cfrg-kangarootwelve/06/)  
-46. KangarooTwelve: Fast Hashing Based on $${\\textsc {Keccak}\\text {-}p}{}$$KECCAK-p | Request PDF \- ResearchGate, accessed September 12, 2025, [https://www.researchgate.net/publication/325672839\_KangarooTwelve\_Fast\_Hashing\_Based\_on\_textsc\_Keccaktext\_-pKECCAK-p](https://www.researchgate.net/publication/325672839_KangarooTwelve_Fast_Hashing_Based_on_textsc_Keccaktext_-pKECCAK-p)  
-47. KangarooTwelve \- ASecuritySite.com, accessed September 12, 2025, [https://asecuritysite.com/hash/gokang](https://asecuritysite.com/hash/gokang)  
-48. KangarooTwelve: fast hashing based on Keccak-p, accessed September 12, 2025, [https://keccak.team/files/K12atACNS.pdf](https://keccak.team/files/K12atACNS.pdf)  
-49. TurboSHAKE \- Keccak Team, accessed September 12, 2025, [https://keccak.team/files/TurboSHAKE.pdf](https://keccak.team/files/TurboSHAKE.pdf)  
-50. Why does KangarooTwelve only use 12 rounds? \- Cryptography Stack Exchange, accessed September 12, 2025, [https://crypto.stackexchange.com/questions/46523/why-does-kangarootwelve-only-use-12-rounds](https://crypto.stackexchange.com/questions/46523/why-does-kangarootwelve-only-use-12-rounds)  
-51. What advantages does Keccak/SHA-3 have over BLAKE2? \- Cryptography Stack Exchange, accessed September 12, 2025, [https://crypto.stackexchange.com/questions/31674/what-advantages-does-keccak-sha-3-have-over-blake2](https://crypto.stackexchange.com/questions/31674/what-advantages-does-keccak-sha-3-have-over-blake2)  
-52. Comparison between this and KangarooTwelve and M14 · Issue \#19 · BLAKE3-team/BLAKE3 \- GitHub, accessed September 12, 2025, [https://github.com/BLAKE3-team/BLAKE3/issues/19](https://github.com/BLAKE3-team/BLAKE3/issues/19)  
-53. eBASH: ECRYPT Benchmarking of All Submitted Hashes, accessed September 12, 2025, [https://bench.cr.yp.to/ebash.html](https://bench.cr.yp.to/ebash.html)  
-54. SUPERCOP \- eBACS (ECRYPT Benchmarking of Cryptographic Systems), accessed September 12, 2025, [https://bench.cr.yp.to/supercop.html](https://bench.cr.yp.to/supercop.html)  
-55. XKCP/K12: XKCP-extracted code for KangarooTwelve (K12) \- GitHub, accessed September 12, 2025, [https://github.com/XKCP/K12](https://github.com/XKCP/K12)

gpt5-canvas-scribbles.md

@@ -1,398 +0,0 @@
-#  BLAKE3 Migration
-
----
-
-## ~~Why touch the cryptographic foundation at all?~~
-
-~~Performance isn't an academic detail — it's dramatic. On modern hardware, BLAKE3 runs an order of magnitude faster than SHA-512 or SHA-256. For example:~~
-
-~~In benchmarks, BLAKE3 achieves \~6.8 GiB/s throughput on a single thread, compared to \~0.7 GiB/s for SHA-512. This headroom matters in a ledger system where *every object key is a hash*. Faster hashing reduces CPU load for consensus, verification, and replay. Here, "performance" primarily means faster **keylet** computation (deriving map/index keys from object components) and less compatibility overhead (LUT hits, try‑both‑hashes), **not** improved data locality between neighboring objects.~~
-
-~~Performance and modern cryptographic hygiene argue strongly for adopting BLAKE3. It's fast, parallelizable, and future-proof. But in this ledger system, the hash is not just a digest: it is the address of every object. Changing the hash function means changing the address of every single entry. This isn't like swapping an internal crypto primitive — it's a rekeying of the entire universe.~~
-
-## Reality Check: BLAKE3 vs SHA-512 on ARM64 (Sept 2025)
-
-**TL;DR: BLAKE3 migration complexity isn't justified by the actual performance gains.**
-
-### Measured Performance (Xahau ledger #16940119)
-- **Keylets (22-102 bytes)**: BLAKE3 is 0.68x speed of SHA-512 (47% SLOWER)
-- **Inner nodes (516 bytes)**: BLAKE3 is 0.52x speed of SHA-512 (92% SLOWER)
-- **Map traversal**: 59-65% of total time (not affected by hash choice)
-- **Actual hashing**: Only 35-41% of total time
-
-### Why BLAKE3 Underperforms
-1. **Small inputs**: Median keylet is 35 bytes; SIMD overhead exceeds benefit
-2. **2020 software vs 2025 hardware**: BLAKE3 NEON intrinsics vs OpenSSL 3.3.2's optimized SHA-512
-3. **No parallelism**: Single-threaded SHAMap walks can't use BLAKE3's parallel design
-4. **SIMD dependency**: Without NEON, BLAKE3 portable C is 2x slower than SHA-512
-
-### The Verdict
-With hashing only 35-41% of total time and BLAKE3 actually SLOWER on typical inputs, the migration would:
-- Increase total validation time by ~10-15%
-- Add massive complexity (LUTs, heterogeneous trees, compatibility layers)
-- Risk consensus stability for negative performance gain
-
-**Recommendation: Abandon BLAKE3 migration. Focus on map traversal optimization instead.**
-
-## Hashes vs Indexes
-
-* **Hashes as keys**: Every blob of data in the NodeStore is keyed by a hash of its contents. This makes the hash the *address* for retrieval.
-* **Hashes as indexes**: In a ShaMap (the Merkle tree that represents ledger state), an `index` is derived by hashing stable identity components (like account ID + other static identifiers). This index determines the path through the tree.
-* **Takeaway**: Hash = storage key. Index = map position. Both are 256-bit values, but they play different roles.
-
-*Terminology note*: throughout, **keylet/key** = deterministic map/index key composition from object components; this is unrelated to users’ cryptographic signing keys.
-
-## LUT at a glance
-
-A **Lookup Table (LUT)** is an exact‑key alias map used to bridge old and new addressing:
-
-* **Purpose:** allow lookups by a legacy (old) key to resolve to an object stored under its canonical (new) key — or vice‑versa where strictly necessary.
-* **Scope:** point lookups only (reads/writes by exact key). Iteration and ordering remain **canonical**; pagination via `next` after a marker requires careful handling (semantics TBD)
-* **Population:** built during migration and optionally **rebuildable** from per‑SLE cross‑key fields (e.g., `sfLegacyKey` for move, or `sfBlake3Key` for non‑move).
-* **Directionality in practice:** after the flip you typically need **both directions**, but for different eras:
-
-  * **Pre‑cutover objects (stored at old keys):** maintain **`BLAKE3 → SHA512Half`** so new‑style callers (BLAKE3) can reach old objects.
-  * **Post‑cutover objects (stored at new keys):** optionally offer a grace **`SHA512Half → BLAKE3`** alias so legacy callers can reach new objects. Time‑box this.
-    **Rule of thumb:** annotate the **opposite side of storage** — if storage is **new** (post‑move), annotate **old**; if storage is **old** (non‑move), annotate **new**.
-
-## What actually breaks if you “just change the hash”?!
-
-Every ledger entry’s key changes. That cascades into:
-
-* **State tree**: SHAMap nodes are keyed by hash; every leaf and inner node address moves.
-* **Directories**: owner dirs, book dirs, hook state dirs, NFT pages — all are lists of hashes, all must be rebuilt.
-* **Order and proofs**: Succession, iteration, and proof-of-inclusion semantics all rely on canonical ordering of keys. Mixing old and new hashes destroys proof integrity.
-* **Caches and history**: Node sharing between ledgers ceases to work; replay and verification of past data must know which hash function was active when.
-
-## Lazy vs Big Bang
-
-If you update tree hashes incrementally as state changes, you are effectively doing a **lazy migration**: slowly moving to the new hashing system over time. That implies heterogeneous trees and ongoing complexity. By contrast, a **big bang** migration rekeys everything in a single, well-defined event. Since roughly 50% of hashing compute goes into creating these keys, most of the performance win from BLAKE3 arrives when the generated keys for a given object are used. This can be achieved if the object is **in place at its new key**, **moved within the tree**, or is **reachable via an exact‑key LUT that aliases old→new**.
-
-*Note:* LUT specifics belong in **Move vs Non‑Move** below. At a high level: aliasing can bridge old/new lookups; iteration/pagination semantics are TBD here and treated separately.
-
-### Pros and Cons
-
-**Lazy migration**
-
-* **Pros**: Less disruptive; avoids one massive compute spike; spreads risk over time.
-* **Cons**: Creates heterogeneous trees; complicates proofs and historical verification; requires bidirectional LUTs forever; analysts and tools must support mixed keyspaces.
-
-**Big bang migration**
-
-* **Pros**: Clean cutover at a known ledger; easier for analysts and tooling; no need to support mixed proofs long-term; maximizes BLAKE3 performance benefits immediately.
-* **Cons**: One heavy compute event; requires strict consensus choreography; higher risk if validators drift or fail mid-migration.
-
-It’s important to distinguish between lazy vs big bang, and also between keys (addresses/indexes) vs hashes (content identifiers).
-
-## Move vs Non‑Move (what does “migrate” change?)
-
-**Non‑Move (annotate‑only):** objects stay at old SHA512‑Half keys; add `sfBlake3Key` (or similar) recording the would‑be BLAKE3 address; alias lookups via **new→old** LUT; iteration/proofs remain in old key order; minimal compute now, **permanent heterogeneity** and LUT dependence; little perf/ordering win.
-
-**Move (rekey):** objects are physically rewritten under BLAKE3 keys either **on‑touch** (per‑tx or at **BuildLedger** end) or **all at once** (Big‑Bang). Requires **old→new** LUT for compatibility; choose a place/time (per‑tx vs BuildLedger vs Big‑Bang) and define iteration contract (prefer canonical‑only).
-
-**Implications to weigh:**
-
-* **LUT shape:** non‑move needs **new→old** (often also old→new for markers); move prefers **old→new** (temporary). Sunsetting is only realistic in the Big‑Bang case; lazy variants may never fully converge.
-* **Iteration/pagination:** canonical‑only iteration keeps proofs stable; translating legacy markers implies **bi‑LUT** and more hot‑path complexity.
-* **Replay:** both need `hash_options{rules(), ledger_index, phase}`; move policies must be consensus‑deterministic.
-* **Compute/ops:** non‑move is cheap now but never converges; move concentrates work (per‑tx, per‑ledger, or one Big‑Bang) and actually delivers BLAKE3’s **iteration/ordering** and **keylet‑compute** benefits (not data‑locality).
-
-### Choice axes (what / when / how)
-
-* **What:** *Move* the object under BLAKE3 **or** *leave in place* and annotate (`sfBlake3Key`).
-* **When:** at **end of txn** or in **BuildLedger** (alongside `updateNegativeUNL()` / `updateSkipList()`), or **all at once** (Big‑Bang).
-* **How:** *All at once* requires special network conditions (quiet window + consensus hash); *on modification* spreads risk but prolongs heterogeneity.
-* **Blob verification note:** a dual‑hash “verify on link” walk works for mixed trees, but you need the same `rules()+phase` plumbing either way, so it doesn’t materially change the engineering lift.
-
-### Client compatibility & new entries
-
-* **Reality:** flipping keylets changes what clients compute. Old clients may still derive SHA512‑Half; new clients may derive BLAKE3.
-* **Lazy non‑move (annotate‑only):**
-
-  * **Reads/updates:** accept BLAKE3 via **new→old LUT**; legacy SHA512 keys keep working.
-  * **Creates (policy choice):**
-
-    * **Create‑at‑new (heterogeneous by design):** store under **BLAKE3** (the natural post‑flip behavior). For **legacy callers**, provide a grace alias **`SHA512Half → BLAKE3`** for *new* entries; stamp `sfLegacyKey` (old) on creation so the alias can be rebuilt by a leaf scan.
-    * *Create‑at‑old (alternative until swap):* store under **old** to keep the map homogeneous; if request included a BLAKE3 key, treat it as a descriptor and translate. *Optional annotation:* add `sfBlake3Key` (new) to make later `new→old` LUT rebuild trivial. *(In the ********move********/post‑swap case, annotate the opposite side: ******`sfLegacyKey`****** = old.)*
-    * *Create‑via‑old‑only:* require old keys for creates until swap (simpler server), and document it for SDKs.
-    * *(Note:)* a LUT alone can’t route a brand‑new create — there’s no mapping yet — so the server must compute the storage key from identity (old or new, per the policy) and record the opposite‑side annotation for future aliasing.
-* **Big‑Bang (move):** creates immediately use **BLAKE3** as canonical; provide **`SHA512Half → BLAKE3`** grace alias for new objects; **old→new** LUT supports stragglers reading old objects by legacy keys.
-* **Bottom line:** you still need **`rules()`**\*\* + phase\*\* plumbing and an explicit **create policy**; don’t pick a strategy based purely on “less plumbing”.
-
-### Post‑cutover lookup policy (directional LUT by era)
-
-* **Old objects (pre‑cutover, stored at old keys):** new‑style callers use **BLAKE3** keys → resolve via **`BLAKE3 → SHA512Half`** (keep as long as needed; deprecate when safe).
-* **New objects (post‑cutover, stored at new keys):** legacy callers may supply **SHA512‑Half** → resolve via **`SHA512Half → BLAKE3`** *during a grace window*; plan a TTL/deprecation for this path.
-* **Iteration/pagination:** always return the **canonical storage key** of the era (old for old objects, new for new objects). Document that markers are era‑canonical; aliases are for **point lookups** only.
-
-### Lazy non‑move: LUT requirements (immediate and ongoing)
-
-* If keylets emit **BLAKE3** keys before a physical swap, you must have a **complete ************`new→old`************ LUT** available at flip time. A cold‑start empty LUT will cause immediate misses because objects still live at old addresses.
-* The LUT must be **built during a quiet window** by walking the full state and computing BLAKE3 addresses; you cannot populate it “on demand” without global scans.
-* **Persist the LUT**: typically a sidecar DB keyed by `BLAKE3 → SHA512Half`, or rely on per‑SLE **new‑side annotation** (`sfBlake3Key`) so any node can rebuild the LUT deterministically by a leaf scan. `sfBlake3Key` helps you rebuild; it does **not** remove the need for a ready‑to‑query LUT at flip.
-* Expect to **carry the LUT indefinitely** in non‑move. Its hit‑rate may drop over time only if you later migrate objects (or switch to Big‑Bang).
-
-## Heterogeneous vs Homogeneous state trees
-
-**Homogeneous** means a single canonical keyspace and ordering (one hash algorithm across the whole state tree). **Heterogeneous** means mixed keys/hashes coexisting (some SHA512‑Half, some BLAKE3), even if reads are made to “work.”
-
-**Why this matters**
-
-* **Proofs & ordering**: Homogeneous trees keep proofs simple and iteration stable. Heterogeneous trees complicate inclusion proofs and `succ()`/pagination semantics.
-* **Read path**: With heterogeneity, you either guess (dual‑hash walk), add **hints** (local "unused" nodestore bytes), or introduce **new prefixes** (network‑visible). All add complexity.
-* **Replay & determinism**: Homogeneous trees let `rules()`+`ledger_index` fully determine hashing. Heterogeneous trees force policy state (when/where items moved) to be consensus‑deterministic and reproduced in replay.
-* **Caches & sharing**: Node sharing across ledgers is cleaner in a homogeneous regime; heterogeneity reduces reuse and increases compute.
-* **Operational risk**: Mixed eras inflate your attack and bug surface (LUT correctness, marker translation, proof ambiguity).
-
-**How you end up heterogeneous**
-
-* Lazy hashing or “annotate‑only” lazy keys (non‑move).
-* Staged moves (on‑touch) that never reach full coverage.
-* Introducing new prefixes and treating both spaces as first‑class for long periods.
-
-**How to avoid it**
-
-* **Big‑Bang** swap in `BuildLedger`, then canonical‑only iteration under BLAKE3.
-* Keep a narrow **old→new** LUT as a safety net (rebuildable from `sfLegacyKey`), and plan deprecation.
-
-**If you must tolerate heterogeneity (temporarily)**
-
-* Use **context‑bound hashing** (`hash_options{rules(), ledger_index, phase, classifier}`) everywhere.
-* Consider **local hint bytes** or **prefixes** only to remove guesswork; define a strict marker policy (normalize to canonical outputs) and accept perf overhead.
-
-## Options matrix — migration + keylet policies
-
-### 1) Migration strategy (what physically moves when)
-
-| Strategy                                                                                                                                                                                                                                         | What moves & when                                                                | Tree heterogeneity | LUT needs                                                                                          | Iteration / pagination                                                                                 | Replay & hashing context                                                    | Operational risk                                   | Pros                                                               | Cons                                                                               |
-| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------- | -------------------------------------------------- | ------------------------------------------------------------------ | ---------------------------------------------------------------------------------- |
-| **Big‑Bang (swap in one ledger, in BuildLedger)**                                                                                                                                                                                                | All SLEs rekeyed in a single, quiet, consensus‑gated ledger; stamp `sfLegacyKey` | None after swap    | **old→new** only (temporary; rebuildable from `sfLegacyKey`)                                       | Immediately canonical under BLAKE3; simple markers                                                     | Straightforward (`rules()`, `ledger_index`, `phase` flip once)              | One heavy compute event; needs strict choreography | Clean proofs & ordering; simplest for tools; fast path to perf win | Requires quiet period + consensus hash; “all‑eggs‑one‑basket” ledger               |
-| **Lazy keys — moved, per‑tx**                                                                                                                                                                                                                    | Touched SLEs are **moved** to BLAKE3 keys during tx commit                       | Long‑lived         | **old→new** and often **new→old** (for markers)                                                    | Mixed keys; must normalize or translate; highest complexity                                            | Hardest: movement timing is per‑tx; requires full `hash_options` everywhere | Low per‑ledger spike, but constant complexity      | Spreads compute over time                                          | Permanent heterogeneity; iterator/marker headaches; error‑prone                    |
-| **Lazy keys — *********************************************************************************************not********************************************************************************************* moved, per‑tx (annotate only)**      | No SLEs move; touched entries get `sfBlake3Key` / annotation only                | Permanent          | **new→old** (lookups by BLAKE3 must alias to old), often also **old→new** if you normalize outputs | Iteration remains in **old** key order unless you add translation; markers inconsistent without bi‑LUT | Hard: you never converge; replay must honor historic “no‑move” semantics    | Low per‑ledger spike                               | Zero relocation churn; simplest writes                             | You never get canonical BLAKE3 ordering/proofs; LUT forever; limited perf win      |
-| **Lazy keys — moved, BuildLedger**                                                                                                                                                                                                               | Touched SLEs are **moved** at end of ledger in BuildLedger                       | Medium‑lived       | **old→new** (likely) and sometimes **new→old** (if you want legacy markers to resume cleanly)      | Still mixed; easier to normalize to canonical at ledger boundary                                       | Moderate: movement is per‑ledger; still need `hash_options`                 | Lower spike than Big‑Bang; simpler than per‑tx     | Centralized move step; cleaner tx metadata                         | Still heterogeneous until coverage is high; LUT on hot paths                       |
-| **Lazy keys — *********************************************************************************************not********************************************************************************************* moved, BuildLedger (annotate only)** | No SLEs move; annotate touched entries in BuildLedger only                       | Permanent          | **new→old** (and possibly **old→new** if you normalize)                                            | Iteration stays in **old** order; translation needed for consistency                                   | Moderate: policy is per‑ledger but never converges                          | Lowest spike                                       | Cleanest ops; no relocation diffs                                  | Same drawbacks as per‑tx annotate‑only: permanent heterogeneity and LUT dependence |
-
-**Notes:**
-
-* Prefer **canonical‑only iteration** (return new keys) and accept legacy markers as input → reduces need for bidirectional LUT.
-* If you insist on round‑tripping legacy markers, you’ll need **bi‑directional LUT** and iterator translation.
-* For **annotate‑only (non‑move)** variants: if you choose **Policy C (flip globally at ledger n)**, you **must** prebuild a complete `new→old` LUT for the entire tree before the flip. To avoid this empty‑LUT hazard, choose **Policy A (flip at swap)** until the physical move occurs.
-
-#### 1a) Big‑Bang — non‑move (alias‑only) at a glance
-
-| What moves & when                                                                                                                                                                              | Tree heterogeneity                                                                                                                                              | LUT needs                                                                                                | Iteration/pagination                               | Pros                                                                                                                 | Cons                                                                                                                                                          |
-| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **No storage move at cutover; global keylet flip; annotate all SLEs with ************`sfBlake3Key`************; full ************`new→old`************ LUT ready or rebuildable by leaf scan** | Ledger map: **old** for legacy, **new** for new objects; NodeStore blobs: **full‑tree rewrite** (choose a single blob‑hash algo post‑cutover to avoid guessing) | Permanent `new→old`; **rebuildable from ************`sfBlake3Key`************ by optimized leaf parser** | Old‑order; document marker policy (no translation) | No **map index** relocation; flip is clean; **LUT always accessible**; rollback = behavior flip only if LUT retained | Proofs/ordering stay old; permanent LUT; **one‑time I/O spike** from full‑tree rewrite (mitigated by preflushing background tree); no homogeneous BLAKE3 tree |
-
-### 2) Keylet flip policy (what keylets *emit*) (what keylets *emit*) (what keylets *emit*) (what keylets *emit*) (what keylets *emit*) (what keylets *emit*)
-
-| Policy                              | What keylets return                     | Empty‑LUT risk        | Need global LUT upfront?        | Client‑visible behavior                 | Pros                                   | Cons                                                       |
-| ----------------------------------- | --------------------------------------- | --------------------- | ------------------------------- | --------------------------------------- | -------------------------------------- | ---------------------------------------------------------- |
-| **A. Flip at swap only**            | Old keys pre‑swap; new keys post‑swap   | None                  | No                              | Single flip; stable semantics           | Simplest; no prep LUT window           | Requires Big‑Bang or near‑equivalent swap moment           |
-| **B. Flip per‑SLE (when migrated)** | New for migrated entries; old otherwise | None                  | No                              | Mixed outputs; must normalize iteration | No global LUT build; smoother ramp     | Clients see mixture unless normalized; still heterogeneous |
-| **C. Flip globally at ledger n**    | New everywhere from n                   | **High** if LUT empty | **Yes** (build in quiet period) | Clean switch for clients                | Global behavior is uniform immediately | Must precompute `new→old` LUT; higher prep complexity      |
-
-### 3) Hashing decision representation (perf & memory)
-
-| Option                                             | What changes                                                                                                                                 | Memory/Perf impact                            | ABI impact      | Benefit                                            |
-| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------- | --------------- | -------------------------------------------------- |
-| **0. Context‑bound keylets (recommended default)** | Keep returning 32‑byte keys; keylets choose SHA512‑Half vs BLAKE3 using a small `HashCtx` (`rules()`, `ledger_index`, `phase`, `classifier`) | Tiny branch; no heap; cache optional per‑View | None            | Avoids empty‑LUT trap; simplest to roll out        |
-| **1. Thin symbolic descriptors (stack‑only)**      | Keylets can return a small descriptor; callers `resolve(desc, ctx)` immediately                                                              | Minimal; POD structs; optional tiny cache     | None externally | Centralizes decision; testable; still light‑weight |
-| **2. Full symbolic (iterators/markers only)**      | Iterators carry `{desc, resolved}` to re‑resolve under different contexts                                                                    | Small per‑iterator cache                      | None externally | Makes pagination/replay robust without broad churn |
-
-### 4) NodeStore hinting for heterogeneous reads (only if you *must* support mixed trees)
-
-| Approach                                    | Scope           | Pros                                               | Cons                                                                   |
-| ------------------------------------------- | --------------- | -------------------------------------------------- | ---------------------------------------------------------------------- |
-| **No hints (dual‑hash walk)**               | Network‑safe    | Simple to reason about; no store changes           | Costly: try‑both‑hashes while walking; awkward                         |
-| **Local hint bytes (use 8–9 unused bytes)** | Local only      | Eliminates guesswork on a node; cheap to implement | Not portable; doesn’t show up in proofs; still need amendment plumbing |
-| **New hash prefixes in blobs**              | Network‑visible | Clear namespace separation; easier debugging       | Prefix explosion; code churn; proof/back‑compat complexity             |
-
-### 5) Recommended defaults
-
-* **Migration**: Big‑Bang in `BuildLedger` with quiet period + consensus hash; stamp `sfLegacyKey`.
-* **Keylets**: Policy **A** (flip at swap) or **B** if you insist on staging; normalize iteration to canonical.
-* **LUT**: **old→new** exact‑key alias as a temporary safety net; rebuildable from `sfLegacyKey`.
-* **Hashing decision**: **Option 0 (context‑bound keylets)**; reserve symbolics for iterators only if needed.
-
-## Heterogeneous trees and possible NodeStore tweaks
-
-When loading from the NodeStore with a root hash, in principle you could walk down the tree and try hashing each blob’s contents to check whether it matches. At each link, you verify the blob by recomputing its hash. In theory you could even try both SHA-512 Half and BLAKE3 until the structure links up. This would eventually work, but it is inefficient.
-
-To avoid that inefficiency, one idea is to tweak the NodeStore blobs themselves. There are 8–9 unused bytes (currently stored as zeros) that could be repurposed as a hint. Another option is to change the stored hash prefixes, which would act as explicit namespace markers separating SHA-512 and BLAKE3 content. With the ledger index also available, heuristics could guide which algorithm to use. But none of this removes the need for amendment plumbing — you still have to know if the cutover has occurred.
-
-### Versioned prefixes (use the spare byte)
-
-**Goal:** eliminate guessing in mixed/historical contexts by making the blob self‑describing.
-
-* **Design:** keep the 3‑letter class tag and use the 4th byte as an **algorithm version**.
-
-```cpp
-enum class HashPrefix : std::uint32_t {
-  innerNode_v0 = detail::make_hash_prefix('M','I','N', 0x00), // SHA512Half
-  innerNode_v1 = detail::make_hash_prefix('M','I','N', 0x01), // BLAKE3
-  leafNode_v0  = detail::make_hash_prefix('M','L','N', 0x00),
-  leafNode_v1  = detail::make_hash_prefix('M','L','N', 0x01),
-  // add tx/dir variants only if their blob hashing changes too
-};
-```
-
-* **Read path:** fetch by hash as usual; after you read the blob, the prefix **discriminates** the hashing algorithm used to produce that key. No dual‑hash trial needed to verify/link.
-* **Write path:** when (re)serializing a node, choose the version byte from `hash_options.rules()/phase`; parent/child content stays consistent because each node carries its own version.
-* **Pros:** zero‑guess verification; offline tools can parse blobs without external context; makes mixed eras debuggable.
-* **Cons:** network‑visible change (new prefixes); code churn where prefixes are assumed fixed; doesn’t solve keylet/index aliasing or iteration semantics — it only removes blob‑hash guessing.
-
-**Note:** you can also avoid guessing entirely by keeping **one blob‑hash algorithm per ledger** (homogeneous per‑ledger eras). Then `rules()+ledger_index` suffices. Versioned prefixes mainly help offline tools and any design that tolerates intra‑ledger mixing.
-
-### Lazy migration headaches
-
-If you attempt a lazy migration, you must decide how keys are rehashed. Is it done during metadata creation at the end of transactions? Do you rely on a LUT to map between new and old indexes? If so, where is this LUT state stored? Another idea is to embed a `LedgerIndexBlake3` in entries, so that keylet helpers can create new indexes while CRUD operations translate through a LUT. But this complicates pagination markers and functions like `ReadView::succ()` that return natural keys. You risk situations where the system must be aware of multiple keys per entry.
-
-Questions like pagination markers and `ReadView::succ()` make this even thornier. One approach might be to encode the hash type in the LUT, and maintain it bidirectionally, so when iteration returns a canonical key it can be translated back to the old form if needed. But this doubles the complexity and still forces every path to be LUT‑aware.
-
-By contrast, in the **Big Bang** version the LUT is just a safety net, handling things that could not be automatically rewritten. This is simpler for analysts and avoids perpetual cross-key complexity.
-
-### Why it feels like a headache
-
-Trying to lazily migrate keys means constantly juggling questions:
-
-* Do you move items immediately when the amendment is enabled, or only on first touch?
-* If you move them, when exactly: during metadata creation, during BuildLedger along with the SkipList?
-* How do you keep CRUD ops working while also updating LUT state?
-* How do you handle pagination markers and `succ()` consistently if multiple keys exist? You would need bidirectional.
-
-Every option adds complexity, requires bidirectional LUTs, and forces awareness of dual keyspaces everywhere. This is why the lazy path feels like a perpetual headache, while the Big Bang keeps the pain contained to one well‑known cutover.
-
-## The Big Bang
-
-From here onward, we focus on the **Big‑Bang** approach (one‑ledger atomic rekey). Lazy/staged variants are summarized above.
-
-### Why Big‑Bang is preferred here
-
-* **Homogeneous immediately:** one canonical keyspace the very next ledger → simple proofs, stable iteration/pagination, no dual‑key semantics.
-* **No empty‑LUT window:** keylets flip at the swap; the LUT is **old→new** only, narrow in scope, and realistically deprecable.
-* **Deterministic & replay‑friendly:** a single, well‑known cutover ledger anchors tooling and historical verification.
-* **Operationally contained risk:** compute is concentrated into the quiet window with explicit consensus checkpoints (single or double), not smeared across months.
-* **Cleaner dev/ops surface:** fewer code paths need LUT/translation logic; easier to reason about `succ()`/markers and caches.
-
-### Variant: Big‑Bang “non‑move” (alias‑only swap)
-
-**What it is:** at the cutover ledger, **annotate the entire state tree** by stamping every SLE with its BLAKE3 address (e.g., `sfBlake3Key`). **Do not** rewrite storage keys. During the quiet window, prebuild a complete `new→old` LUT **or** rely on the new field so any node can rebuild the LUT deterministically by scanning leaves with an optimized parser. Flip keylets to emit BLAKE3. Optionally commit a small on‑ledger **annotation/LUT commitment hash** in `MigrationState` so operators can verify their sidecar.
-
-**How it behaves:** point lookups by BLAKE3 resolve via the LUT; writes/erases resolve to the canonical **old** storage key before touching disk; **new objects** are stored under **BLAKE3** keys (post‑flip); legacy callers may be served by a grace **`SHA512Half → BLAKE3`** alias for *new* objects. Iteration/pagination remain in the old order for legacy entries (document marker policy).
-
-**I/O reality & mitigation:**
-
-* Annotating every leaf **changes its bytes**, forcing a **full‑tree NodeStore rewrite** (leaf blob hashes change; inner nodes update). This is a **mass write**, even though map indexes don’t relocate.
-* Mitigate the spike by **streaming/staged flush** of the staging tree during BuildLedger (chunked passes), back‑pressure on caches, and rate‑limited node writes; total bytes remain \~“rewrite the tree once.”
-
-**LUT reconstruction paths:**
-
-* **From annotation (fastest):** for each leaf, read `sfBlake3Key` and the current (old) key; record `BLAKE3 → old`.
-* **From recompute (belt‑and‑suspenders):** recompute BLAKE3 via keylet helpers from identity components and pair with the observed old key.
-
-**Pros:** no **map index** relocation for legacy entries; minimal end‑user surprise; clean flip semantics; **LUT always reconstructible** from the annotated tree; **rollback is behavioral‑only if the LUT is retained**.
-
-**Cons:** ordering/proofs remain old indefinitely; LUT becomes permanent; you forgo a homogeneous BLAKE3 tree and its simplifications; **full‑tree NodeStore rewrite** (leaf annotation changes bytes → new blob hashes → inner nodes update) causing a one‑time I/O spike.
-
-**Rollback reality:** Once clients rely on BLAKE3 keys on the wire, a “rollback” without a LUT breaks them. Practical rollback means flipping keylet behavior back to SHA512‑Half **and** continuing to serve BLAKE3 lookups via the LUT indefinitely (or performing a reverse‑migration). In other words, rollback is only “easy” if you accept a **permanent LUT**.
-
-**When to pick:** you want Big‑Bang’s clean flip and operational containment, but can’t (or don’t want to) rewrite the entire state tree; you still want a deterministic, cheap way to rebuild the LUT by scanning.
-
-### How to message this (without scaring users)
-
-**Elevator pitch**
-
-> We’re flipping key derivation to BLAKE3 for *new* addresses, but we’re **not relocating existing entries**. We annotate the tree in a maintenance window, so old data stays where it is, new data goes to BLAKE3 addresses, and both key forms work via an alias. Transactions, TxIDs, and signatures don’t change.
-
-**What users/operators should expect**
-
-* **No surprise breakage:** Old clients that synthesize SHA512‑Half keys still read old objects; new clients can use BLAKE3 keys everywhere (old objects resolve via alias).
-* **New vs old objects:** Old objects remain at their old locations; **new objects** are stored at **BLAKE3** locations. A **grace alias** can accept SHA512‑Half for *new* objects for a limited time.
-* **Ordering/proofs unchanged for old entries:** Iteration order and proofs remain canonical‑old for legacy entries. No bidirectional iteration translation.
-* **TxIDs & signing stay the same:** Transaction IDs and signing digests are **unchanged**; do **not** hand‑derive ledger indexes—use keylet APIs.
-* **One‑time write spike (planned):** Annotating every leaf causes a **single full‑tree blob rewrite** during the quiet window; we stage/stream this as part of `BuildLedger`.
-
-**Soundbite**
-
-> *“Not a scary rekey-everything rewrite.”* It’s a one‑time annotation and an API flip: old stays reachable, new is faster, and we give legacy callers a grace window.
-
-### Decision & next steps (short list)
-
-1. **Amendment & timing:** finalize `featureBlake3Migration`, `MIGRATION_TIME`, and quiet‑period length.
-2. **BuildLedger swap/annotate pass:** implement two‑pass **rekey** (plan → commit), **or** two‑pass **annotate** (stamp `sfBlake3Key` on all SLEs). For rekey, stamp `sfLegacyKey` and materialize **old→new** LUT; for non‑move, stamp `sfBlake3Key` and materialize **new→old** LUT (both rebuildable by leaf scan).
-3. **API rules:** reads/writes = canonical‑first, LUT‑on‑miss (point lookups only); **iteration is canonical‑only**; document marker semantics.
-4. **Hash context plumbing:** ensure `hash_options{rules(), ledger_index, phase, classifier}` are available down to `SHAMap::getHash()` and relevant callers.
-5. **Consensus choreography:** pick **single** vs **double** hash checkpoint; wire pseudo‑tx for the pre‑hash if using two‑step.
-6. **Telemetry & deprecation:** ship metrics for LUT hit‑rate and schedule a sunset once hits are negligible.
-7. **Test plan:** simulate slow validators, partial LUT rebuilds, replay across the swap, and hook workloads with hardcoded keys.
-
-## Governance first: permission to cut over
-
-Such a migration cannot be unilateral. An amendment (`featureBlake3Migration`) acts as the governance switch, enabling the network to authorize the cutover. This amendment does not itself rekey the world, but it declares consensus intent: from a certain point, ledgers may be rebuilt under the new rules.
-
-A pseudo-transaction (e.g. `ttHASH_MIGRATION`) provides the on-ledger coordination. It marks the trigger point, updates the migration state SLE, and ensures every validator knows exactly *when* and *what* to execute.
-
-## Why not just do it in the pseudo-transaction?
-
-A naive attempt to treat the entire migration as a simple pseudo-transaction — a one-off entry applied like any other — would explode into metadata churn, duplicate entries, and lost referential integrity. The scale of rekeying every SLE makes it unsuitable for a normal transaction context; it has to run in a special execution venue like `BuildLedger` to remain atomic and manageable.
-
-## Choose the battlefield: BuildLedger
-
-The right place to run the migration is inside `BuildLedger` — after applying the (quiet) transaction set, and before finalization. This avoids flooding transaction metadata with millions of deletes and creates, and guarantees atomicity: one ledger before = SHA-512 Half; one ledger after = BLAKE3.
-
-This is also exactly where other ledger-maintenance updates happen: for example `updateNegativeUNL()` runs when processing a flag ledger if the feature is enabled, and `updateSkipList()` is invoked just before flushing SHAMap nodes to the NodeStore. By piggybacking the migration here, it integrates cleanly into the existing lifecycle:
-
-```cpp
-if (built->isFlagLedger() && built->rules().enabled(featureNegativeUNL))
-{
-    built->updateNegativeUNL();
-}
-
-OpenView accum(&*built);
-applyTxs(accum, built);
-accum.apply(*built);
-
-built->updateSkipList();
-
-// Flush modified SHAMap nodes to NodeStore
-built->stateMap().flushDirty(hotACCOUNT_NODE);
-built->txMap().flushDirty(hotTRANSACTION_NODE);
-built->unshare();
-```
-
-By inserting the BLAKE3 migration pass into this sequence, it runs atomically alongside the skip list and NegativeUNL updates, ensuring the new canonical tree is finalized consistently. 
-
-## Hashing and consensus choreography
-
-It may make sense to stretch the choreography into more than one consensus checkpoint, especially given the amount of compute involved. A possible flow:
-
-* **Quiet period** — block transactions so everyone is aligned.
-* **Phase 1: Hash the static tree** — compute a BLAKE3 hash of the ledger state, excluding churny structures like skip lists and the migration state.
-* **Consensus** — validators agree on this static-hash checkpoint.
-* **Phase 2: Hash the full tree** — compute the full state tree hash under BLAKE3.
-* **Consensus** — converge again on the complete view.
-* **Atomic swap** — only after both steps succeed, rewrite the ledger under new keys.
-
-This extra step could make it easier for validators to stay in sync without network drift, because they checkpoint on a smaller, stable hash before tackling the full-tree rebuild. It reduces wasted compute if things diverge. The downside is protocol complexity: two ballots instead of one. But given the gnarliness of concurrent full-tree rekeying, a double consensus phase could be safer in practice.
-
-Supporting this implies the hash function must be aware of more than just `ledger_index`; it also needs `rules()` (to know if the amendment is enabled) and an explicit state flag indicating whether the swap is pending, in progress, or complete. To safely support background builds of multiple tree variants, `hash_options` must be plumbed everywhere — from `SHAMap::getHash()` down into all call sites, and even up into callers.
-
-## Two-pass rekey with a safety rope
-
-* **Pass 1 (plan)**: Walk the state tree, compute new BLAKE3 keys, build an in-memory LUT (old→new), and stamp each SLE with its legacy key (`sfLegacyKey`).
-* **Pass 2 (commit)**: Rebuild the SHAMap with BLAKE3 keys, rewrite all directories and secondary structures from the LUT, and finalize the new canonical tree.
-
-This two-pass structure ensures determinism and lets every validator converge on the same new map without risk of divergence.
-
-## Keep consensus boring during the scary bit
-
-Migration must not race against normal transaction flow. The procedure anchors on **network time**, not ledger index. Once a ledger closes with `closeTime ≥ MIGRATION_TIME`, the network enters a quiet period: all user and pseudo-transactions are blocked, only trivial skip list mechanics advance. During this window, everyone builds the same hash in the background.
-
-When consensus converges on the special BLAKE3 hash (excluding skip lists and migration state), it appears in a validated ledger. In the next ledger, the atomic swap happens — one big bang, then back to normal life.
-
-## Owning the ugly edges (hooks and hardcoded keys)
-
-Hooks may carry hardcoded 32-byte constants. Detecting them with static analysis is brittle; runtime tracing is too heavy. Instead, the LUT strategy provides a compatibility shim: lookups can still resolve old keys, while all new creations require canonical BLAKE3 keys. Over time, policy can deprecate this fallback.
-
----

hashing-ripple-structures-blake3-vs-sha512.md

@@ -1,99 +0,0 @@
-# BLAKE3 vs SHA-512 Performance Analysis for Ripple Data Structures
-
-## Executive Summary
-
-This document presents empirical performance comparisons between BLAKE3 and SHA-512 (specifically SHA512Half) when hashing Ripple/Xahau blockchain data structures. Tests were conducted on Apple Silicon (M-series) hardware using real-world data distributions.
-
-## Test Environment
-
-- **Platform**: Apple Silicon (ARM64)
-- **OpenSSL Version**: 1.1.1u (likely without ARMv8.2 SHA-512 hardware acceleration)
-- **BLAKE3**: C reference implementation with NEON optimizations
-- **Test Data**: Production ledger #16940119 from Xahau network
-
-## Results by Data Type
-
-### 1. Keylet Lookups (22-102 bytes, 35-byte weighted average)
-
-Keylets are namespace discriminators used for ledger lookups. The SHAMap requires high-entropy keys for balanced tree structure, necessitating cryptographic hashing even for small inputs.
-
-**Distribution:**
-- 76,478 ACCOUNT lookups (22 bytes)
-- 41,740 HOOK lookups (22 bytes)  
-- 19,939 HOOK_STATE_DIR lookups (54 bytes)
-- 17,587 HOOK_DEFINITION lookups (34 bytes)
-- 17,100 HOOK_STATE lookups (86 bytes)
-- Other types: ~15,000 operations (22-102 bytes)
-
-**Performance (627,131 operations):**
-- **BLAKE3**: 128 ns/hash, 7.81M hashes/sec
-- **SHA-512**: 228 ns/hash, 4.38M hashes/sec
-- **Speedup**: 1.78x
-
-### 2. Leaf Node Data (167-byte average)
-
-Leaf nodes contain serialized ledger entries (accounts, trustlines, offers, etc.). These represent the actual state data in the ledger.
-
-**Distribution:**
-- 626,326 total leaf nodes
-- 104.6 MB total data
-- Types: AccountRoot (145k), DirectoryNode (118k), RippleState (115k), HookState (124k), URIToken (114k)
-
-**Performance (from production benchmark):**
-- **SHA-512**: 446 ns/hash, 357 MB/s (measured)
-- **BLAKE3**: ~330 ns/hash, 480 MB/s (projected)
-- **Expected Speedup**: ~1.35x
-
-### 3. Inner Nodes (516 bytes exactly)
-
-Inner nodes contain 16 child hashes (32 bytes each) plus a 4-byte prefix. These form the Merkle tree structure enabling cryptographic proofs.
-
-**Distribution:**
-- 211,364 inner nodes
-- 104.1 MB total data (nearly equal to leaf data volume)
-
-**Performance (211,364 operations):**
-- **BLAKE3**: 898 ns/hash, 548 MB/s
-- **SHA-512**: 1081 ns/hash, 455 MB/s
-- **Speedup**: 1.20x
-
-## Overall Impact Analysis
-
-### Current System Profile
-
-From production measurements, the ledger validation process shows:
-- **Map traversal**: 47% of time
-- **SHA-512 hashing**: 53% of time
-
-Within the hashing time specifically:
-- **Keylet lookups**: ~50% of hashing time
-- **Leaf/inner nodes**: ~50% of hashing time
-
-### Projected Improvement with BLAKE3
-
-Given the measured speedups:
-- Keylet operations: 1.78x faster → 28% time reduction
-- Leaf operations: 1.35x faster → 26% time reduction  
-- Inner operations: 1.20x faster → 17% time reduction
-
-**Net improvement**: ~20-25% reduction in total hashing time, or **10-13% reduction in overall validation time**.
-
-## Key Observations
-
-1. **Small Input Performance**: BLAKE3 shows its greatest advantage (1.78x) on small keylet inputs where function call overhead dominates.
-
-2. **Diminishing Returns**: As input size increases to SHA-512's block size (128 bytes) and multiples thereof, the performance gap narrows significantly.
-
-3. **Architectural Constraint**: The SHAMap design requires cryptographic hashing for all operations to maintain high-entropy keys, preventing optimization through non-cryptographic alternatives.
-
-4. **Implementation Effort**: Transitioning from SHA-512 to BLAKE3 would require:
-   - Updating all hash generation code
-   - Maintaining backward compatibility
-   - Extensive testing of consensus-critical code
-   - Potential network upgrade coordination
-
-## Conclusion
-
-BLAKE3 offers measurable performance improvements over SHA-512 for Ripple data structures, particularly for small inputs. However, the gains are modest (1.2-1.78x depending on input size) rather than revolutionary. With map traversal consuming nearly half the total time, even perfect hashing would only double overall performance.
-
-For keylet operations specifically, the 1.78x speedup is significant given that keylet hashing accounts for approximately 50% of all hashing time. However, the measured improvements must be weighed against the engineering effort and risk of modifying consensus-critical cryptographic primitives. A 10-13% overall performance gain may not justify the migration complexity unless combined with other architectural improvements.

migrate_keylet_calls.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-import argparse
-from pathlib import Path
-from collections import defaultdict
-from typing import Dict, List, Tuple, Optional
-
-# Mapping of keylet functions to their specific HashContext classifiers
-KEYLET_CLASSIFIERS = {
-    'account': 'KEYLET_ACCOUNT',
-    'amendments': 'KEYLET_AMENDMENTS',
-    'book': 'KEYLET_BOOK',
-    'check': 'KEYLET_CHECK',
-    'child': 'KEYLET_CHILD',
-    'depositPreauth': 'KEYLET_DEPOSIT_PREAUTH',
-    'emittedDir': 'KEYLET_EMITTED_DIR',
-    'emittedTxn': 'KEYLET_EMITTED_TXN',
-    'escrow': 'KEYLET_ESCROW',
-    'fees': 'KEYLET_FEES',
-    'hook': 'KEYLET_HOOK',
-    'hookDefinition': 'KEYLET_HOOK_DEFINITION',
-    'hookState': 'KEYLET_HOOK_STATE',
-    'hookStateDir': 'KEYLET_HOOK_STATE_DIR',
-    'import_vlseq': 'KEYLET_IMPORT_VLSEQ',
-    'line': 'KEYLET_TRUSTLINE',
-    'negativeUNL': 'KEYLET_NEGATIVE_UNL',
-    'nft_buys': 'KEYLET_NFT_BUYS',
-    'nft_sells': 'KEYLET_NFT_SELLS',
-    'nftoffer': 'KEYLET_NFT_OFFER',
-    'nftpage': 'KEYLET_NFT_PAGE',
-    'nftpage_max': 'KEYLET_NFT_PAGE',
-    'nftpage_min': 'KEYLET_NFT_PAGE',
-    'offer': 'KEYLET_OFFER',
-    'ownerDir': 'KEYLET_OWNER_DIR',
-    'page': 'KEYLET_DIR_PAGE',
-    'payChan': 'KEYLET_PAYCHAN',
-    'signers': 'KEYLET_SIGNERS',
-    'skip': 'KEYLET_SKIP_LIST',
-    'ticket': 'KEYLET_TICKET',
-    'UNLReport': 'KEYLET_UNL_REPORT',
-    'unchecked': 'KEYLET_UNCHECKED',
-    'uritoken': 'KEYLET_URI_TOKEN',
-}
-
-def add_classifiers_to_digest_h(digest_h_path: str, dry_run: bool = True) -> bool:
-    """Add the new KEYLET_ classifiers to digest.h if they don't exist."""
-    
-    # Read the file
-    with open(digest_h_path, 'r') as f:
-        content = f.read()
-    
-    # Check if we already have KEYLET_ classifiers
-    if 'KEYLET_ACCOUNT' in content:
-        print("KEYLET classifiers already exist in digest.h")
-        return True
-    
-    # Find the end of the HashContext enum (before the closing brace and semicolon)
-    pattern = r'(enum HashContext[^{]*\{[^}]*)(HOOK_DEFINITION\s*=\s*\d+,?)([^}]*\};)'
-    
-    match = re.search(pattern, content, re.DOTALL)
-    if not match:
-        print("ERROR: Could not find HashContext enum in digest.h")
-        return False
-    
-    # Build the new classifiers text
-    new_classifiers = []
-    
-    # Get the last number used (HOOK_DEFINITION = 17)
-    last_num = 17
-    
-    # Add all KEYLET classifiers
-    unique_classifiers = sorted(set(KEYLET_CLASSIFIERS.values()))
-    for i, classifier in enumerate(unique_classifiers, start=1):
-        new_classifiers.append(f"    {classifier} = {last_num + i},")
-    
-    # Join them with newlines
-    new_text = '\n'.join(new_classifiers)
-    
-    # Create the replacement
-    replacement = match.group(1) + match.group(2) + ',\n\n    // Keylet-specific hash contexts\n' + new_text + match.group(3)
-    
-    # Replace in content
-    new_content = content[:match.start()] + replacement + content[match.end():]
-    
-    if dry_run:
-        print("=" * 80)
-        print("WOULD ADD TO digest.h:")
-        print("=" * 80)
-        print(new_text)
-        print("=" * 80)
-    else:
-        with open(digest_h_path, 'w') as f:
-            f.write(new_content)
-        print(f"Updated {digest_h_path} with KEYLET classifiers")
-    
-    return True
-
-def migrate_keylet_call(content: str, func_name: str, dry_run: bool = True) -> Tuple[str, int]:
-    """
-    Migrate keylet calls from single ledger_index to ledger_index + classifier.
-    Returns (modified_content, number_of_replacements)
-    """
-    
-    classifier = KEYLET_CLASSIFIERS.get(func_name)
-    if not classifier:
-        print(f"WARNING: No classifier mapping for keylet::{func_name}")
-        return content, 0
-    
-    # Pattern to match keylet::<func>(hash_options{<ledger_seq>}, ...)
-    # where ledger_seq doesn't already contain a comma (no classifier yet)
-    pattern = re.compile(
-        rf'keylet::{re.escape(func_name)}\s*\(\s*hash_options\s*\{{\s*([^,}}]+)\s*\}}',
-        re.MULTILINE
-    )
-    
-    count = 0
-    
-    def replacer(match):
-        nonlocal count
-        ledger_seq = match.group(1).strip()
-        # Check if it already has a classifier (contains comma)
-        if ',' in ledger_seq:
-            return match.group(0)  # Already migrated
-        
-        count += 1
-        # Add the classifier
-        return f'keylet::{func_name}(hash_options{{{ledger_seq}, {classifier}}}'
-    
-    new_content = pattern.sub(replacer, content)
-    
-    return new_content, count
-
-def process_file(filepath: str, dry_run: bool = True) -> int:
-    """Process a single file. Returns number of replacements made."""
-    
-    with open(filepath, 'r', encoding='utf-8') as f:
-        original_content = f.read()
-    
-    content = original_content
-    total_replacements = 0
-    replacements_by_func = {}
-    
-    # Process each keylet function
-    for func_name in KEYLET_CLASSIFIERS.keys():
-        new_content, count = migrate_keylet_call(content, func_name, dry_run)
-        if count > 0:
-            content = new_content
-            total_replacements += count
-            replacements_by_func[func_name] = count
-    
-    if total_replacements > 0:
-        if dry_run:
-            print(f"Would modify {filepath}: {total_replacements} replacements")
-            for func, count in sorted(replacements_by_func.items()):
-                print(f"  - keylet::{func}: {count}")
-        else:
-            with open(filepath, 'w', encoding='utf-8') as f:
-                f.write(content)
-            print(f"Modified {filepath}: {total_replacements} replacements")
-            for func, count in sorted(replacements_by_func.items()):
-                print(f"  - keylet::{func}: {count}")
-    
-    return total_replacements
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Migrate keylet calls to use HashContext classifiers'
-    )
-    parser.add_argument(
-        '--dry-run',
-        action='store_true',
-        default=True,
-        help='Show what would be changed without modifying files (default: True)'
-    )
-    parser.add_argument(
-        '--apply',
-        action='store_true',
-        help='Actually apply the changes (disables dry-run)'
-    )
-    parser.add_argument(
-        '--file',
-        help='Process a specific file only'
-    )
-    parser.add_argument(
-        '--add-classifiers',
-        action='store_true',
-        help='Add KEYLET_ classifiers to digest.h'
-    )
-    
-    args = parser.parse_args()
-    
-    if args.apply:
-        args.dry_run = False
-    
-    project_root = "/Users/nicholasdudfield/projects/xahaud-worktrees/xahaud-map-stats-rpc"
-    
-    # First, optionally add classifiers to digest.h
-    if args.add_classifiers:
-        digest_h = os.path.join(project_root, "src/ripple/protocol/digest.h")
-        if not add_classifiers_to_digest_h(digest_h, args.dry_run):
-            return 1
-        print()
-    
-    # Process files
-    if args.file:
-        # Process single file
-        filepath = os.path.join(project_root, args.file)
-        if not os.path.exists(filepath):
-            print(f"ERROR: File not found: {filepath}")
-            return 1
-        
-        process_file(filepath, args.dry_run)
-    else:
-        # Process all files
-        total_files = 0
-        total_replacements = 0
-        
-        print(f"{'DRY RUN: ' if args.dry_run else ''}Processing files in {project_root}/src/ripple")
-        print("=" * 80)
-        
-        for root, dirs, files in os.walk(Path(project_root) / "src" / "ripple"):
-            dirs[:] = [d for d in dirs if d not in ['.git', 'build', '__pycache__']]
-            
-            for file in files:
-                if file.endswith(('.cpp', '.h', '.hpp')):
-                    filepath = os.path.join(root, file)
-                    count = process_file(filepath, args.dry_run)
-                    if count > 0:
-                        total_files += 1
-                        total_replacements += count
-        
-        print("=" * 80)
-        print(f"{'Would modify' if args.dry_run else 'Modified'} {total_files} files")
-        print(f"Total replacements: {total_replacements}")
-        
-        if args.dry_run:
-            print("\nTo apply these changes, run with --apply flag:")
-            print(f"  python3 {sys.argv[0]} --apply")
-            print("\nTo first add classifiers to digest.h:")
-            print(f"  python3 {sys.argv[0]} --add-classifiers --apply")
-
-if __name__ == "__main__":
-    sys.exit(main())

src/quickjs/buffer-utils.c

@@ -0,0 +1,820 @@
+#include "defines.h"
+#include "char-utils.h"
+#include "buffer-utils.h"
+#include "utils.h"
+#ifdef _WIN32
+#include <windows.h>
+#elif defined(HAVE_TERMIOS_H)
+#include <termios.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#endif
+#include "debug.h"
+
+/**
+ * \addtogroup buffer-utils
+ * @{
+ */
+size_t
+ansi_length(const char* str, size_t len) {
+  size_t i, n = 0, p;
+  for(i = 0; i < len;) {
+    if(str[i] == 0x1b && (p = ansi_skip(&str[i], len - i)) > 0) {
+      i += p;
+      continue;
+    }
+    n++;
+    i++;
+  }
+  return n;
+}
+
+size_t
+ansi_skip(const char* str, size_t len) {
+  size_t pos = 0;
+  if(str[pos] == 0x1b) {
+    if(++pos < len && str[pos] == '[') {
+      while(++pos < len)
+        if(is_alphanumeric_char(str[pos]))
+          break;
+      if(++pos < len && str[pos] == '~')
+        ++pos;
+      return pos;
+    }
+  }
+  return 0;
+}
+
+size_t
+ansi_truncate(const char* str, size_t len, size_t limit) {
+  size_t i, n = 0, p;
+  for(i = 0; i < len;) {
+    if((p = ansi_skip(&str[i], len - i)) > 0) {
+      i += p;
+      continue;
+    }
+    n += is_escape_char(str[i]) ? 2 : 1;
+
+    i++;
+    if(n > limit)
+      break;
+  }
+  return i;
+}
+
+int64_t
+array_search(void* a, size_t m, size_t elsz, void* needle) {
+  char* ptr = a;
+  int64_t n, ret;
+  n = m / elsz;
+  for(ret = 0; ret < n; ret++) {
+    if(!memcmp(ptr, needle, elsz))
+      return ret;
+
+    ptr += elsz;
+  }
+  return -1;
+}
+
+char*
+str_escape(const char* s) {
+  DynBuf dbuf;
+  dbuf_init2(&dbuf, 0, 0);
+  dbuf_put_escaped(&dbuf, s, strlen(s));
+  dbuf_0(&dbuf);
+  return (char*)dbuf.buf;
+}
+
+char*
+byte_escape(const void* s, size_t n) {
+  DynBuf dbuf;
+  dbuf_init2(&dbuf, 0, 0);
+  dbuf_put_escaped(&dbuf, s, n);
+  dbuf_0(&dbuf);
+  return (char*)dbuf.buf;
+}
+
+size_t
+byte_findb(const void* haystack, size_t hlen, const void* what, size_t wlen) {
+  size_t i, last;
+  const char* s = (const char*)haystack;
+  if(hlen < wlen)
+    return hlen;
+  last = hlen - wlen;
+  for(i = 0; i <= last; i++) {
+    if(byte_equal(s, wlen, what))
+      return i;
+    s++;
+  }
+  return hlen;
+}
+
+size_t
+byte_finds(const void* haystack, size_t hlen, const char* what) {
+  return byte_findb(haystack, hlen, what, strlen(what));
+}
+
+size_t
+byte_equal(const void* s, size_t n, const void* t) {
+  return memcmp(s, t, n) == 0;
+}
+
+void
+byte_copy(void* out, size_t len, const void* in) {
+  char* s = (char*)out;
+  const char* t = (const char*)in;
+  size_t i;
+  for(i = 0; i < len; ++i)
+    s[i] = t[i];
+}
+
+void
+byte_copyr(void* out, size_t len, const void* in) {
+  char* s = (char*)out + len;
+  const char* t = (const char*)in;
+  const char* u = t + len;
+
+  for(;;) {
+    if(t >= u)
+      break;
+    --u;
+    --s;
+    *s = *u;
+  }
+}
+
+size_t
+byte_rchrs(const char* in, size_t len, const char needles[], size_t nn) {
+  const char *s = in, *end = in + len, *found = 0;
+  size_t i;
+  for(; s < end; s++) {
+    for(i = 0; i < nn; ++i) {
+      if(*s == needles[i])
+        found = s;
+    }
+  }
+  return (size_t)((found ? found : s) - in);
+}
+
+char*
+dbuf_at_n(const DynBuf* db, size_t i, size_t* n, char sep) {
+  size_t p, l = 0;
+  for(p = 0; p < db->size; ++p) {
+    if(l == i) {
+      *n = byte_chr((const char*)&db->buf[p], db->size - p, sep);
+      return (char*)&db->buf[p];
+    }
+    if(db->buf[p] == sep)
+      ++l;
+  }
+  *n = 0;
+  return 0;
+}
+
+const char*
+dbuf_last_line(DynBuf* db, size_t* len) {
+  size_t i;
+
+  if((i = byte_rchr(db->buf, db->size, '\n')) < db->size)
+    i++;
+  else
+    i = 0;
+
+  if(len)
+    *len = db->size - i;
+
+  return (const char*)&db->buf[i];
+}
+
+int
+dbuf_prepend(DynBuf* s, const uint8_t* data, size_t len) {
+  int ret;
+  if(!(ret = dbuf_reserve_start(s, len)))
+    memcpy(s->buf, data, len);
+
+  return 0;
+}
+
+void
+dbuf_put_colorstr(DynBuf* db, const char* str, const char* color, int with_color) {
+  if(with_color)
+    dbuf_putstr(db, color);
+
+  dbuf_putstr(db, str);
+  if(with_color)
+    dbuf_putstr(db, COLOR_NONE);
+}
+
+void
+dbuf_put_escaped_pred(DynBuf* db, const char* str, size_t len, int (*pred)(int)) {
+  size_t i = 0, j;
+  char c;
+
+  while(i < len) {
+    if((j = predicate_find(&str[i], len - i, pred))) {
+      dbuf_append(db, (const uint8_t*)&str[i], j);
+      i += j;
+    }
+
+    if(i == len)
+      break;
+
+    dbuf_putc(db, '\\');
+
+    if(str[i] == 0x1b) {
+      dbuf_append(db, (const uint8_t*)"x1b", 3);
+    } else {
+      int r = pred(str[i]);
+
+      dbuf_putc(db, (r > 1 && r <= 127) ? r : (c = escape_char_letter(str[i])) ? c : str[i]);
+
+      if(r == 'u' || r == 'x')
+        dbuf_printf(db, r == 'u' ? "%04x" : "%02x", str[i]);
+    }
+
+    i++;
+  }
+}
+
+const uint8_t escape_url_tab[256] = {
+    '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%',  '%', '%', '%',
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,    0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0x5c, 0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,    0,   0,   '%',
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,    0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,    0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,    0,   0,   0,
+    '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%', '%',  '%', '%', '%',
+};
+
+const uint8_t escape_noquote_tab[256] = {
+    'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 0x62, 0x74, 0x6e, 0x76, 0x66, 0x72, 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+    'x', 'x', 'x', 'x', 0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0x5c, 0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   'x', 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   'u', 'u', 'u', 'u', 'u',
+    'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',  'u',  'u',  'u',  'u',  'u',  'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',
+};
+
+const uint8_t escape_singlequote_tab[256] = {
+    'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 0x62, 0x74, 0x6e, 0x76, 0x66, 0x72, 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+    'x', 'x', 'x', 'x', 0,   0,   0,   0,   0,    0,    0,    0x27, 0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0x5c, 0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   'x', 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   'u', 'u', 'u', 'u', 'u',
+    'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',  'u',  'u',  'u',  'u',  'u',  'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',
+};
+
+const uint8_t escape_doublequote_tab[256] = {
+    'x', 'x', 'x', 'x', 'x', 'x', 'x',  'x', 0x62, 0x74, 0x6e, 0x76, 0x66, 0x72, 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+
+    'x', 'x', 'x', 'x', 0,   0,   0x22, 0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,    0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,    0,   0x5c, 0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,    0,   0,    0,    0,    0,    0,    0,    0,   'x', 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,    0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,    0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   'u', 'u', 'u', 'u', 'u',
+    'u', 'u', 'u', 'u', 'u', 'u', 'u',  'u', 'u',  'u',  'u',  'u',  'u',  'u',  'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',
+};
+
+const uint8_t escape_backquote_tab[256] = {
+    'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 0x62, 0x74, 0,   0x76, 0x66, 0,   'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+    'x', 'x', 'x', 'x', 0,   0,   0,   0,   0x24, 0,    0,   0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,   0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0x5c, 0,    0,   0,    0x60, 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,   0,    0,    0,   0,   'x', 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,   0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+    0,   0,   0,   0,   0,   0,   0,   0,   0,    0,    0,   0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   'u', 'u', 'u', 'u', 'u',
+    'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',  'u',  'u', 'u',  'u',  'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u',
+};
+
+void
+dbuf_put_escaped_table(DynBuf* db, const char* str, size_t len, const uint8_t table[256]) {
+  size_t i = 0, clen;
+  int32_t c;
+  const uint8_t *pos, *end, *next;
+
+  for(pos = (const uint8_t*)str, end = pos + len; pos < end; pos = next) {
+    uint8_t r, ch;
+
+    if((c = unicode_from_utf8(pos, end - pos, &next)) < 0)
+      break;
+    clen = next - pos;
+    ch = c;
+    r = (clen >= 2 || c > 0xff) ? 'u' : table[c];
+
+    if(r == 'u' && clen > 1 && (c & 0xffffff00) == 0) {
+      r = 'x';
+      // ch = c >> 8;
+    }
+
+    if(r == '%') {
+      static const char hexdigits[] = "0123456789ABCDEF";
+
+      dbuf_putc(db, '%');
+      dbuf_putc(db, hexdigits[c >> 4]);
+      dbuf_putc(db, hexdigits[c & 0xf]);
+    } else if(c == 0x1b) {
+      dbuf_putstr(db, "\\x1b");
+    } else if(r == 'u') {
+      dbuf_printf(db, c > 0xffff ? "\\u{%X}" : "\\u%04x", c);
+    } else if(r == 'x') {
+      dbuf_printf(db, "\\x%02x", ch);
+    } else if(r) {
+      dbuf_putc(db, '\\');
+      dbuf_putc(db, (r > 1 && r <= 127) ? r : (c = escape_char_letter(ch)) ? c : ch);
+    } else {
+      dbuf_put(db, pos, next - pos);
+    }
+
+    i++;
+  }
+}
+
+void
+dbuf_put_unescaped_pred(DynBuf* db, const char* str, size_t len, int (*pred)(const char*, size_t*)) {
+  size_t i = 0, j;
+  // char c;
+
+  while(i < len) {
+    int r = 0;
+    if((j = byte_chr(&str[i], len - i, '\\'))) {
+      dbuf_append(db, (const uint8_t*)&str[i], j);
+      i += j;
+    }
+    if(i == len)
+      break;
+    size_t n = 1;
+
+    if(pred) {
+      r = pred(&str[i + 1], &n);
+
+      if(!r && n == 1)
+        dbuf_putc(db, '\\');
+    }
+
+    if(r >= 0)
+      dbuf_putc(db, /*n > 1 ||*/ r ? /*(r > 1 && r < 256) ?*/ r : str[i]);
+    i += n;
+  }
+}
+
+static inline int
+hexdigit(char c) {
+  if(c >= '0' && c <= '9')
+    return c - '0';
+
+  if(c >= 'a' && c <= 'f')
+    c -= 32;
+
+  if(c >= 'A' && c <= 'F')
+    return c - 'A' + 10;
+
+  return -1;
+}
+
+void
+dbuf_put_unescaped_table(DynBuf* db, const char* str, size_t len, const uint8_t table[256]) {
+  size_t i = 0, j;
+  char escape_char = table == escape_url_tab ? '%' : '\\';
+
+  while(i < len) {
+    if((j = byte_chr(&str[i], len - i, escape_char))) {
+      dbuf_append(db, (const uint8_t*)&str[i], j);
+      i += j;
+    }
+
+    if(i == len)
+      break;
+
+    if(escape_char == '%') {
+      int hi = hexdigit(str[i + 1]), lo = hexdigit(str[i + 2]);
+      uint8_t c = (hi << 4) | (lo & 0xf);
+      dbuf_putc(db, c);
+
+      i += 2;
+
+    } else {
+      ++i;
+
+      uint8_t c;
+
+      switch(str[i]) {
+        case 'b': c = '\b'; break;
+        case 't': c = '\t'; break;
+        case 'n': c = '\n'; break;
+        case 'v': c = '\v'; break;
+        case 'f': c = '\f'; break;
+        case 'r': c = '\r'; break;
+        default: c = str[i]; break;
+      }
+
+      uint8_t r = table[c];
+
+      if(!(r && r != 'x' && r != 'u')) {
+        dbuf_putc(db, '\\');
+        dbuf_putc(db, c);
+      } else {
+        dbuf_putc(db, str[i] == r ? c : r);
+      }
+    }
+
+    ++i;
+  }
+}
+
+void
+dbuf_put_escaped(DynBuf* db, const char* str, size_t len) {
+  return dbuf_put_escaped_table(db, str, len, escape_noquote_tab);
+}
+
+void
+dbuf_put_value(DynBuf* db, JSContext* ctx, JSValueConst value) {
+  const char* str;
+  size_t len;
+  str = JS_ToCStringLen(ctx, &len, value);
+  dbuf_append(db, str, len);
+  JS_FreeCString(ctx, str);
+}
+
+void
+dbuf_put_uint32(DynBuf* db, uint32_t num) {
+  char buf[FMT_ULONG];
+  dbuf_put(db, (const uint8_t*)buf, fmt_ulong(buf, num));
+}
+
+void
+dbuf_put_atom(DynBuf* db, JSContext* ctx, JSAtom atom) {
+  const char* str;
+  str = JS_AtomToCString(ctx, atom);
+  dbuf_putstr(db, str);
+  JS_FreeCString(ctx, str);
+}
+
+int
+dbuf_reserve_start(DynBuf* s, size_t len) {
+  if(unlikely((s->size + len) > s->allocated_size)) {
+    if(dbuf_realloc(s, s->size + len))
+      return -1;
+  }
+  if(s->size > 0)
+    memcpy(s->buf + len, s->buf, s->size);
+
+  s->size += len;
+  return 0;
+}
+
+uint8_t*
+dbuf_reserve(DynBuf* s, size_t len) {
+  if(unlikely((s->size + len) > s->allocated_size))
+    if(dbuf_realloc(s, s->size + len))
+      return 0;
+
+  return &s->buf[s->size];
+}
+
+size_t
+dbuf_token_pop(DynBuf* db, char delim) {
+  size_t n, p, len;
+  len = db->size;
+  for(n = db->size; n > 0;) {
+    if((p = byte_rchr(db->buf, n, delim)) == n) {
+      db->size = 0;
+      break;
+    }
+    if(p > 0 && db->buf[p - 1] == '\\') {
+      n = p - 1;
+      continue;
+    }
+    db->size = p;
+    break;
+  }
+  return len - db->size;
+}
+
+size_t
+dbuf_token_push(DynBuf* db, const char* str, size_t len, char delim) {
+  size_t pos;
+  if(db->size)
+    dbuf_putc(db, delim);
+
+  pos = db->size;
+  dbuf_put_escaped_pred(db, str, len, is_dot_char);
+  return db->size - pos;
+}
+
+JSValue
+dbuf_tostring_free(DynBuf* s, JSContext* ctx) {
+  JSValue r;
+  r = JS_NewStringLen(ctx, s->buf ? (const char*)s->buf : "", s->buf ? s->size : 0);
+  dbuf_free(s);
+  return r;
+}
+
+ssize_t
+dbuf_load(DynBuf* s, const char* filename) {
+  FILE* fp;
+  size_t nbytes = 0;
+  if((fp = fopen(filename, "rb"))) {
+    char buf[4096];
+    size_t r;
+    while(!feof(fp)) {
+      if((r = fread(buf, 1, sizeof(buf), fp)) == 0) {
+        fclose(fp);
+        return -1;
+      }
+      dbuf_put(s, (uint8_t const*)buf, r);
+      nbytes += r;
+    }
+    fclose(fp);
+  }
+  return nbytes;
+}
+
+int
+dbuf_vprintf(DynBuf* s, const char* fmt, va_list ap) {
+
+  s->size += vsnprintf((char*)(s->buf + s->size), s->allocated_size - s->size, fmt, ap);
+
+  return 0;
+}
+
+InputBuffer
+js_input_buffer(JSContext* ctx, JSValueConst value) {
+  InputBuffer ret = {{{0, 0}}, 0, &input_buffer_free_default, JS_UNDEFINED, {0, INT64_MAX}};
+
+  if(js_is_typedarray(ctx, value)) {
+    ret.value = offset_typedarray(&ret.range, value, ctx);
+  } else if(js_is_arraybuffer(ctx, value) || js_is_sharedarraybuffer(ctx, value)) {
+    ret.value = JS_DupValue(ctx, value);
+  }
+
+  if(js_is_arraybuffer(ctx, ret.value) || js_is_sharedarraybuffer(ctx, ret.value)) {
+    block_arraybuffer(&ret.block, ret.value, ctx);
+  } else {
+    JS_ThrowTypeError(ctx, "Invalid type (%s) for input buffer", js_value_typestr(ctx, ret.value));
+    JS_FreeValue(ctx, ret.value);
+    ret.value = JS_EXCEPTION;
+  }
+
+  return ret;
+}
+
+#undef free
+
+InputBuffer
+js_input_chars(JSContext* ctx, JSValueConst value) {
+  InputBuffer ret = {{{0, 0}}, 0, &input_buffer_free_default, JS_UNDEFINED, OFFSET_INIT()};
+
+  if(JS_IsString(value)) {
+    ret.data = (uint8_t*)JS_ToCStringLen(ctx, &ret.size, value);
+    ret.value = JS_DupValue(ctx, value);
+    ret.free = &input_buffer_free_default;
+  } else {
+    ret = js_input_buffer(ctx, value);
+  }
+
+  return ret;
+}
+
+InputBuffer
+js_input_args(JSContext* ctx, int argc, JSValueConst argv[]) {
+  InputBuffer input = js_input_chars(ctx, argv[0]);
+
+  if(argc > 1)
+    js_offset_length(ctx, input.size, argc - 1, argv + 1, &input.range);
+
+  return input;
+}
+
+InputBuffer
+js_output_args(JSContext* ctx, int argc, JSValueConst argv[]) {
+  InputBuffer output = js_input_buffer(ctx, argv[0]);
+
+  if(argc > 1)
+    js_offset_length(ctx, output.size, argc - 1, argv + 1, &output.range);
+
+  return output;
+}
+
+BOOL
+input_buffer_valid(const InputBuffer* in) {
+  return !JS_IsException(in->value);
+}
+
+InputBuffer
+input_buffer_clone(const InputBuffer* in, JSContext* ctx) {
+  InputBuffer ret = js_input_buffer(ctx, in->value);
+
+  ret.pos = in->pos;
+  ret.size = in->size;
+  ret.free = in->free;
+
+  return ret;
+}
+
+void
+input_buffer_dump(const InputBuffer* in, DynBuf* db) {
+  dbuf_printf(db, "(InputBuffer){ .data = %p, .size = %lu, .pos = %lu, .free = %p }", in->data, (unsigned long)in->size, (unsigned long)in->pos, in->free);
+}
+
+void
+input_buffer_free(InputBuffer* in, JSContext* ctx) {
+  if(in->data) {
+    in->free(ctx, (const char*)in->data, in->value);
+    in->data = 0;
+    in->size = 0;
+    in->pos = 0;
+    in->value = JS_UNDEFINED;
+  }
+}
+
+const uint8_t*
+input_buffer_peek(InputBuffer* in, size_t* lenp) {
+  input_buffer_peekc(in, lenp);
+  return input_buffer_data(in) + in->pos;
+}
+
+const uint8_t*
+input_buffer_get(InputBuffer* in, size_t* lenp) {
+  const uint8_t* ret = input_buffer_peek(in, lenp);
+
+  in->pos += *lenp;
+
+  return ret;
+}
+
+const char*
+input_buffer_currentline(InputBuffer* in, size_t* len) {
+  size_t i;
+
+  if((i = byte_rchr(input_buffer_data(in), in->pos, '\n')) < in->pos)
+    i++;
+
+  if(len)
+    *len = in->pos - i;
+
+  return (const char*)&input_buffer_data(in)[i];
+}
+
+size_t
+input_buffer_column(InputBuffer* in, size_t* len) {
+  size_t i;
+
+  if((i = byte_rchr(input_buffer_data(in), in->pos, '\n')) < in->pos)
+    i++;
+
+  return in->pos - i;
+}
+
+int
+js_offset_length(JSContext* ctx, int64_t size, int argc, JSValueConst argv[], OffsetLength* off_len_p) {
+  int ret = 0;
+  int64_t off = 0, len = size;
+
+  if(argc >= 1 && JS_IsNumber(argv[0])) {
+    if(!JS_ToInt64(ctx, &off, argv[0]))
+      ret = 1;
+
+    if(argc >= 2 && JS_IsNumber(argv[1]))
+      if(!JS_ToInt64(ctx, &len, argv[1]))
+        ret = 2;
+
+    if(size && off != size)
+      off = ((off % size) + size) % size;
+
+    if(len >= 0)
+      len = MIN_NUM(len, size - off);
+    else
+      len = size - off;
+
+    if(off_len_p) {
+      off_len_p->offset = off;
+      off_len_p->length = len;
+    }
+  }
+
+  return ret;
+}
+
+int
+js_index_range(JSContext* ctx, int64_t size, int argc, JSValueConst argv[], IndexRange* idx_rng_p) {
+  int ret = 0;
+  int64_t start = 0, end = size;
+
+  if(argc >= 1 && JS_IsNumber(argv[0])) {
+    if(!JS_ToInt64(ctx, &start, argv[0]))
+      ret = 1;
+
+    if(argc >= 2 && JS_IsNumber(argv[1]))
+      if(!JS_ToInt64(ctx, &end, argv[1]))
+        ret = 2;
+
+    if(size > 0) {
+      start = ((start % size) + size) % size;
+      end = ((end % size) + size) % size;
+    }
+
+    if(end > size)
+      end = size;
+
+    if(idx_rng_p) {
+      idx_rng_p->start = start;
+      idx_rng_p->end = end;
+    }
+  }
+
+  return ret;
+}
+
+int
+screen_size(int size[2]) {
+#ifdef _WIN32
+  CONSOLE_SCREEN_BUFFER_INFO csbi;
+
+  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
+  size[0] = csbi.srWindow.Right - csbi.srWindow.Left + 1;
+  size[1] = csbi.srWindow.Bottom - csbi.srWindow.Top + 1;
+  return 0;
+
+#elif defined(HAVE_TERMIOS_H)
+  {
+    struct winsize w = {.ws_col = -1, .ws_row = -1};
+
+    if(isatty(STDIN_FILENO))
+      ioctl(STDIN_FILENO, TIOCGWINSZ, &w);
+    else if(isatty(STDOUT_FILENO))
+      ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
+    else if(isatty(STDERR_FILENO))
+      ioctl(STDERR_FILENO, TIOCGWINSZ, &w);
+
+    size[0] = w.ws_col;
+    size[1] = w.ws_row;
+    return 0;
+  }
+#else
+  size[0] = 80;
+  size[1] = 25;
+  return 0;
+#endif
+  return -1;
+}
+
+#undef js_realloc_rt
+void
+js_dbuf_allocator(JSContext* ctx, DynBuf* s) {
+  dbuf_init2(s, JS_GetRuntime(ctx), (DynBufReallocFunc*)js_realloc_rt);
+}
+
+inline int
+input_buffer_peekc(InputBuffer* in, size_t* lenp) {
+  const uint8_t *pos, *end, *next;
+  int cp;
+  pos = input_buffer_data(in) + in->pos;
+  end = input_buffer_data(in) + input_buffer_length(in);
+  cp = unicode_from_utf8(pos, end - pos, &next);
+
+  *lenp = next - pos;
+
+  return cp;
+}
+
+inline int
+input_buffer_putc(InputBuffer* in, unsigned int c, JSContext* ctx) {
+  int len;
+
+  if(in->pos + UTF8_CHAR_LEN_MAX > in->size)
+    if(block_realloc(&in->block, in->pos + UTF8_CHAR_LEN_MAX, ctx))
+      return -1;
+
+  len = unicode_to_utf8(&in->data[in->pos], c);
+
+  in->pos += len;
+
+  return len;
+}
+
+size_t
+dbuf_bitflags(DynBuf* db, uint32_t bits, const char* const names[]) {
+  size_t i, n = 0;
+  for(i = 0; i < sizeof(bits) * 8; i++) {
+    if(bits & (1 << i)) {
+      size_t len = strlen(names[i]);
+      if(n) {
+        n++;
+        dbuf_putstr(db, "|");
+      }
+      dbuf_append(db, names[i], len);
+      n += len;
+    }
+  }
+  return n;
+}
+
+/**
+ * @}
+ */

src/quickjs/buffer-utils.h

@@ -0,0 +1,348 @@
+#ifndef BUFFER_UTILS_H
+#define BUFFER_UTILS_H
+
+#include <quickjs.h>
+#include <cutils.h>
+#include <stdarg.h>
+
+#include "char-utils.h"
+
+/**
+ * \defgroup buffer-utils buffer-utils: Buffer Utilities
+ * @{
+ */
+int64_t array_search(void* a, size_t m, size_t elsz, void* needle);
+#define array_contains(a, m, elsz, needle) (array_search((a), (m), (elsz), (needle)) != -1)
+
+size_t ansi_length(const char*, size_t);
+size_t ansi_skip(const char*, size_t);
+size_t ansi_truncate(const char*, size_t, size_t limit);
+int64_t array_search(void*, size_t, size_t elsz, void* needle);
+char* str_escape(const char*);
+
+char* byte_escape(const void*, size_t);
+size_t byte_findb(const void*, size_t, const void* what, size_t wlen);
+size_t byte_finds(const void*, size_t, const char* what);
+size_t byte_equal(const void* s, size_t n, const void* t);
+void byte_copy(void* out, size_t len, const void* in);
+void byte_copyr(void* out, size_t len, const void* in);
+size_t byte_rchrs(const char* in, size_t len, const char needles[], size_t nn);
+
+#define DBUF_INIT_0() \
+  (DynBuf) { 0, 0, 0, 0, 0, 0 }
+#define DBUF_INIT_CTX(ctx) \
+  (DynBuf) { 0, 0, 0, 0, (DynBufReallocFunc*)js_realloc_rt, JS_GetRuntime(ctx) }
+
+extern const uint8_t escape_url_tab[256], escape_noquote_tab[256], escape_singlequote_tab[256], escape_doublequote_tab[256], escape_backquote_tab[256];
+
+char* dbuf_at_n(const DynBuf*, size_t, size_t* n, char sep);
+const char* dbuf_last_line(DynBuf*, size_t*);
+int dbuf_prepend(DynBuf*, const uint8_t*, size_t len);
+void dbuf_put_colorstr(DynBuf*, const char*, const char* color, int with_color);
+void dbuf_put_escaped_pred(DynBuf*, const char*, size_t len, int (*pred)(int));
+void dbuf_put_escaped_table(DynBuf*, const char*, size_t len, const uint8_t table[256]);
+void dbuf_put_unescaped_table(DynBuf* db, const char* str, size_t len, const uint8_t table[256]);
+void dbuf_put_unescaped_pred(DynBuf*, const char*, size_t len, int (*pred)());
+void dbuf_put_escaped(DynBuf*, const char*, size_t len);
+void dbuf_put_value(DynBuf*, JSContext*, JSValue value);
+void dbuf_put_uint32(DynBuf* db, uint32_t num);
+void dbuf_put_atom(DynBuf* db, JSContext* ctx, JSAtom atom);
+int dbuf_reserve_start(DynBuf*, size_t);
+uint8_t* dbuf_reserve(DynBuf*, size_t);
+size_t dbuf_token_pop(DynBuf*, char);
+size_t dbuf_token_push(DynBuf*, const char*, size_t len, char delim);
+JSValue dbuf_tostring_free(DynBuf*, JSContext*);
+ssize_t dbuf_load(DynBuf*, const char*);
+int dbuf_vprintf(DynBuf*, const char*, va_list);
+
+int screen_size(int size[2]);
+
+static inline int
+dbuf_putm(DynBuf* db, ...) {
+  int r = 0;
+  va_list a;
+  const char* s;
+  va_start(a, db);
+  while((s = va_arg(a, char*)))
+    if(dbuf_putstr(db, s))
+      return -1;
+  va_end(a);
+  return r;
+}
+
+#define dbuf_append(d, x, n) dbuf_put((d), (const uint8_t*)(x), (n))
+
+static inline size_t
+dbuf_count(DynBuf* db, int ch) {
+  return byte_count(db->buf, db->size, ch);
+}
+
+static inline void
+dbuf_0(DynBuf* db) {
+  dbuf_putc(db, '\0');
+  db->size--;
+}
+
+static inline void
+dbuf_zero(DynBuf* db) {
+  dbuf_realloc(db, 0);
+  db->size = 0;
+}
+
+size_t dbuf_bitflags(DynBuf* db, uint32_t bits, const char* const names[]);
+
+#define js_dbuf_init(ctx, buf) dbuf_init2((buf), (ctx), (realloc_func*)&utils_js_realloc)
+#define js_dbuf_init_rt(rt, buf) dbuf_init2((buf), (rt), (realloc_func*)&utils_js_realloc_rt)
+
+void js_dbuf_allocator(JSContext* ctx, DynBuf* s);
+
+typedef struct {
+  uint8_t* base;
+  size_t size;
+} MemoryBlock;
+
+static inline void
+block_init(MemoryBlock* mb) {
+  mb->base = 0;
+  mb->size = 0;
+}
+
+/* clang-format off */
+static inline void* block_data(const MemoryBlock* mb) { return mb->base; }
+static inline size_t block_length(const MemoryBlock* mb) { return mb->size; }
+static inline void* block_begin(const MemoryBlock* mb) { return mb->base; }
+static inline void* block_end(const MemoryBlock* mb) { return mb->base + mb->size; }
+/* clang-format on */
+
+static inline BOOL
+block_arraybuffer(MemoryBlock* mb, JSValueConst ab, JSContext* ctx) {
+  mb->base = JS_GetArrayBuffer(ctx, &mb->size, ab);
+  return mb->base != 0;
+}
+
+typedef struct {
+  uint8_t *start, *end;
+} PointerRange;
+
+static inline void
+range_init(PointerRange* pr) {
+  pr->end = pr->start = 0;
+}
+
+static inline PointerRange
+range_from(const MemoryBlock* mb) {
+  return (PointerRange){mb->base, mb->base + mb->size};
+}
+
+typedef struct {
+  int64_t start, end;
+} IndexRange;
+
+typedef struct {
+  int64_t offset, length;
+} OffsetLength;
+
+#define OFFSET_INIT() \
+  (OffsetLength) { 0, INT64_MAX }
+
+static inline void
+offset_init(OffsetLength* ol) {
+  ol->offset = 0;
+  ol->length = INT64_MAX;
+}
+
+static inline BOOL
+offset_is_default(const OffsetLength* ol) {
+  return ol->offset == 0 && ol->length == INT64_MAX;
+}
+
+static inline void*
+offset_data(const OffsetLength* ol, const void* x) {
+  return (uint8_t*)x + ol->offset;
+}
+
+static inline size_t
+offset_size(const OffsetLength* ol, size_t n) {
+  if(ol->length == -1)
+    return (signed long)n - ol->offset;
+  return MIN_NUM(ol->length, (signed long)n - ol->offset);
+}
+
+static inline MemoryBlock
+offset_block(const OffsetLength* ol, const void* x, size_t n) {
+  return (MemoryBlock){offset_data(ol, x), offset_size(ol, n)};
+}
+
+static inline PointerRange
+offset_range(const OffsetLength* ol, const void* x, size_t n) {
+  MemoryBlock mb = offset_block(ol, x, n);
+  return range_from(&mb);
+}
+
+static inline OffsetLength
+offset_slice(const OffsetLength ol, int64_t start, int64_t end) {
+  if(start < 0)
+    start = ol.length + (start % ol.length);
+  else if(start > ol.length)
+    start = ol.length;
+  if(end < 0)
+    end = ol.length + (end % ol.length);
+  else if(end > ol.length)
+    end = ol.length;
+
+  return (OffsetLength){start, end - start};
+}
+
+static inline OffsetLength
+offset_offset(const OffsetLength* ol, const OffsetLength* by) {
+  OffsetLength ret;
+  ret.offset = ol->offset + by->offset;
+  ret.length = MIN_NUM(by->length, ol->length - by->offset);
+  return ret;
+}
+
+static inline OffsetLength
+offset_from_indexrange(const IndexRange* ir) {
+  OffsetLength ret;
+  ret.offset = ir->start;
+  ret.length = ir->end - ir->start;
+  return ret;
+}
+
+static inline JSValue
+offset_typedarray(OffsetLength* ol, JSValueConst array, JSContext* ctx) {
+  JSValue ret;
+  size_t offset, length;
+
+  ret = JS_GetTypedArrayBuffer(ctx, array, &offset, &length, NULL);
+
+  if(!JS_IsException(ret)) {
+    ol->offset = offset;
+    ol->length = length;
+  }
+
+  return ret;
+}
+
+static inline IndexRange
+indexrange_from_offset(const OffsetLength* ol) {
+  IndexRange ret;
+  ret.start = ol->offset;
+  ret.end = ol->offset + ol->length;
+  return ret;
+}
+
+static inline MemoryBlock
+block_range(const MemoryBlock* mb, const OffsetLength* range) {
+  MemoryBlock ret;
+  ret.base = mb->base + range->offset;
+  ret.size = MIN_NUM((size_t)range->length, mb->size - range->offset);
+  return ret;
+}
+
+static inline int
+block_realloc(MemoryBlock* mb, size_t new_size, JSContext* ctx) {
+  if((mb->base = js_realloc(ctx, mb->base, new_size))) {
+    mb->size = new_size;
+    return 0;
+  }
+  return -1;
+}
+
+typedef struct InputBuffer {
+  union {
+    MemoryBlock block;
+    struct {
+      uint8_t* data;
+      size_t size;
+    };
+  };
+  size_t pos;
+  void (*free)(JSContext*, const char*, JSValue);
+  JSValue value;
+  OffsetLength range;
+} InputBuffer;
+
+static inline void
+input_buffer_free_default(JSContext* ctx, const char* str, JSValue val) {
+  if(JS_IsString(val))
+    JS_FreeCString(ctx, str);
+
+  if(!JS_IsUndefined(val))
+    JS_FreeValue(ctx, val);
+}
+
+InputBuffer js_input_buffer(JSContext* ctx, JSValueConst value);
+InputBuffer js_input_chars(JSContext* ctx, JSValueConst value);
+InputBuffer js_input_args(JSContext* ctx, int argc, JSValueConst argv[]);
+InputBuffer js_output_args(JSContext* ctx, int argc, JSValueConst argv[]);
+
+InputBuffer input_buffer_clone(const InputBuffer* in, JSContext* ctx);
+BOOL input_buffer_valid(const InputBuffer* in);
+void input_buffer_dump(const InputBuffer* in, DynBuf* db);
+void input_buffer_free(InputBuffer* in, JSContext* ctx);
+
+static inline uint8_t*
+input_buffer_data(const InputBuffer* in) {
+  return offset_data(&in->range, in->data);
+}
+
+static inline size_t
+input_buffer_length(const InputBuffer* in) {
+  return offset_size(&in->range, in->size);
+}
+
+static inline MemoryBlock
+input_buffer_block(InputBuffer* in) {
+  return (MemoryBlock){input_buffer_data(in), input_buffer_length(in)};
+}
+
+static inline MemoryBlock*
+input_buffer_blockptr(InputBuffer* in) {
+  return &in->block;
+}
+
+const uint8_t* input_buffer_get(InputBuffer* in, size_t* lenp);
+const uint8_t* input_buffer_peek(InputBuffer* in, size_t* lenp);
+const char* input_buffer_currentline(InputBuffer*, size_t* len);
+size_t input_buffer_column(InputBuffer*, size_t* len);
+
+int input_buffer_peekc(InputBuffer* in, size_t* lenp);
+int input_buffer_putc(InputBuffer*, unsigned int, JSContext*);
+
+static inline int
+input_buffer_getc(InputBuffer* in) {
+  size_t n;
+  int ret;
+  ret = input_buffer_peekc(in, &n);
+  in->pos += n;
+  return ret;
+}
+
+static inline void*
+input_buffer_begin(const InputBuffer* in) {
+  return input_buffer_data(in);
+}
+
+static inline void*
+input_buffer_end(const InputBuffer* in) {
+  return input_buffer_data(in) + input_buffer_length(in);
+}
+
+static inline BOOL
+input_buffer_eof(const InputBuffer* in) {
+  return in->pos == input_buffer_length(in);
+}
+
+static inline size_t
+input_buffer_remain(const InputBuffer* in) {
+  return input_buffer_length(in) - in->pos;
+}
+
+int js_offset_length(JSContext*, int64_t size, int argc, JSValueConst argv[], OffsetLength* off_len_p);
+int js_index_range(JSContext*, int64_t size, int argc, JSValueConst argv[], IndexRange* idx_rng_p);
+
+/**
+ * @}
+ */
+#endif /* defined(BUFFER_UTILS) */

src/quickjs/char-utils.c

@@ -0,0 +1,578 @@
+#include "char-utils.h"
+#include "libutf.h"
+#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MSYS__)
+#include <winnls.h>
+#include <windows.h>
+#include <wchar.h>
+#endif
+
+/**
+ * \addtogroup char-utils
+ * @{
+ */
+size_t
+token_length(const char* str, size_t len, char delim) {
+  const char *s, *e;
+  size_t pos;
+
+  for(s = str, e = s + len; s < e; s += pos + 1) {
+    pos = byte_chr(s, e - s, delim);
+
+    if(s + pos == e)
+      break;
+
+    if(pos == 0 || s[pos - 1] != '\\') {
+      s += pos;
+      break;
+    }
+  }
+
+  return s - str;
+}
+
+size_t
+fmt_ulong(char* dest, uint32_t i) {
+  uint32_t len, tmp, len2;
+
+  for(len = 1, tmp = i; tmp > 9; ++len)
+    tmp /= 10;
+
+  if(dest)
+    for(tmp = i, dest += len, len2 = len + 1; --len2; tmp /= 10)
+      *--dest = (char)((tmp % 10) + '0');
+
+  return len;
+}
+
+size_t
+fmt_longlong(char* dest, int64_t i) {
+  if(i < 0) {
+    if(dest)
+      *dest++ = '-';
+    return fmt_ulonglong(dest, (uint64_t)-i) + 1;
+  }
+
+  return fmt_ulonglong(dest, (uint64_t)i);
+}
+
+size_t
+fmt_ulonglong(char* dest, uint64_t i) {
+  size_t len;
+  uint64_t tmp, len2;
+
+  for(len = 1, tmp = i; tmp > 9ll; ++len)
+    tmp /= 10ll;
+
+  if(dest)
+    for(tmp = i, dest += len, len2 = len + 1; --len2; tmp /= 10ll)
+      *--dest = (tmp % 10ll) + '0';
+
+  return len;
+}
+
+#define tohex(c) (char)((c) >= 10 ? (c)-10 + 'a' : (c) + '0')
+
+size_t
+fmt_xlonglong(char* dest, uint64_t i) {
+  uint64_t len, tmp;
+
+  for(len = 1, tmp = i; tmp > 15ll; ++len)
+    tmp >>= 4ll;
+
+  if(dest)
+    for(tmp = i, dest += len;;) {
+      *--dest = tohex(tmp & 15ll);
+
+      if(!(tmp >>= 4ll))
+        break;
+    }
+
+  return len;
+}
+
+size_t
+fmt_xlonglong0(char* dest, uint64_t num, size_t n) {
+  size_t i = 0, len;
+
+  if((len = fmt_xlonglong(NULL, num)) < n) {
+    len = n - len;
+
+    while(i < len)
+      dest[i++] = '0';
+  }
+
+  i += fmt_xlonglong(&dest[i], num);
+  return i;
+}
+
+size_t
+fmt_8long(char* dest, uint32_t i) {
+  uint32_t len, tmp;
+
+  /* first count the number of bytes needed */
+  for(len = 1, tmp = i; tmp > 7; ++len)
+    tmp >>= 3;
+
+  if(dest)
+    for(tmp = i, dest += len;;) {
+      *--dest = (char)((tmp & 7) + '0');
+
+      if(!(tmp >>= 3))
+        break;
+    }
+
+  return len;
+}
+
+#define tohex(c) (char)((c) >= 10 ? (c)-10 + 'a' : (c) + '0')
+
+size_t
+fmt_xlong(char* dest, uint32_t i) {
+  uint32_t len, tmp;
+
+  /* first count the number of bytes needed */
+  for(len = 1, tmp = i; tmp > 15; ++len)
+    tmp >>= 4;
+
+  if(dest)
+    for(tmp = i, dest += len;;) {
+      *--dest = tohex(tmp & 15);
+
+      if(!(tmp >>= 4))
+        break;
+    }
+
+  return len;
+}
+
+size_t
+fmt_xlong0(char* dest, uint32_t num, size_t n) {
+  size_t i = 0, len;
+
+  if((len = fmt_xlong(NULL, num)) < n) {
+    len = n - len;
+
+    while(i < len)
+      dest[i++] = '0';
+  }
+
+  i += fmt_xlong(&dest[i], num);
+  return i;
+}
+
+size_t
+scan_ushort(const char* src, uint16_t* dest) {
+  const char* cur;
+  uint16_t l;
+
+  for(cur = src, l = 0; *cur >= '0' && *cur <= '9'; ++cur) {
+    uint32_t tmp = l * 10ul + *cur - '0';
+
+    if((uint16_t)tmp != tmp)
+      break;
+
+    l = tmp;
+  }
+
+  if(cur > src)
+    *dest = l;
+
+  return (size_t)(cur - src);
+}
+
+size_t
+scan_uint(const char* src, uint32_t* dest) {
+  uint64_t u64;
+  size_t r = scan_ulonglong(src, &u64);
+  *dest = u64;
+  return r;
+}
+
+size_t
+scan_int(const char* src, int32_t* dest) {
+  int64_t i64;
+  size_t r = scan_longlong(src, &i64);
+  *dest = i64;
+  return r;
+}
+
+#ifndef MAXLONG
+#define MAXLONG (((uint32_t)-1) >> 1)
+#endif
+
+size_t
+scan_longlong(const char* src, int64_t* dest) {
+  size_t i, o;
+  uint64_t l;
+  char c = src[0];
+  unsigned int neg = c == '-';
+  o = c == '-' || c == '+';
+
+  if((i = scan_ulonglong(src + o, &l))) {
+    if(i > 0ll && l > MAXLONG + neg) {
+      l /= 10ll;
+      --i;
+    }
+
+    if(i + o)
+      *dest = (int64_t)(c == '-' ? -l : l);
+
+    return i + o;
+  }
+
+  return 0;
+}
+
+size_t
+scan_ulonglong(const char* src, uint64_t* dest) {
+  const char* tmp = src;
+  uint64_t l = 0;
+  unsigned char c;
+
+  while((c = (unsigned char)(*tmp - '0')) < 10) {
+    uint64_t n;
+    n = l << 3ll;
+
+    if((n >> 3ll) != l)
+      break;
+
+    if(n + (l << 1ll) < n)
+      break;
+
+    n += l << 1ll;
+
+    if(n + c < n)
+      break;
+
+    l = n + c;
+    ++tmp;
+  }
+
+  if(tmp - src)
+    *dest = l;
+
+  return (size_t)(tmp - src);
+}
+
+size_t
+scan_xlonglong(const char* src, uint64_t* dest) {
+  const char* tmp = src;
+  int64_t l = 0;
+  unsigned char c;
+
+  while((c = scan_fromhex(*tmp)) < 16) {
+    l = (l << 4) + c;
+    ++tmp;
+  }
+
+  *dest = l;
+  return tmp - src;
+}
+
+size_t
+scan_8longn(const char* src, size_t n, uint32_t* dest) {
+  const char* tmp = src;
+  uint32_t l = 0;
+  unsigned char c;
+
+  while(n-- > 0 && (c = (unsigned char)(*tmp - '0')) < 8) {
+    if(l >> (sizeof(l) * 8 - 3))
+      break;
+
+    l = l * 8 + c;
+    ++tmp;
+  }
+
+  *dest = l;
+  return (size_t)(tmp - src);
+}
+
+size_t
+scan_whitenskip(const char* s, size_t limit) {
+  const char *t, *u;
+
+  for(t = s, u = t + limit; t < u; ++t)
+    if(!is_whitespace_char(*t))
+      break;
+
+  return (size_t)(t - s);
+}
+
+size_t
+scan_nonwhitenskip(const char* s, size_t limit) {
+  const char *t, *u;
+
+  for(t = s, u = t + limit; t < u; ++t)
+    if(is_whitespace_char(*t))
+      break;
+
+  return (size_t)(t - s);
+}
+
+size_t
+scan_line(const char* s, size_t limit) {
+  const char *t, *u;
+
+  for(t = s, u = s + limit; t < u; ++t)
+    if(*t == '\n' || *t == '\r')
+      break;
+
+  return (size_t)(t - s);
+}
+
+size_t
+scan_lineskip(const char* s, size_t limit) {
+  const char *t, *u;
+
+  for(t = s, u = s + limit; t < u; ++t)
+    if(*t == '\n') {
+      ++t;
+      break;
+    }
+
+  return (size_t)(t - s);
+}
+
+size_t
+scan_lineskip_escaped(const char* s, size_t limit) {
+  const char *t, *u;
+
+  for(t = s, u = s + limit; t < u; ++t) {
+    if(*t == '\\') {
+      ++t;
+      continue;
+    }
+    if(*t == '\n') {
+      ++t;
+      break;
+    }
+  }
+
+  return (size_t)(t - s);
+}
+
+size_t
+scan_eolskip(const char* s, size_t limit) {
+  size_t n = 0;
+
+  if(n + 1 < limit && s[0] == '\r' && s[1] == '\n')
+    n += 2;
+  else if(n < limit && s[0] == '\n')
+    n += 1;
+
+  return n;
+}
+
+size_t
+utf8_strlen(const void* in, size_t len) {
+  const uint8_t *pos, *end, *next;
+  size_t i = 0;
+
+  for(pos = (const uint8_t*)in, end = pos + len; pos < end; pos = next, ++i)
+    unicode_from_utf8(pos, end - pos, &next);
+
+  return i;
+}
+
+#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MSYS__)
+wchar_t*
+utf8_towcs(const char* s) {
+  int len = (int)strlen(s);
+  int n = MultiByteToWideChar(CP_UTF8, 0, s, len, NULL, 0);
+  wchar_t* ret;
+
+  if((ret = (wchar_t*)malloc((n + 1) * sizeof(wchar_t)))) {
+    MultiByteToWideChar(CP_UTF8, 0, s, len, ret, n);
+    ret[n] = L'\0';
+  }
+
+  return ret;
+}
+
+char*
+utf8_fromwcs(const wchar_t* wstr) {
+  int len = (int)wcslen(wstr);
+  int n = WideCharToMultiByte(CP_UTF8, 0, wstr, len, NULL, 0, NULL, NULL);
+  char* ret;
+
+  if((ret = malloc((n + 1)))) {
+    WideCharToMultiByte(CP_UTF8, 0, wstr, len, ret, n, NULL, NULL);
+    ret[n] = '\0';
+  }
+
+  return ret;
+}
+#endif
+
+BOOL
+utf16_multiword(const void* in) {
+  const uint16_t* p16 = in;
+  LibutfC16Type type = libutf_c16_type(p16[0]);
+
+  return !((LIBUTF_UTF16_NOT_SURROGATE == type) || (LIBUTF_UTF16_SURROGATE_HIGH != type || LIBUTF_UTF16_SURROGATE_LOW != libutf_c16_type(p16[1])));
+}
+
+int
+case_lowerc(int c) {
+  if(c >= 'A' && c <= 'Z')
+    c += 'a' - 'A';
+
+  return c;
+}
+
+int
+case_starts(const char* a, const char* b) {
+  const char *s, *t;
+
+  for(s = a, t = b;; ++s, ++t) {
+    unsigned char x, y;
+
+    if(!*t)
+      return 1;
+
+    x = case_lowerc(*s);
+    y = case_lowerc(*t);
+
+    if(x != y)
+      break;
+
+    if(!x)
+      break;
+  }
+
+  return 0;
+}
+
+int
+case_diffb(const void* S, size_t len, const void* T) {
+  unsigned char x, y;
+  const char *s, *t;
+
+  for(s = (const char*)S, t = (const char*)T; len > 0;) {
+    --len;
+    x = case_lowerc(*s);
+    y = case_lowerc(*t);
+
+    ++s;
+    ++t;
+
+    if(x != y)
+      return ((int)(unsigned int)x) - ((int)(unsigned int)y);
+  }
+
+  return 0;
+}
+
+size_t
+case_findb(const void* haystack, size_t hlen, const void* what, size_t wlen) {
+  size_t i, last;
+  const char* s = haystack;
+
+  if(hlen < wlen)
+    return hlen;
+
+  last = hlen - wlen;
+
+  for(i = 0; i <= last; i++, s++)
+    if(!case_diffb(s, wlen, what))
+      return i;
+
+  return hlen;
+}
+
+size_t
+case_finds(const void* haystack, const char* what) {
+  return case_findb(haystack, strlen(haystack), what, strlen(what));
+}
+
+ssize_t
+write_file(const char* file, const void* buf, size_t len) {
+  FILE* f;
+  ssize_t ret = -1;
+
+  if((f = fopen(file, "w+")))
+    switch(fwrite(buf, len, 1, f)) {
+      case 1: {
+        ret = len;
+        break;
+      }
+    }
+
+  fflush(f);
+  ret = ftell(f);
+  fclose(f);
+
+  return ret;
+}
+
+ssize_t
+puts_file(const char* file, const char* s) {
+  return write_file(file, s, strlen(s));
+}
+
+size_t
+u64toa(char* x, uint64_t num, int base) {
+  size_t len = 0;
+  uint64_t n = num;
+
+  do {
+    n /= base;
+    len++;
+    x++;
+  } while(n != 0);
+
+  *x-- = '\0';
+
+  do {
+    char c = num % base;
+    num /= base;
+
+    if(c >= 10)
+      c += 'a' - '0' - 10;
+    *x-- = c + '0';
+  } while(num != 0);
+
+  return len;
+}
+
+size_t
+i64toa(char* x, int64_t num, int base) {
+  size_t pos = 0, len;
+
+  if(num < 0) {
+    x[pos++] = '-';
+    num = -num;
+  }
+
+  len = u64toa(&x[pos], num, base);
+
+  return pos + len;
+}
+
+size_t
+str_findb(const char* s1, const char* x, size_t n) {
+  const char* b;
+  size_t i, j, len = strlen(s1);
+
+  if(len >= n) {
+    size_t end = len - n + 1;
+
+    for(i = 0; i < end; i++) {
+      b = &s1[i];
+
+      for(j = 0; x[j] == b[j];)
+        if(++j == n)
+          return i;
+    }
+  }
+
+  return len;
+}
+
+size_t
+str_find(const void* s, const void* what) {
+  return str_findb(s, what, strlen(what));
+}
+/**
+ * @}
+ */

src/quickjs/char-utils.h

@@ -0,0 +1,441 @@
+#ifndef CHAR_UTILS_H
+#define CHAR_UTILS_H
+
+#include <cutils.h>
+#include <string.h>
+#include "debug.h"
+
+/**
+ * \defgroup char-utils char-utils: Character Utilities
+ * @{
+ */
+#define is_control_char(c) ((c) == '\a' || (c) == '\b' || (c) == '\t' || (c) == '\n' || (c) == '\v' || (c) == '\f' || (c) == '\r')
+#define is_alphanumeric_char(c) (((c) >= 'A' && (c) <= 'Z') || ((c) >= 'a' && (c) <= 'z'))
+
+#define is_digit_char(c) ((c) >= '0' && (c) <= '9')
+#define is_print_char(c) ((c) >= ' ' && (c) <= '\x7f')
+#define is_newline_char(c) ((c) == '\n')
+#define is_identifier_char(c) (is_alphanumeric_char(c) || is_digit_char(c) || (c) == '$' || (c) == '_')
+#define is_whitespace_char(c) ((c) == ' ' || (c) == '\t' || (c) == '\v' || (c) == '\n' || (c) == '\r')
+
+#define str_equal(s, t) (!strcmp((s), (t)))
+
+static inline int
+escape_char_pred(int c) {
+  static const unsigned char table[256] = {
+      'x', 'x', 'x', 'x', 'x', 'x',  'x', 'x', 0x62, 0x74, 0x6e, 0x76, 0x66, 0x72, 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+      'x', 'x', 'x', 0,   0,   0,    0,   0,   0,    0,    0x27, 0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0x5c, 0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    'x',  0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+      0,   0,   0,   0,   0,   0,    0,   0,   0,    0,    0,    0,    0,    0,    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+  };
+
+  return table[(unsigned char)c];
+}
+
+static inline int
+unescape_char_pred(int c) {
+  switch(c) {
+    case 'b': return 8;
+    case 'f': return 12;
+    case 'n': return 10;
+    case 'r': return 13;
+    case 't': return 9;
+    case 'v': return 11;
+    case '\'': return 39;
+    case '\\': return 92;
+  }
+
+  return 0;
+}
+
+static inline int
+is_escape_char(int c) {
+  return is_control_char(c) || c == '\\' || c == '\'' || c == 0x1b || c == 0;
+}
+
+static inline int
+is_backslash_char(int c) {
+  return c == '\\';
+}
+
+//#define is_dot_char(c) ((c) == '.')0
+//#define is_backslash_char(c) ((c) == '\\')
+
+static inline int
+is_dot_char(int c) {
+  return c == '.';
+}
+
+static inline int
+is_identifier(const char* str) {
+  if(!((*str >= 'A' && *str <= 'Z') || (*str >= 'a' && *str <= 'z') || *str == '$' || *str == '_'))
+    return 0;
+
+  while(*++str)
+    if(!is_identifier_char(*str))
+      return 0;
+
+  return 1;
+}
+
+static inline int
+is_integer(const char* str) {
+  if(*str == '-')
+    ++str;
+
+  if(!(*str >= '1' && *str <= '9') && !(*str == '0' && str[1] == '\0'))
+    return 0;
+
+  while(*++str)
+    if(!is_digit_char(*str))
+      return 0;
+
+  return 1;
+}
+
+static inline size_t
+byte_count(const void* s, size_t n, char c) {
+  const uint8_t* t;
+  uint8_t ch = (uint8_t)c;
+  size_t count;
+
+  for(t = (uint8_t*)s, count = 0; n; ++t, --n)
+    if(*t == ch)
+      ++count;
+
+  return count;
+}
+
+static inline size_t
+byte_chr(const void* str, size_t len, char c) {
+  const char* s = memchr(str, c, len);
+
+  if(s)
+    return s - (const char*)str;
+
+  return len;
+}
+
+static inline size_t
+byte_rchr(const void* haystack, size_t len, char needle) {
+  const char *s, *t;
+
+  for(s = (const char*)haystack, t = s + len;;) {
+    --t;
+
+    if(s > t)
+      break;
+
+    if(*t == needle)
+      return (size_t)(t - s);
+  }
+
+  return len;
+}
+
+/*size_t
+byte_rchr(const void* str, size_t len, char c) {
+  const char* s = memrchr(str, c, len);
+  if(s)
+    return s - (const char*)str;
+  return len;
+}*/
+
+static inline size_t
+byte_chrs(const void* str, size_t len, const char needle[], size_t nl) {
+  const char *s, *t;
+
+  for(s = str, t = s + len; s != t; s++)
+    if(byte_chr(needle, nl, *s) < nl)
+      break;
+
+  return s - (const char*)str;
+}
+
+static inline int
+byte_diff(const void* a, size_t len, const void* b) {
+  size_t i;
+
+  for(i = 0; i < len; ++i) {
+    int r = ((unsigned char*)a)[i] - ((unsigned char*)b)[i];
+
+    if(r)
+      return r;
+  }
+
+  return 0;
+}
+
+static inline int
+byte_diff2(const char* a, size_t alen, const char* b, size_t blen) {
+
+  if(alen < blen)
+    return -b[alen];
+
+  if(blen < alen)
+    return a[blen];
+
+  return byte_diff(a, alen, b);
+}
+
+static inline size_t
+str_chr(const char* in, char needle) {
+  const char *t, c = needle;
+
+  for(t = in; *t; ++t)
+    if(*t == c)
+      break;
+
+  return (size_t)(t - in);
+}
+
+static inline size_t
+str_chrs(const char* in, const char needles[], size_t nn) {
+  const char* t;
+  size_t i;
+
+  for(t = in; *t; ++t)
+    for(i = 0; i < nn; i++)
+      if(*t == needles[i])
+        return (size_t)(t - in);
+
+  return (size_t)(t - in);
+}
+
+static inline size_t
+str_rchr(const char* s, char needle) {
+  const char *in, *found = 0;
+
+  for(in = s; *in; ++in)
+    if(*in == needle)
+      found = in;
+
+  return (size_t)((found ? found : in) - s);
+}
+
+static inline size_t
+str_rchrs(const char* in, const char needles[], size_t nn) {
+  const char *s, *found = 0;
+  size_t i;
+
+  for(s = in; *s; ++s)
+    for(i = 0; i < nn; ++i)
+      if(*s == needles[i])
+        found = s;
+
+  return (size_t)((found ? found : s) - in);
+}
+
+static inline int
+str_endb(const char* a, const char* x, size_t n) {
+  size_t alen = strlen(a);
+  a += alen - n;
+
+  return alen >= n && !memcmp(a, x, n);
+}
+
+/* str_ends returns 1 if the b is a suffix of a, 0 otherwise */
+static inline int
+str_ends(const char* a, const char* b) {
+  return str_endb(a, b, strlen(b));
+}
+
+static inline int
+str_startb(const char* a, const char* x, size_t len) {
+  size_t i;
+
+  for(i = 0;; i++) {
+    if(i == len)
+      return 1;
+    if(a[i] != x[i])
+      break;
+  }
+
+  return 0;
+}
+
+static inline int
+str_start(const char* a, const char* b) {
+  return str_startb(a, b, strlen(b));
+}
+
+#define str_contains(s, needle) (!!strchr((s), (needle)))
+
+char* str_escape(const char*);
+
+static inline size_t
+str_count(const char* s, char c) {
+  size_t i, count = 0;
+
+  for(i = 0; s[i]; i++)
+    if(s[i] == c)
+      ++count;
+
+  return count;
+}
+
+static inline size_t
+str_copy(char* out, const char* in) {
+  char* s;
+
+  for(s = out; (*s = *in); ++s)
+    ++in;
+
+  return (size_t)(s - out);
+}
+
+static inline size_t
+str_copyn(char* out, const char* in, size_t n) {
+  char* s;
+
+  for(s = out; n-- && (*s = *in); ++s)
+    ++in;
+
+  *s = '\0';
+  return (size_t)(s - out);
+}
+
+static inline char*
+str_ndup(const char* s, size_t n) {
+  char* r = malloc(n + 1);
+
+  if(r == NULL)
+    return NULL;
+
+  memcpy(r, s, n);
+  r[n] = '\0';
+  return r;
+}
+
+size_t str_findb(const char*, const char*, size_t);
+size_t str_find(const void*, const void*);
+
+static inline size_t
+predicate_find(const char* str, size_t len, int (*pred)(int32_t)) {
+  size_t pos;
+
+  for(pos = 0; pos < len; pos++)
+    if(pred(str[pos]))
+      break;
+
+  return pos;
+}
+
+static inline size_t
+lookup_find(const char* str, size_t len, const char table[256]) {
+  size_t pos;
+
+  for(pos = 0; pos < len; pos++)
+    if(table[(unsigned char)str[pos]])
+      break;
+
+  return pos;
+}
+
+static inline char
+escape_char_letter(char c) {
+  switch(c) {
+    case '\0': return '0';
+    case '\a': return 'a';
+    case '\b': return 'b';
+    case '\t': return 't';
+    case '\n': return 'n';
+    case '\v': return 'v';
+    case '\f': return 'f';
+    case '\r': return 'r';
+    case '\\': return '\\';
+    case '\'': return '\'';
+  }
+
+  return 0;
+}
+
+#define FMT_LONG 41  /* enough space to hold -2^127 in decimal, plus \0 */
+#define FMT_ULONG 40 /* enough space to hold 2^128 - 1 in decimal, plus \0 */
+#define FMT_8LONG 44 /* enough space to hold 2^128 - 1 in octal, plus \0 */
+#define FMT_XLONG 33 /* enough space to hold 2^128 - 1 in hexadecimal, plus \0 */
+
+size_t token_length(const char*, size_t, char delim);
+size_t fmt_ulong(char*, uint32_t);
+size_t scan_ushort(const char*, uint16_t*);
+size_t fmt_longlong(char*, int64_t);
+size_t fmt_ulonglong(char*, uint64_t);
+size_t fmt_xlonglong(char*, uint64_t);
+size_t fmt_xlonglong0(char*, uint64_t, size_t);
+size_t fmt_8long(char* dest, uint32_t i);
+size_t fmt_xlong(char* dest, uint32_t num);
+size_t fmt_xlong0(char* dest, uint32_t num, size_t n);
+size_t scan_longlong(const char*, int64_t*);
+size_t scan_int(const char*, int32_t*);
+size_t scan_uint(const char*, uint32_t*);
+size_t scan_ulonglong(const char*, uint64_t*);
+size_t scan_xlonglong(const char*, uint64_t*);
+size_t scan_8longn(const char*, size_t, uint32_t* dest);
+size_t scan_whitenskip(const char*, size_t);
+size_t scan_nonwhitenskip(const char*, size_t);
+size_t scan_line(const char*, size_t);
+size_t scan_lineskip(const char*, size_t);
+size_t scan_lineskip_escaped(const char*, size_t);
+size_t scan_eolskip(const char*, size_t);
+size_t utf8_strlen(const void*, size_t);
+wchar_t* utf8_towcs(const char*);
+char* utf8_fromwcs(const wchar_t*);
+BOOL utf16_multiword(const void*);
+int case_lowerc(int);
+int case_starts(const char*, const char*);
+int case_diffb(const void*, size_t, const void* T);
+size_t case_findb(const void*, size_t, const void* what, size_t wlen);
+size_t case_finds(const void*, const char*);
+
+static inline int
+scan_fromhex(unsigned char c) {
+  c -= '0';
+
+  if(c <= 9)
+    return c;
+
+  c &= ~0x20;
+  c -= 'A' - '0';
+
+  if(c < 6)
+    return c + 10;
+
+  return -1;
+}
+
+static inline size_t
+scan_8long(const char* src, uint32_t* dest) {
+  return scan_8longn(src, (size_t)-1, dest);
+}
+
+static inline size_t
+utf8_charlen(const char* in, size_t len) {
+  const uint8_t* next = (const void*)in;
+  int r = unicode_from_utf8((const uint8_t*)in, len, &next);
+  return r == -1 ? 0 : next - (const uint8_t*)in;
+}
+
+static inline int
+utf8_charcode(const char* in, size_t len) {
+  const uint8_t* next = (const void*)in;
+  int r = unicode_from_utf8((const uint8_t*)in, len, &next);
+  return next > in ? r : -1;
+}
+
+BOOL utf16_multiword(const void*);
+
+ssize_t write_file(const char* file, const void* buf, size_t len);
+ssize_t puts_file(const char* file, const char* s);
+
+size_t u64toa(char*, uint64_t num, int base);
+size_t i64toa(char*, int64_t num, int base);
+
+/**
+ * @}
+ */
+#endif /* defined(CHAR_UTILS_H) */

src/quickjs/cutils.c

@@ -0,0 +1,631 @@
+/*
+ * C utilities
+ *
+ * Copyright (c) 2017 Fabrice Bellard
+ * Copyright (c) 2018 Charlie Gordon
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+
+#include "cutils.h"
+
+void pstrcpy(char *buf, int buf_size, const char *str)
+{
+    int c;
+    char *q = buf;
+
+    if (buf_size <= 0)
+        return;
+
+    for(;;) {
+        c = *str++;
+        if (c == 0 || q >= buf + buf_size - 1)
+            break;
+        *q++ = c;
+    }
+    *q = '\0';
+}
+
+/* strcat and truncate. */
+char *pstrcat(char *buf, int buf_size, const char *s)
+{
+    int len;
+    len = strlen(buf);
+    if (len < buf_size)
+        pstrcpy(buf + len, buf_size - len, s);
+    return buf;
+}
+
+int strstart(const char *str, const char *val, const char **ptr)
+{
+    const char *p, *q;
+    p = str;
+    q = val;
+    while (*q != '\0') {
+        if (*p != *q)
+            return 0;
+        p++;
+        q++;
+    }
+    if (ptr)
+        *ptr = p;
+    return 1;
+}
+
+int has_suffix(const char *str, const char *suffix)
+{
+    size_t len = strlen(str);
+    size_t slen = strlen(suffix);
+    return (len >= slen && !memcmp(str + len - slen, suffix, slen));
+}
+
+/* Dynamic buffer package */
+
+static void *dbuf_default_realloc(void *opaque, void *ptr, size_t size)
+{
+    return realloc(ptr, size);
+}
+
+void dbuf_init2(DynBuf *s, void *opaque, DynBufReallocFunc *realloc_func)
+{
+    memset(s, 0, sizeof(*s));
+    if (!realloc_func)
+        realloc_func = dbuf_default_realloc;
+    s->opaque = opaque;
+    s->realloc_func = realloc_func;
+}
+
+void dbuf_init(DynBuf *s)
+{
+    dbuf_init2(s, NULL, NULL);
+}
+
+/* return < 0 if error */
+int dbuf_realloc(DynBuf *s, size_t new_size)
+{
+    size_t size;
+    uint8_t *new_buf;
+    if (new_size > s->allocated_size) {
+        if (s->error)
+            return -1;
+        size = s->allocated_size * 3 / 2;
+        if (size > new_size)
+            new_size = size;
+        new_buf = s->realloc_func(s->opaque, s->buf, new_size);
+        if (!new_buf) {
+            s->error = TRUE;
+            return -1;
+        }
+        s->buf = new_buf;
+        s->allocated_size = new_size;
+    }
+    return 0;
+}
+
+int dbuf_write(DynBuf *s, size_t offset, const uint8_t *data, size_t len)
+{
+    size_t end;
+    end = offset + len;
+    if (dbuf_realloc(s, end))
+        return -1;
+    memcpy(s->buf + offset, data, len);
+    if (end > s->size)
+        s->size = end;
+    return 0;
+}
+
+int dbuf_put(DynBuf *s, const uint8_t *data, size_t len)
+{
+    if (unlikely((s->size + len) > s->allocated_size)) {
+        if (dbuf_realloc(s, s->size + len))
+            return -1;
+    }
+    memcpy_no_ub(s->buf + s->size, data, len);
+    s->size += len;
+    return 0;
+}
+
+int dbuf_put_self(DynBuf *s, size_t offset, size_t len)
+{
+    if (unlikely((s->size + len) > s->allocated_size)) {
+        if (dbuf_realloc(s, s->size + len))
+            return -1;
+    }
+    memcpy(s->buf + s->size, s->buf + offset, len);
+    s->size += len;
+    return 0;
+}
+
+int dbuf_putc(DynBuf *s, uint8_t c)
+{
+    return dbuf_put(s, &c, 1);
+}
+
+int dbuf_putstr(DynBuf *s, const char *str)
+{
+    return dbuf_put(s, (const uint8_t *)str, strlen(str));
+}
+
+int __attribute__((format(printf, 2, 3))) dbuf_printf(DynBuf *s,
+                                                      const char *fmt, ...)
+{
+    va_list ap;
+    char buf[128];
+    int len;
+
+    va_start(ap, fmt);
+    len = vsnprintf(buf, sizeof(buf), fmt, ap);
+    va_end(ap);
+    if (len < sizeof(buf)) {
+        /* fast case */
+        return dbuf_put(s, (uint8_t *)buf, len);
+    } else {
+        if (dbuf_realloc(s, s->size + len + 1))
+            return -1;
+        va_start(ap, fmt);
+        vsnprintf((char *)(s->buf + s->size), s->allocated_size - s->size,
+                  fmt, ap);
+        va_end(ap);
+        s->size += len;
+    }
+    return 0;
+}
+
+void dbuf_free(DynBuf *s)
+{
+    /* we test s->buf as a fail safe to avoid crashing if dbuf_free()
+       is called twice */
+    if (s->buf) {
+        s->realloc_func(s->opaque, s->buf, 0);
+    }
+    memset(s, 0, sizeof(*s));
+}
+
+/* Note: at most 31 bits are encoded. At most UTF8_CHAR_LEN_MAX bytes
+   are output. */
+int unicode_to_utf8(uint8_t *buf, unsigned int c)
+{
+    uint8_t *q = buf;
+
+    if (c < 0x80) {
+        *q++ = c;
+    } else {
+        if (c < 0x800) {
+            *q++ = (c >> 6) | 0xc0;
+        } else {
+            if (c < 0x10000) {
+                *q++ = (c >> 12) | 0xe0;
+            } else {
+                if (c < 0x00200000) {
+                    *q++ = (c >> 18) | 0xf0;
+                } else {
+                    if (c < 0x04000000) {
+                        *q++ = (c >> 24) | 0xf8;
+                    } else if (c < 0x80000000) {
+                        *q++ = (c >> 30) | 0xfc;
+                        *q++ = ((c >> 24) & 0x3f) | 0x80;
+                    } else {
+                        return 0;
+                    }
+                    *q++ = ((c >> 18) & 0x3f) | 0x80;
+                }
+                *q++ = ((c >> 12) & 0x3f) | 0x80;
+            }
+            *q++ = ((c >> 6) & 0x3f) | 0x80;
+        }
+        *q++ = (c & 0x3f) | 0x80;
+    }
+    return q - buf;
+}
+
+static const unsigned int utf8_min_code[5] = {
+    0x80, 0x800, 0x10000, 0x00200000, 0x04000000,
+};
+
+static const unsigned char utf8_first_code_mask[5] = {
+    0x1f, 0xf, 0x7, 0x3, 0x1,
+};
+
+/* return -1 if error. *pp is not updated in this case. max_len must
+   be >= 1. The maximum length for a UTF8 byte sequence is 6 bytes. */
+int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp)
+{
+    int l, c, b, i;
+
+    c = *p++;
+    if (c < 0x80) {
+        *pp = p;
+        return c;
+    }
+    switch(c) {
+    case 0xc0: case 0xc1: case 0xc2: case 0xc3:
+    case 0xc4: case 0xc5: case 0xc6: case 0xc7:
+    case 0xc8: case 0xc9: case 0xca: case 0xcb:
+    case 0xcc: case 0xcd: case 0xce: case 0xcf:
+    case 0xd0: case 0xd1: case 0xd2: case 0xd3:
+    case 0xd4: case 0xd5: case 0xd6: case 0xd7:
+    case 0xd8: case 0xd9: case 0xda: case 0xdb:
+    case 0xdc: case 0xdd: case 0xde: case 0xdf:
+        l = 1;
+        break;
+    case 0xe0: case 0xe1: case 0xe2: case 0xe3:
+    case 0xe4: case 0xe5: case 0xe6: case 0xe7:
+    case 0xe8: case 0xe9: case 0xea: case 0xeb:
+    case 0xec: case 0xed: case 0xee: case 0xef:
+        l = 2;
+        break;
+    case 0xf0: case 0xf1: case 0xf2: case 0xf3:
+    case 0xf4: case 0xf5: case 0xf6: case 0xf7:
+        l = 3;
+        break;
+    case 0xf8: case 0xf9: case 0xfa: case 0xfb:
+        l = 4;
+        break;
+    case 0xfc: case 0xfd:
+        l = 5;
+        break;
+    default:
+        return -1;
+    }
+    /* check that we have enough characters */
+    if (l > (max_len - 1))
+        return -1;
+    c &= utf8_first_code_mask[l - 1];
+    for(i = 0; i < l; i++) {
+        b = *p++;
+        if (b < 0x80 || b >= 0xc0)
+            return -1;
+        c = (c << 6) | (b & 0x3f);
+    }
+    if (c < utf8_min_code[l - 1])
+        return -1;
+    *pp = p;
+    return c;
+}
+
+#if 0
+
+#if defined(EMSCRIPTEN) || defined(__ANDROID__)
+
+static void *rqsort_arg;
+static int (*rqsort_cmp)(const void *, const void *, void *);
+
+static int rqsort_cmp2(const void *p1, const void *p2)
+{
+    return rqsort_cmp(p1, p2, rqsort_arg);
+}
+
+/* not reentrant, but not needed with emscripten */
+void rqsort(void *base, size_t nmemb, size_t size,
+            int (*cmp)(const void *, const void *, void *),
+            void *arg)
+{
+    rqsort_arg = arg;
+    rqsort_cmp = cmp;
+    qsort(base, nmemb, size, rqsort_cmp2);
+}
+
+#endif
+
+#else
+
+typedef void (*exchange_f)(void *a, void *b, size_t size);
+typedef int (*cmp_f)(const void *, const void *, void *opaque);
+
+static void exchange_bytes(void *a, void *b, size_t size) {
+    uint8_t *ap = (uint8_t *)a;
+    uint8_t *bp = (uint8_t *)b;
+
+    while (size-- != 0) {
+        uint8_t t = *ap;
+        *ap++ = *bp;
+        *bp++ = t;
+    }
+}
+
+static void exchange_one_byte(void *a, void *b, size_t size) {
+    uint8_t *ap = (uint8_t *)a;
+    uint8_t *bp = (uint8_t *)b;
+    uint8_t t = *ap;
+    *ap = *bp;
+    *bp = t;
+}
+
+static void exchange_int16s(void *a, void *b, size_t size) {
+    uint16_t *ap = (uint16_t *)a;
+    uint16_t *bp = (uint16_t *)b;
+
+    for (size /= sizeof(uint16_t); size-- != 0;) {
+        uint16_t t = *ap;
+        *ap++ = *bp;
+        *bp++ = t;
+    }
+}
+
+static void exchange_one_int16(void *a, void *b, size_t size) {
+    uint16_t *ap = (uint16_t *)a;
+    uint16_t *bp = (uint16_t *)b;
+    uint16_t t = *ap;
+    *ap = *bp;
+    *bp = t;
+}
+
+static void exchange_int32s(void *a, void *b, size_t size) {
+    uint32_t *ap = (uint32_t *)a;
+    uint32_t *bp = (uint32_t *)b;
+
+    for (size /= sizeof(uint32_t); size-- != 0;) {
+        uint32_t t = *ap;
+        *ap++ = *bp;
+        *bp++ = t;
+    }
+}
+
+static void exchange_one_int32(void *a, void *b, size_t size) {
+    uint32_t *ap = (uint32_t *)a;
+    uint32_t *bp = (uint32_t *)b;
+    uint32_t t = *ap;
+    *ap = *bp;
+    *bp = t;
+}
+
+static void exchange_int64s(void *a, void *b, size_t size) {
+    uint64_t *ap = (uint64_t *)a;
+    uint64_t *bp = (uint64_t *)b;
+
+    for (size /= sizeof(uint64_t); size-- != 0;) {
+        uint64_t t = *ap;
+        *ap++ = *bp;
+        *bp++ = t;
+    }
+}
+
+static void exchange_one_int64(void *a, void *b, size_t size) {
+    uint64_t *ap = (uint64_t *)a;
+    uint64_t *bp = (uint64_t *)b;
+    uint64_t t = *ap;
+    *ap = *bp;
+    *bp = t;
+}
+
+static void exchange_int128s(void *a, void *b, size_t size) {
+    uint64_t *ap = (uint64_t *)a;
+    uint64_t *bp = (uint64_t *)b;
+
+    for (size /= sizeof(uint64_t) * 2; size-- != 0; ap += 2, bp += 2) {
+        uint64_t t = ap[0];
+        uint64_t u = ap[1];
+        ap[0] = bp[0];
+        ap[1] = bp[1];
+        bp[0] = t;
+        bp[1] = u;
+    }
+}
+
+static void exchange_one_int128(void *a, void *b, size_t size) {
+    uint64_t *ap = (uint64_t *)a;
+    uint64_t *bp = (uint64_t *)b;
+    uint64_t t = ap[0];
+    uint64_t u = ap[1];
+    ap[0] = bp[0];
+    ap[1] = bp[1];
+    bp[0] = t;
+    bp[1] = u;
+}
+
+static inline exchange_f exchange_func(const void *base, size_t size) {
+    switch (((uintptr_t)base | (uintptr_t)size) & 15) {
+    case 0:
+        if (size == sizeof(uint64_t) * 2)
+            return exchange_one_int128;
+        else
+            return exchange_int128s;
+    case 8:
+        if (size == sizeof(uint64_t))
+            return exchange_one_int64;
+        else
+            return exchange_int64s;
+    case 4:
+    case 12:
+        if (size == sizeof(uint32_t))
+            return exchange_one_int32;
+        else
+            return exchange_int32s;
+    case 2:
+    case 6:
+    case 10:
+    case 14:
+        if (size == sizeof(uint16_t))
+            return exchange_one_int16;
+        else
+            return exchange_int16s;
+    default:
+        if (size == 1)
+            return exchange_one_byte;
+        else
+            return exchange_bytes;
+    }
+}
+
+static void heapsortx(void *base, size_t nmemb, size_t size, cmp_f cmp, void *opaque)
+{
+    uint8_t *basep = (uint8_t *)base;
+    size_t i, n, c, r;
+    exchange_f swap = exchange_func(base, size);
+
+    if (nmemb > 1) {
+        i = (nmemb / 2) * size;
+        n = nmemb * size;
+
+        while (i > 0) {
+            i -= size;
+            for (r = i; (c = r * 2 + size) < n; r = c) {
+                if (c < n - size && cmp(basep + c, basep + c + size, opaque) <= 0)
+                    c += size;
+                if (cmp(basep + r, basep + c, opaque) > 0)
+                    break;
+                swap(basep + r, basep + c, size);
+            }
+        }
+        for (i = n - size; i > 0; i -= size) {
+            swap(basep, basep + i, size);
+
+            for (r = 0; (c = r * 2 + size) < i; r = c) {
+                if (c < i - size && cmp(basep + c, basep + c + size, opaque) <= 0)
+                    c += size;
+                if (cmp(basep + r, basep + c, opaque) > 0)
+                    break;
+                swap(basep + r, basep + c, size);
+            }
+        }
+    }
+}
+
+static inline void *med3(void *a, void *b, void *c, cmp_f cmp, void *opaque)
+{
+    return cmp(a, b, opaque) < 0 ?
+        (cmp(b, c, opaque) < 0 ? b : (cmp(a, c, opaque) < 0 ? c : a )) :
+        (cmp(b, c, opaque) > 0 ? b : (cmp(a, c, opaque) < 0 ? a : c ));
+}
+
+/* pointer based version with local stack and insertion sort threshhold */
+void rqsort(void *base, size_t nmemb, size_t size, cmp_f cmp, void *opaque)
+{
+    struct { uint8_t *base; size_t count; int depth; } stack[50], *sp = stack;
+    uint8_t *ptr, *pi, *pj, *plt, *pgt, *top, *m;
+    size_t m4, i, lt, gt, span, span2;
+    int c, depth;
+    exchange_f swap = exchange_func(base, size);
+    exchange_f swap_block = exchange_func(base, size | 128);
+
+    if (nmemb < 2 || size <= 0)
+        return;
+
+    sp->base = (uint8_t *)base;
+    sp->count = nmemb;
+    sp->depth = 0;
+    sp++;
+
+    while (sp > stack) {
+        sp--;
+        ptr = sp->base;
+        nmemb = sp->count;
+        depth = sp->depth;
+
+        while (nmemb > 6) {
+            if (++depth > 50) {
+                /* depth check to ensure worst case logarithmic time */
+                heapsortx(ptr, nmemb, size, cmp, opaque);
+                nmemb = 0;
+                break;
+            }
+            /* select median of 3 from 1/4, 1/2, 3/4 positions */
+            /* should use median of 5 or 9? */
+            m4 = (nmemb >> 2) * size;
+            m = med3(ptr + m4, ptr + 2 * m4, ptr + 3 * m4, cmp, opaque);
+            swap(ptr, m, size);  /* move the pivot to the start or the array */
+            i = lt = 1;
+            pi = plt = ptr + size;
+            gt = nmemb;
+            pj = pgt = top = ptr + nmemb * size;
+            for (;;) {
+                while (pi < pj && (c = cmp(ptr, pi, opaque)) >= 0) {
+                    if (c == 0) {
+                        swap(plt, pi, size);
+                        lt++;
+                        plt += size;
+                    }
+                    i++;
+                    pi += size;
+                }
+                while (pi < (pj -= size) && (c = cmp(ptr, pj, opaque)) <= 0) {
+                    if (c == 0) {
+                        gt--;
+                        pgt -= size;
+                        swap(pgt, pj, size);
+                    }
+                }
+                if (pi >= pj)
+                    break;
+                swap(pi, pj, size);
+                i++;
+                pi += size;
+            }
+            /* array has 4 parts:
+             * from 0 to lt excluded: elements identical to pivot
+             * from lt to pi excluded: elements smaller than pivot
+             * from pi to gt excluded: elements greater than pivot
+             * from gt to n excluded: elements identical to pivot
+             */
+            /* move elements identical to pivot in the middle of the array: */
+            /* swap values in ranges [0..lt[ and [i-lt..i[
+               swapping the smallest span between lt and i-lt is sufficient
+             */
+            span = plt - ptr;
+            span2 = pi - plt;
+            lt = i - lt;
+            if (span > span2)
+                span = span2;
+            swap_block(ptr, pi - span, span);
+            /* swap values in ranges [gt..top[ and [i..top-(top-gt)[
+               swapping the smallest span between top-gt and gt-i is sufficient
+             */
+            span = top - pgt;
+            span2 = pgt - pi;
+            pgt = top - span2;
+            gt = nmemb - (gt - i);
+            if (span > span2)
+                span = span2;
+            swap_block(pi, top - span, span);
+
+            /* now array has 3 parts:
+             * from 0 to lt excluded: elements smaller than pivot
+             * from lt to gt excluded: elements identical to pivot
+             * from gt to n excluded: elements greater than pivot
+             */
+            /* stack the larger segment and keep processing the smaller one
+               to minimize stack use for pathological distributions */
+            if (lt > nmemb - gt) {
+                sp->base = ptr;
+                sp->count = lt;
+                sp->depth = depth;
+                sp++;
+                ptr = pgt;
+                nmemb -= gt;
+            } else {
+                sp->base = pgt;
+                sp->count = nmemb - gt;
+                sp->depth = depth;
+                sp++;
+                nmemb = lt;
+            }
+        }
+        /* Use insertion sort for small fragments */
+        for (pi = ptr + size, top = ptr + nmemb * size; pi < top; pi += size) {
+            for (pj = pi; pj > ptr && cmp(pj - size, pj, opaque) > 0; pj -= size)
+                swap(pj, pj - size, size);
+        }
+    }
+}
+
+#endif

src/quickjs/cutils.h

@@ -0,0 +1,341 @@
+/*
+ * C utilities
+ *
+ * Copyright (c) 2017 Fabrice Bellard
+ * Copyright (c) 2018 Charlie Gordon
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef CUTILS_H
+#define CUTILS_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+
+#define likely(x)       __builtin_expect(!!(x), 1)
+#define unlikely(x)     __builtin_expect(!!(x), 0)
+#define force_inline inline __attribute__((always_inline))
+#define no_inline __attribute__((noinline))
+#define __maybe_unused __attribute__((unused))
+
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)    tostring(s)
+#define tostring(s)     #s
+
+#ifndef offsetof
+#define offsetof(type, field) ((size_t) &((type *)0)->field)
+#endif
+#ifndef countof
+#define countof(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+#ifndef container_of
+/* return the pointer of type 'type *' containing 'ptr' as field 'member' */
+#define container_of(ptr, type, member) ((type *)((uint8_t *)(ptr) - offsetof(type, member)))
+#endif
+
+typedef int BOOL;
+
+#ifndef FALSE
+enum {
+    FALSE = 0,
+    TRUE = 1,
+};
+#endif
+
+void pstrcpy(char *buf, int buf_size, const char *str);
+char *pstrcat(char *buf, int buf_size, const char *s);
+int strstart(const char *str, const char *val, const char **ptr);
+int has_suffix(const char *str, const char *suffix);
+
+/* Prevent UB when n == 0 and (src == NULL or dest == NULL) */
+static inline void memcpy_no_ub(void *dest, const void *src, size_t n) {
+    if (n)
+        memcpy(dest, src, n);
+}
+
+static inline int max_int(int a, int b)
+{
+    if (a > b)
+        return a;
+    else
+        return b;
+}
+
+static inline int min_int(int a, int b)
+{
+    if (a < b)
+        return a;
+    else
+        return b;
+}
+
+static inline uint32_t max_uint32(uint32_t a, uint32_t b)
+{
+    if (a > b)
+        return a;
+    else
+        return b;
+}
+
+static inline uint32_t min_uint32(uint32_t a, uint32_t b)
+{
+    if (a < b)
+        return a;
+    else
+        return b;
+}
+
+static inline int64_t max_int64(int64_t a, int64_t b)
+{
+    if (a > b)
+        return a;
+    else
+        return b;
+}
+
+static inline int64_t min_int64(int64_t a, int64_t b)
+{
+    if (a < b)
+        return a;
+    else
+        return b;
+}
+
+/* WARNING: undefined if a = 0 */
+static inline int clz32(unsigned int a)
+{
+    return __builtin_clz(a);
+}
+
+/* WARNING: undefined if a = 0 */
+static inline int clz64(uint64_t a)
+{
+    return __builtin_clzll(a);
+}
+
+/* WARNING: undefined if a = 0 */
+static inline int ctz32(unsigned int a)
+{
+    return __builtin_ctz(a);
+}
+
+/* WARNING: undefined if a = 0 */
+static inline int ctz64(uint64_t a)
+{
+    return __builtin_ctzll(a);
+}
+
+struct __attribute__((packed)) packed_u64 {
+    uint64_t v;
+};
+
+struct __attribute__((packed)) packed_u32 {
+    uint32_t v;
+};
+
+struct __attribute__((packed)) packed_u16 {
+    uint16_t v;
+};
+
+static inline uint64_t get_u64(const uint8_t *tab)
+{
+    return ((const struct packed_u64 *)tab)->v;
+}
+
+static inline int64_t get_i64(const uint8_t *tab)
+{
+    return (int64_t)((const struct packed_u64 *)tab)->v;
+}
+
+static inline void put_u64(uint8_t *tab, uint64_t val)
+{
+    ((struct packed_u64 *)tab)->v = val;
+}
+
+static inline uint32_t get_u32(const uint8_t *tab)
+{
+    return ((const struct packed_u32 *)tab)->v;
+}
+
+static inline int32_t get_i32(const uint8_t *tab)
+{
+    return (int32_t)((const struct packed_u32 *)tab)->v;
+}
+
+static inline void put_u32(uint8_t *tab, uint32_t val)
+{
+    ((struct packed_u32 *)tab)->v = val;
+}
+
+static inline uint32_t get_u16(const uint8_t *tab)
+{
+    return ((const struct packed_u16 *)tab)->v;
+}
+
+static inline int32_t get_i16(const uint8_t *tab)
+{
+    return (int16_t)((const struct packed_u16 *)tab)->v;
+}
+
+static inline void put_u16(uint8_t *tab, uint16_t val)
+{
+    ((struct packed_u16 *)tab)->v = val;
+}
+
+static inline uint32_t get_u8(const uint8_t *tab)
+{
+    return *tab;
+}
+
+static inline int32_t get_i8(const uint8_t *tab)
+{
+    return (int8_t)*tab;
+}
+
+static inline void put_u8(uint8_t *tab, uint8_t val)
+{
+    *tab = val;
+}
+
+#ifndef bswap16
+static inline uint16_t bswap16(uint16_t x)
+{
+    return (x >> 8) | (x << 8);
+}
+#endif
+
+#ifndef bswap32
+static inline uint32_t bswap32(uint32_t v)
+{
+    return ((v & 0xff000000) >> 24) | ((v & 0x00ff0000) >>  8) |
+        ((v & 0x0000ff00) <<  8) | ((v & 0x000000ff) << 24);
+}
+#endif
+
+#ifndef bswap64
+static inline uint64_t bswap64(uint64_t v)
+{
+    return ((v & ((uint64_t)0xff << (7 * 8))) >> (7 * 8)) |
+        ((v & ((uint64_t)0xff << (6 * 8))) >> (5 * 8)) |
+        ((v & ((uint64_t)0xff << (5 * 8))) >> (3 * 8)) |
+        ((v & ((uint64_t)0xff << (4 * 8))) >> (1 * 8)) |
+        ((v & ((uint64_t)0xff << (3 * 8))) << (1 * 8)) |
+        ((v & ((uint64_t)0xff << (2 * 8))) << (3 * 8)) |
+        ((v & ((uint64_t)0xff << (1 * 8))) << (5 * 8)) |
+        ((v & ((uint64_t)0xff << (0 * 8))) << (7 * 8));
+}
+#endif
+
+/* XXX: should take an extra argument to pass slack information to the caller */
+typedef void *DynBufReallocFunc(void *opaque, void *ptr, size_t size);
+
+typedef struct DynBuf {
+    uint8_t *buf;
+    size_t size;
+    size_t allocated_size;
+    BOOL error; /* true if a memory allocation error occurred */
+    DynBufReallocFunc *realloc_func;
+    void *opaque; /* for realloc_func */
+} DynBuf;
+
+void dbuf_init(DynBuf *s);
+void dbuf_init2(DynBuf *s, void *opaque, DynBufReallocFunc *realloc_func);
+int dbuf_realloc(DynBuf *s, size_t new_size);
+int dbuf_write(DynBuf *s, size_t offset, const uint8_t *data, size_t len);
+int dbuf_put(DynBuf *s, const uint8_t *data, size_t len);
+int dbuf_put_self(DynBuf *s, size_t offset, size_t len);
+int dbuf_putc(DynBuf *s, uint8_t c);
+int dbuf_putstr(DynBuf *s, const char *str);
+static inline int dbuf_put_u16(DynBuf *s, uint16_t val)
+{
+    return dbuf_put(s, (uint8_t *)&val, 2);
+}
+static inline int dbuf_put_u32(DynBuf *s, uint32_t val)
+{
+    return dbuf_put(s, (uint8_t *)&val, 4);
+}
+static inline int dbuf_put_u64(DynBuf *s, uint64_t val)
+{
+    return dbuf_put(s, (uint8_t *)&val, 8);
+}
+int __attribute__((format(printf, 2, 3))) dbuf_printf(DynBuf *s,
+                                                      const char *fmt, ...);
+void dbuf_free(DynBuf *s);
+static inline BOOL dbuf_error(DynBuf *s) {
+    return s->error;
+}
+static inline void dbuf_set_error(DynBuf *s)
+{
+    s->error = TRUE;
+}
+
+#define UTF8_CHAR_LEN_MAX 6
+
+int unicode_to_utf8(uint8_t *buf, unsigned int c);
+int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp);
+
+static inline BOOL is_surrogate(uint32_t c)
+{
+    return (c >> 11) == (0xD800 >> 11); // 0xD800-0xDFFF
+}
+
+static inline BOOL is_hi_surrogate(uint32_t c)
+{
+    return (c >> 10) == (0xD800 >> 10); // 0xD800-0xDBFF
+}
+
+static inline BOOL is_lo_surrogate(uint32_t c)
+{
+    return (c >> 10) == (0xDC00 >> 10); // 0xDC00-0xDFFF
+}
+
+static inline uint32_t get_hi_surrogate(uint32_t c)
+{
+    return (c >> 10) - (0x10000 >> 10) + 0xD800;
+}
+
+static inline uint32_t get_lo_surrogate(uint32_t c)
+{
+    return (c & 0x3FF) | 0xDC00;
+}
+
+static inline uint32_t from_surrogate(uint32_t hi, uint32_t lo)
+{
+    return 0x10000 + 0x400 * (hi - 0xD800) + (lo - 0xDC00);
+}
+
+static inline int from_hex(int c)
+{
+    if (c >= '0' && c <= '9')
+        return c - '0';
+    else if (c >= 'A' && c <= 'F')
+        return c - 'A' + 10;
+    else if (c >= 'a' && c <= 'f')
+        return c - 'a' + 10;
+    else
+        return -1;
+}
+
+void rqsort(void *base, size_t nmemb, size_t size,
+            int (*cmp)(const void *, const void *, void *),
+            void *arg);
+
+#endif  /* CUTILS_H */

src/quickjs/debug.c

@@ -0,0 +1,483 @@
+#define _IN_DEBUG_C 1
+#include "debug.h"
+#include <quickjs.h>
+#include <list.h>
+#include <cutils.h>
+#include <assert.h>
+#include "defines.h"
+#include <string.h>
+#include <stdlib.h>
+
+/**
+ * \addtogroup debug
+ * @{
+ */
+struct alloc_block {
+  struct list_head link;
+  const char* file;
+  int line;
+  size_t size;
+};
+
+#define ALLOC_BLOCK_SIZE sizeof(struct alloc_block)
+#define ALLOC_BLOCK(p) (((struct alloc_block*)(p)) - 1)
+#define ALLOC_PTR struct alloc_block*
+
+#undef malloc
+#undef calloc
+#undef realloc
+#undef strdup
+#undef free
+#undef js_malloc
+#undef js_mallocz
+#undef js_realloc
+#undef js_strdup
+#undef js_strndup
+#undef js_free
+#undef js_malloc_usable_size
+#undef js_malloc_rt
+#undef js_mallocz_rt
+#undef js_realloc_rt
+#undef js_free_rt
+#undef js_malloc_usable_size_rt
+
+thread_local struct list_head alloc_block_list = {0, 0};
+
+static inline void
+add_to_list(struct list_head* el, struct list_head* head) {
+  if(alloc_block_list.prev == 0 && alloc_block_list.next == 0)
+    init_list_head(&alloc_block_list);
+
+  list_add_tail(el, head);
+}
+
+int64_t
+check_pointer(void* p) {
+  ALLOC_PTR ptr = ALLOC_BLOCK(p);
+  struct list_head* link;
+  int64_t ret = 0;
+  list_for_each(link, &alloc_block_list) {
+    if(link == &ptr->link)
+      return ret;
+    ret++;
+  }
+  // assert(0);
+  return -1;
+}
+
+void*
+debug_malloc(size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = malloc(n + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_calloc(size_t m, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  m *= n;
+  n = 1;
+
+  if((ptr = calloc(m + ALLOC_BLOCK_SIZE, 1))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_realloc(void* p, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if(p) {
+
+    check_pointer(p);
+
+    ptr = ALLOC_BLOCK(p);
+    list_del(&ptr->link);
+
+    if(n == 0) {
+      free(ptr);
+      return 0;
+    }
+
+    ptr = realloc(ptr, n + ALLOC_BLOCK_SIZE);
+  } else {
+    ptr = malloc(n + ALLOC_BLOCK_SIZE);
+  }
+
+  if(ptr) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_strdup(const char* s, const char* file, int line) {
+  ALLOC_PTR ptr;
+  size_t len = strlen(s);
+
+  if((ptr = malloc(len + 1 + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = len + 1;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+
+    memcpy(&ptr[1], s, len + 1);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void
+debug_free(void* p, const char* file, int line) {
+  ALLOC_PTR ptr = ALLOC_BLOCK(p);
+
+  list_del(&ptr->link);
+
+  memset(ptr, 0xff, ALLOC_BLOCK_SIZE);
+
+  free(ptr);
+}
+
+void*
+debug_js_malloc(JSContext* ctx, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = js_malloc(ctx, n + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_js_mallocz(JSContext* ctx, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = js_mallocz(ctx, n + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_js_realloc(JSContext* ctx, void* p, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if(p) {
+
+    check_pointer(p);
+
+    ptr = ALLOC_BLOCK(p);
+    list_del(&ptr->link);
+
+    if(n == 0) {
+      js_free(ctx, ptr);
+      return 0;
+    }
+
+    ptr = js_realloc(ctx, ptr, n + ALLOC_BLOCK_SIZE);
+  } else {
+    ptr = js_malloc(ctx, n + ALLOC_BLOCK_SIZE);
+  }
+
+  if(ptr) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_js_realloc2(JSContext* ctx, void* p, size_t size, size_t* pslack, const char* file, int line) {
+  void* ptr;
+
+  if((ptr = debug_js_realloc(ctx, p, size, file, line))) {
+    if(pslack) {
+      size_t new_size = debug_js_malloc_usable_size(ctx, ptr, file, line);
+      *pslack = (new_size > size) ? new_size - size : 0;
+    }
+  }
+  return ptr;
+}
+
+void*
+debug_js_strdup(JSContext* ctx, const char* s, const char* file, int line) {
+  ALLOC_PTR ptr;
+  size_t len = strlen(s);
+
+  if((ptr = js_malloc(ctx, len + 1 + ALLOC_BLOCK_SIZE))) {
+    char* p = (char*)&ptr[1];
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = len + 1;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+
+    memcpy(p, s, len + 1);
+    return p;
+  }
+  return 0;
+}
+
+void*
+debug_js_strndup(JSContext* ctx, const char* s, size_t len, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = js_malloc(ctx, len + 1 + ALLOC_BLOCK_SIZE))) {
+    char* p = (char*)&ptr[1];
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = len + 1;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+
+    memcpy(p, s, len);
+    p[len] = '\0';
+
+    return p;
+  }
+  return 0;
+}
+
+size_t
+debug_js_malloc_usable_size(JSContext* ctx, const void* p, const char* file, int line) {
+  ALLOC_PTR ptr = ALLOC_BLOCK(p);
+
+  return js_malloc_usable_size(ctx, ptr) - ALLOC_BLOCK_SIZE;
+}
+
+void
+debug_js_free(JSContext* ctx, void* p, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  check_pointer(p);
+  ptr = ALLOC_BLOCK(p);
+  list_del(&ptr->link);
+
+  memset(ptr, 0xff, ALLOC_BLOCK_SIZE);
+
+  js_free(ctx, ptr);
+}
+
+void*
+debug_js_malloc_rt(JSRuntime* rt, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = js_malloc_rt(rt, n + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_js_mallocz_rt(JSRuntime* rt, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if((ptr = js_mallocz_rt(rt, n + ALLOC_BLOCK_SIZE))) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+void*
+debug_js_realloc_rt(JSRuntime* rt, void* p, size_t n, const char* file, int line) {
+  ALLOC_PTR ptr;
+
+  if(p) {
+    check_pointer(p);
+
+    ptr = ALLOC_BLOCK(p);
+    list_del(&ptr->link);
+    if(n == 0) {
+      js_free_rt(rt, ptr);
+      return 0;
+    }
+    ptr = js_realloc_rt(rt, ptr, n + ALLOC_BLOCK_SIZE);
+  } else {
+    ptr = js_malloc_rt(rt, n + ALLOC_BLOCK_SIZE);
+  }
+
+  if(ptr) {
+    ptr->file = file;
+    ptr->line = line;
+    ptr->size = n;
+
+    add_to_list(&ptr->link, &alloc_block_list);
+    return &ptr[1];
+  }
+  return 0;
+}
+
+size_t
+debug_js_malloc_usable_size_rt(JSRuntime* rt, const void* p, const char* file, int line) {
+  ALLOC_PTR ptr = ALLOC_BLOCK(p);
+
+  return js_malloc_usable_size_rt(rt, ptr) - ALLOC_BLOCK_SIZE;
+}
+
+void
+debug_js_free_rt(JSRuntime* rt, void* p, const char* file, int line) {
+  ALLOC_PTR ptr;
+  check_pointer(p);
+
+  ptr = ALLOC_BLOCK(p);
+
+  // printf("debug_js_free_rt %p\n", p);
+
+  list_del(&ptr->link);
+
+  memset(ptr, 0xff, ALLOC_BLOCK_SIZE);
+
+  js_free_rt(rt, ptr);
+}
+
+#undef malloc
+#undef calloc
+#undef realloc
+#undef strdup
+#undef free
+#undef js_malloc
+#undef js_mallocz
+#undef js_realloc
+#undef js_strdup
+#undef js_strndup
+#undef js_malloc_usable_size
+#undef js_free
+#undef js_malloc_rt
+#undef js_mallocz_rt
+#undef js_realloc_rt
+#undef js_malloc_usable_size_rt
+#undef js_free_rt
+
+void*
+orig_malloc(size_t size) {
+  return malloc(size);
+}
+
+void*
+orig_calloc(size_t nelem, size_t elemsz) {
+  return calloc(nelem, elemsz);
+}
+
+void*
+orig_realloc(void* ptr, size_t size) {
+  return realloc(ptr, size);
+}
+
+void*
+orig_strdup(const char* str) {
+  return strdup(str);
+}
+
+void
+orig_free(void* ptr) {
+  free(ptr);
+}
+
+void*
+orig_js_malloc(JSContext* ctx, size_t size) {
+  return js_malloc(ctx, size);
+}
+
+void*
+orig_js_mallocz(JSContext* ctx, size_t size) {
+  return js_mallocz(ctx, size);
+}
+
+void*
+orig_js_realloc(JSContext* ctx, void* p, size_t size) {
+  return js_realloc(ctx, p, size);
+}
+
+void*
+orig_js_strdup(JSContext* ctx, const char* str) {
+  return js_strdup(ctx, str);
+}
+
+void*
+orig_js_strndup(JSContext* ctx, const char* str, size_t size) {
+  return js_strndup(ctx, str, size);
+}
+
+size_t
+orig_js_malloc_usable_size(JSContext* ctx, const void* p) {
+  return js_malloc_usable_size(ctx, p);
+}
+
+void
+orig_js_free(JSContext* ctx, void* p) {
+  return js_free(ctx, p);
+}
+
+void*
+orig_js_malloc_rt(JSRuntime* rt, size_t size) {
+  return js_malloc_rt(rt, size);
+}
+
+void*
+orig_js_mallocz_rt(JSRuntime* rt, size_t size) {
+  return js_mallocz_rt(rt, size);
+}
+
+void*
+orig_js_realloc_rt(JSRuntime* rt, void* p, size_t size) {
+  return js_realloc_rt(rt, p, size);
+}
+
+size_t
+orig_js_malloc_usable_size_rt(JSRuntime* rt, const void* p) {
+  return js_malloc_usable_size_rt(rt, p);
+}
+
+void
+orig_js_free_rt(JSRuntime* rt, void* p) {
+  return js_free_rt(rt, p);
+}
+
+/**
+ * @}
+ */

src/quickjs/debug.h

@@ -0,0 +1,165 @@
+#ifndef DEBUG_H
+#define DEBUG_H
+
+#include <quickjs.h>
+#include <cutils.h>
+#include "defines.h"
+
+#ifndef QUICKJS_H
+#error "quickjs.h not included"
+#endif
+
+/**
+ * \defgroup debug debug: Debugging helpers
+ * @{
+ */
+
+extern thread_local struct list_head alloc_block_list;
+
+int64_t check_pointer(void*);
+void* debug_malloc(size_t, const char*, int);
+void* debug_calloc(size_t, size_t, const char*, int line);
+void* debug_realloc(void*, size_t, const char*, int line);
+void* debug_strdup(const char*, const char*, int);
+void debug_free(void*, const char*, int);
+void* debug_js_malloc(JSContext*, size_t, const char*, int line);
+void* debug_js_mallocz(JSContext*, size_t, const char*, int line);
+void* debug_js_realloc(JSContext*, void*, size_t, const char* file, int line);
+void* debug_js_realloc2(JSContext*, void*, size_t, size_t* pslack, const char* file, int line);
+void* debug_js_strdup(JSContext*, const char*, const char*, int line);
+void* debug_js_strndup(JSContext*, const char*, size_t, const char* file, int line);
+size_t debug_js_malloc_usable_size(JSContext*, const void*, const char*, int line);
+void debug_js_free(JSContext*, void*, const char*, int line);
+void* debug_js_malloc_rt(JSRuntime*, size_t, const char*, int line);
+void* debug_js_mallocz_rt(JSRuntime*, size_t, const char*, int line);
+void* debug_js_realloc_rt(JSRuntime*, void*, size_t, const char* file, int line);
+size_t debug_js_malloc_usable_size_rt(JSRuntime*, const void*, const char*, int line);
+void debug_js_free_rt(JSRuntime*, void*, const char*, int line);
+
+#if !defined(_IN_DEBUG_C)
+
+#if defined(DEBUG_ALLOC)
+#define malloc(size) debug_malloc(size, __FILE__, __LINE__)
+#define calloc(nelem, size) debug_calloc(nelem, size, __FILE__, __LINE__)
+#define realloc(ptr, size) debug_realloc(ptr, size, __FILE__, __LINE__)
+#define strdup(str) debug_strdup(str, __FILE__, __LINE__)
+#define free(ptr) debug_free(ptr, __FILE__, __LINE__)
+#define js_malloc(ctx, size) debug_js_malloc(ctx, size, __FILE__, __LINE__)
+#define js_mallocz(ctx, size) debug_js_mallocz(ctx, size, __FILE__, __LINE__)
+#define js_realloc(ctx, ptr, size) debug_js_realloc(ctx, ptr, size, __FILE__, __LINE__)
+#define js_strdup(ctx, str) debug_js_strdup(ctx, str, __FILE__, __LINE__)
+#define js_strndup(ctx, str, len) debug_js_strndup(ctx, str, len, __FILE__, __LINE__)
+#define js_free(ctx, ptr) debug_js_free(ctx, ptr, __FILE__, __LINE__)
+#define js_malloc_usable_size(ctx, ptr) debug_js_malloc_usable_size(ctx, ptr, __FILE__, __LINE__)
+#define js_malloc_rt(rt, size) debug_js_malloc_rt(rt, size, __FILE__, __LINE__)
+#define js_mallocz_rt(rt, size) debug_js_mallocz_rt(rt, size, __FILE__, __LINE__)
+#define js_realloc_rt(rt, ptr, size) debug_js_realloc_rt(rt, ptr, size, __FILE__, __LINE__)
+#define js_malloc_usable_size_rt(rt, ptr) debug_js_malloc_usable_size_rt(rt, ptr, __FILE__, __LINE__)
+#define js_free_rt(rt, ptr) debug_js_free_rt(rt, ptr, __FILE__, __LINE__)
+#endif
+
+#ifdef DEBUG_ALLOC
+#define realloc_helper(name) \
+  void* name(void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return debug_malloc(size, __FILE__, __LINE__); \
+    if(size == 0) \
+      return debug_free(ptr, __FILE__, __LINE__); \
+    return debug_realloc(ptr, size, __FILE__, __LINE__); \
+  }
+#define realloc2_helper(name) \
+  void* name(void* opaque, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return debug_malloc(size, __FILE__, __LINE__); \
+    if(size == 0) { \
+      debug_free(ptr, __FILE__, __LINE__); \
+      return 0; \
+    } \
+    return debug_realloc(ptr, size, __FILE__, __LINE__); \
+  }
+#define js_realloc_helper(name) \
+  void* name(JSContext* ctx, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return debug_js_malloc(ctx, size, __FILE__, __LINE__); \
+    if(size == 0) { \
+      debug_js_free(ctx, ptr, __FILE__, __LINE__); \
+      return 0; \
+    } \
+    return debug_js_realloc(ctx, ptr, size, __FILE__, __LINE__); \
+  }
+#define js_realloc_rt_helper(name) \
+  void* name(JSRuntime* rt, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return debug_js_malloc_rt(rt, size, __FILE__, __LINE__); \
+    if(size == 0) { \
+      debug_js_free_rt(rt, ptr, __FILE__, __LINE__); \
+      return 0; \
+    } \
+    return debug_js_realloc_rt(rt, ptr, size, __FILE__, __LINE__); \
+  }
+#else
+#define realloc_helper(name) \
+  void* name(void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return malloc(size); \
+    if(size == 0) { \
+      free(ptr); \
+      return 0; \
+    } \
+    return realloc(ptr, size); \
+  }
+#define realloc2_helper(name) \
+  void* name(void* opaque, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return malloc(size); \
+    if(size == 0) { \
+      free(ptr); \
+      return 0; \
+    } \
+    return realloc(ptr, size); \
+  }
+#define js_realloc_helper(name) \
+  void* name(JSContext* ctx, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return orig_js_malloc(ctx, size); \
+    if(size == 0) { \
+      orig_js_free(ctx, ptr); \
+      return 0; \
+    } \
+    return orig_js_realloc(ctx, ptr, size); \
+  }
+#define js_realloc_rt_helper(name) \
+  void* name(JSRuntime* rt, void* ptr, size_t size) { \
+    if(ptr == 0) \
+      return orig_js_malloc_rt(rt, size); \
+    if(size == 0) { \
+      orig_js_free_rt(rt, ptr); \
+      return 0; \
+    } \
+    return orig_js_realloc_rt(rt, ptr, size); \
+  }
+#endif
+#endif
+
+void* orig_malloc(size_t);
+void* orig_calloc(size_t, size_t);
+void* orig_realloc(void*, size_t);
+void* orig_strdup(const char*);
+void orig_free(void*);
+void* orig_js_malloc(JSContext*, size_t);
+void* orig_js_mallocz(JSContext*, size_t);
+void* orig_js_realloc(JSContext*, void*, size_t);
+void* orig_js_strdup(JSContext*, const char*);
+void* orig_js_strndup(JSContext*, const char*, size_t);
+size_t orig_js_malloc_usable_size(JSContext*, const void*);
+void orig_js_free(JSContext*, void*);
+void* orig_js_malloc_rt(JSRuntime*, size_t);
+void* orig_js_mallocz_rt(JSRuntime*, size_t);
+void* orig_js_realloc_rt(JSRuntime*, void*, size_t);
+size_t orig_js_malloc_usable_size_rt(JSRuntime*, const void*);
+void orig_js_free_rt(JSRuntime*, void*);
+
+/**
+ * @}
+ */
+#endif /* defined(DEBUG_H) */

src/quickjs/defines.h

@@ -0,0 +1,181 @@
+#ifndef DEFINES_H
+#define DEFINES_H
+
+/**
+ * \defgroup defines defines: Preprocessor definitions
+ * @{
+ */
+
+#ifdef _WIN32
+#include <io.h>
+#define FD_TO_SOCKET(fd) ((SOCKET)_get_osfhandle((fd)))
+#define SOCKET_TO_FD(fh) (_open_osfhandle((intptr_t)(fh), O_RDWR | O_BINARY))
+#else
+#define FD_TO_SOCKET(fd) (fd)
+#define SOCKET_TO_FD(fh) (fh)
+#endif
+
+#ifndef offsetof
+#define offsetof(type, field) ((size_t) & ((type*)0)->field)
+#endif
+
+#ifndef inrange
+#define inrange(value, min, max) ((value) >= (min) && (value) <= (max))
+#endif
+
+#define trim_dotslash(str) (!strncmp((str), "./", 2) ? (str) + 2 : (str))
+
+#ifndef thread_local
+#ifdef _Thread_local
+#define thread_local _Thread_local
+#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__SUNPRO_CC) || defined(__IBMCPP__)
+#define thread_local __thread
+#elif defined(_WIN32)
+#define thread_local __declspec(thread)
+#else
+#error No TLS implementation found.
+#endif
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#define PACK __attribute__((packed))
+#define ENDPACK
+#else
+#define PACK #pragma pack(push, 1)
+#define ENDPACK #pragma pack(pop)
+#endif
+
+#define JS_CGETSET_ENUMERABLE_DEF(prop_name, fgetter, fsetter, magic_num) \
+  { \
+    .name = prop_name, .prop_flags = JS_PROP_ENUMERABLE | JS_PROP_CONFIGURABLE, .def_type = JS_DEF_CGETSET_MAGIC, .magic = magic_num, .u = { \
+      .getset = {.get = {.getter_magic = fgetter}, .set = {.setter_magic = fsetter}} \
+    } \
+  }
+
+#define JS_CGETSET_MAGIC_FLAGS_DEF(prop_name, fgetter, fsetter, magic_num, flags) \
+  { \
+    .name = prop_name, .prop_flags = flags, .def_type = JS_DEF_CGETSET_MAGIC, .magic = magic_num, .u = { \
+      .getset = {.get = {.getter_magic = fgetter}, .set = {.setter_magic = fsetter}} \
+    } \
+  }
+
+#define JS_CFUNC_DEF_FLAGS(prop_name, length, func1, flags) \
+  { \
+    .name = prop_name, .prop_flags = flags, .def_type = JS_DEF_CFUNC, .magic = 0, .u = {.func = {length, JS_CFUNC_generic, {.generic = func1}} } \
+  }
+
+#define JS_CONSTANT_FLAGS(name, flags) JS_PROP_INT32_DEF(#name, name, (flags))
+#define JS_CONSTANT(name) JS_PROP_INT32_DEF(#name, name, JS_PROP_CONFIGURABLE | JS_PROP_ENUMERABLE)
+#define JS_CONSTANT_NONENUMERABLE(name) JS_PROP_INT32_DEF(#name, name, JS_PROP_CONFIGURABLE)
+
+#ifdef JS_SHARED_LIBRARY
+#if defined(_WIN32) || defined(__MINGW32__)
+#define VISIBLE __declspec(dllexport)
+#define HIDDEN
+#else
+#define VISIBLE __attribute__((visibility("default")))
+#define HIDDEN __attribute__((visibility("hidden")))
+#endif
+#else
+#define VISIBLE
+#define HIDDEN
+#endif
+
+#ifndef MAX_NUM
+#define MAX_NUM(a, b) ((a) > (b) ? (a) : (b))
+#endif
+#ifndef MIN_NUM
+#define MIN_NUM(a, b) ((a) < (b) ? (a) : (b))
+#endif
+#ifndef ABS_NUM
+#define ABS_NUM(n) ((n) < 0 ? -(n) : (n))
+#endif
+#ifndef MOD_NUM
+#define MOD_NUM(n, divisor) ((((n) % (divisor)) + (divisor)) % (divisor))
+#endif
+#ifndef SIGN_NUM
+#define SIGN_NUM(n) ((n) < 0)
+#endif
+
+#define COLOR_BLACK "\x1b[0;30m"
+#define COLOR_RED "\x1b[0;31m"
+#define COLOR_GREEN "\x1b[0;32m"
+#define COLOR_BROWN "\x1b[0;33m"
+#define COLOR_BLUE "\x1b[0;34m"
+#define COLOR_PURPLE "\x1b[0;35m"
+#define COLOR_MARINE "\x1b[0;36m"
+#define COLOR_LIGHTGRAY "\x1b[0;37m"
+#define COLOR_GRAY "\x1b[1;30m"
+#define COLOR_NONE "\x1b[0m"
+
+#define COLOR_LIGHTRED "\x1b[1;31m"
+
+#define COLOR_LIGHTGREEN "\x1b[1;32m"
+#define COLOR_YELLOW "\x1b[1;33m"
+#define COLOR_LIGHTBLUE "\x1b[1;34m"
+#define COLOR_MAGENTA "\x1b[1;35m"
+#define COLOR_CYAN "\x1b[1;36m"
+#define COLOR_WHITE "\x1b[1;37m"
+
+#define BGCOLOR_RED "\x1b[48;5;124m"
+#define BGCOLOR_BLUE "\x1b[48;5;20m"
+#define BGCOLOR_YELLOW "\x1b[48;5;214m"
+#define BGCOLOR_GREEN "\x1b[48;5;28m"
+#define BGCOLOR_PINK "\x1b[48;5;165m"
+
+#define JS_VALUE_FREE(ctx, value) \
+  do { \
+    JS_FreeValue((ctx), (value)); \
+    (value) = JS_UNDEFINED; \
+  } while(0);
+#define JS_VALUE_FREE_RT(ctx, value) \
+  do { \
+    JS_FreeValueRT((ctx), (value)); \
+    (value) = JS_UNDEFINED; \
+  } while(0);
+
+#if 0
+#define js_object_tmpmark_set(value) \
+  do { \
+    ((uint8_t*)JS_VALUE_GET_OBJ((value)))[5] |= 0x40; \
+  } while(0);
+#define js_object_tmpmark_clear(value) \
+  do { \
+    ((uint8_t*)JS_VALUE_GET_OBJ((value)))[5] &= ~0x40; \
+  } while(0);
+#define js_object_tmpmark_isset(value) (((uint8_t*)JS_VALUE_GET_OBJ((value)))[5] & 0x40)
+#else
+#define js_object_tmpmark_set(value) \
+  do { \
+    JS_VALUE_GET_OBJ((value))->tmp_mark |= 0x40; \
+  } while(0);
+#define js_object_tmpmark_clear(value) \
+  do { \
+    JS_VALUE_GET_OBJ((value))->tmp_mark &= ~0x40; \
+  } while(0);
+#define js_object_tmpmark_isset(value) (JS_VALUE_GET_OBJ((value))->tmp_mark & 0x40)
+#endif
+
+#define js_runtime_exception_set(rt, value) \
+  do { \
+    *(JSValue*)((uint8_t*)(rt) + 216) = value; \
+  } while(0);
+#define js_runtime_exception_get(rt) (*(JSValue*)((uint8_t*)(rt) + 216))
+#define js_runtime_exception_clear(rt) \
+  do { \
+    if(!JS_IsNull(js_runtime_exception_get(rt))) \
+      JS_FreeValueRT((rt), js_runtime_exception_get(rt)); \
+    js_runtime_exception_set(rt, JS_NULL); \
+  } while(0)
+
+#define JS_ATOM_TAG_INT (1U << 31)
+#define JS_ATOM_MAX_INT (JS_ATOM_TAG_INT - 1)
+
+#define JS_ATOM_ISINT(i) ((JSAtom)((i)&JS_ATOM_TAG_INT))
+#define JS_ATOM_FROMINT(i) ((JSAtom)((i)&JS_ATOM_MAX_INT) | JS_ATOM_TAG_INT)
+#define JS_ATOM_TOINT(i) (unsigned int)(((JSAtom)(i) & (~(JS_ATOM_TAG_INT))))
+
+/**
+ * @}
+ */
+#endif /* defined(DEFINES_H) */

src/quickjs/hello.c

@@ -0,0 +1,45 @@
+/* File generated automatically by the QuickJS compiler. */
+
+#include "quickjs-libc.h"
+
+const uint32_t qjsc_hello_size = 87;
+
+const uint8_t qjsc_hello[87] = {
+ 0x43, 0x04, 0x0e, 0x63, 0x6f, 0x6e, 0x73, 0x6f,
+ 0x6c, 0x65, 0x06, 0x6c, 0x6f, 0x67, 0x16, 0x48,
+ 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72,
+ 0x6c, 0x64, 0x22, 0x65, 0x78, 0x61, 0x6d, 0x70,
+ 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c, 0x6c,
+ 0x6f, 0x2e, 0x6a, 0x73, 0x0c, 0x00, 0x06, 0x00,
+ 0xa2, 0x01, 0x00, 0x01, 0x00, 0x03, 0x00, 0x00,
+ 0x14, 0x01, 0xa4, 0x01, 0x00, 0x00, 0x00, 0x38,
+ 0xe3, 0x00, 0x00, 0x00, 0x42, 0xe4, 0x00, 0x00,
+ 0x00, 0x04, 0xe5, 0x00, 0x00, 0x00, 0x24, 0x01,
+ 0x00, 0xcf, 0x28, 0xcc, 0x03, 0x01, 0x00,
+};
+
+static JSContext *JS_NewCustomContext(JSRuntime *rt)
+{
+  JSContext *ctx = JS_NewContextRaw(rt);
+  if (!ctx)
+    return NULL;
+  JS_AddIntrinsicBaseObjects(ctx);
+  return ctx;
+}
+
+int main(int argc, char **argv)
+{
+  JSRuntime *rt;
+  JSContext *ctx;
+  rt = JS_NewRuntime();
+  js_std_set_worker_new_context_func(JS_NewCustomContext);
+  js_std_init_handlers(rt);
+  ctx = JS_NewCustomContext(rt);
+  js_std_add_helpers(ctx, argc, argv);
+  js_std_eval_binary(ctx, qjsc_hello, qjsc_hello_size, 0);
+  js_std_loop(ctx);
+  js_std_free_handlers(rt);
+  JS_FreeContext(ctx);
+  JS_FreeRuntime(rt);
+  return 0;
+}

src/quickjs/iso_8859_1.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_1_H
+#define TUTF8E_ISO_8859_1_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_1_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_1, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_1_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_1, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_1_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_1, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_1_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_1, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_10.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_10_H
+#define TUTF8E_ISO_8859_10_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_10_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_10, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_10_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_10, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_10_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_10, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_10_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_10, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_11.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_11_H
+#define TUTF8E_ISO_8859_11_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_11_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_11, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_11_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_11, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_11_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_11, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_11_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_11, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_13.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_13_H
+#define TUTF8E_ISO_8859_13_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_13_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_13, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_13_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_13, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_13_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_13, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_13_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_13, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_14.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_14_H
+#define TUTF8E_ISO_8859_14_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_14_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_14, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_14_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_14, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_14_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_14, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_14_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_14, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_15.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_15_H
+#define TUTF8E_ISO_8859_15_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_15_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_15, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_15_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_15, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_15_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_15, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_15_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_15, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_16.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_16_H
+#define TUTF8E_ISO_8859_16_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_16_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_16, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_16_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_16, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_16_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_16, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_16_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_16, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_2.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_2_H
+#define TUTF8E_ISO_8859_2_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_2_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_2, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_2_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_2, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_2_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_2, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_2_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_2, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_3.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_3_H
+#define TUTF8E_ISO_8859_3_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_3_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_3, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_3_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_3, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_3_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_3, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_3_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_3, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_4.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_4_H
+#define TUTF8E_ISO_8859_4_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_4_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_4, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_4_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_4, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_4_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_4, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_4_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_4, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_5.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_5_H
+#define TUTF8E_ISO_8859_5_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_5_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_5, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_5_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_5, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_5_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_5, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_5_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_5, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_6.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_6_H
+#define TUTF8E_ISO_8859_6_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_6_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_6, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_6_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_6, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_6_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_6, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_6_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_6, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_7.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_7_H
+#define TUTF8E_ISO_8859_7_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_7_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_7, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_7_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_7, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_7_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_7, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_7_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_7, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_8.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_8_H
+#define TUTF8E_ISO_8859_8_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_8_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_8, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_8_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_8, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_8_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_8, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_8_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_8, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/iso_8859_9.h

@@ -0,0 +1,27 @@
+
+#ifndef TUTF8E_ISO_8859_9_H
+#define TUTF8E_ISO_8859_9_H
+
+#include <tutf8e.h>
+
+static inline int tutf8e_iso_8859_9_string_length(const char *input, size_t *input_length, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_length(tutf8e_encoder_iso_8859_9, input, input_length, invalid, output_length);
+}
+
+static inline int tutf8e_iso_8859_9_string_encode(const char *input, char *output, const char *invalid, size_t *output_length)
+{
+  return tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_9, input, invalid, output, output_length);
+}
+
+static inline int tutf8e_iso_8859_9_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length)
+{
+  return tutf8e_encoder_buffer_length(tutf8e_encoder_iso_8859_9, input, input_length, invalid, length);
+}
+
+static inline int tutf8e_iso_8859_9_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length)
+{
+  return tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_9, input, input_length, invalid, output, output_length);
+}
+
+#endif

src/quickjs/libbf.h

@@ -0,0 +1,535 @@
+/*
+ * Tiny arbitrary precision floating point library
+ *
+ * Copyright (c) 2017-2021 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef LIBBF_H
+#define LIBBF_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__SIZEOF_INT128__) && (INTPTR_MAX >= INT64_MAX)
+#define LIMB_LOG2_BITS 6
+#else
+#define LIMB_LOG2_BITS 5
+#endif
+
+#define LIMB_BITS (1 << LIMB_LOG2_BITS)
+
+#if LIMB_BITS == 64
+typedef __int128 int128_t;
+typedef unsigned __int128 uint128_t;
+typedef int64_t slimb_t;
+typedef uint64_t limb_t;
+typedef uint128_t dlimb_t;
+#define BF_RAW_EXP_MIN INT64_MIN
+#define BF_RAW_EXP_MAX INT64_MAX
+
+#define LIMB_DIGITS 19
+#define BF_DEC_BASE UINT64_C(10000000000000000000)
+
+#else
+
+typedef int32_t slimb_t;
+typedef uint32_t limb_t;
+typedef uint64_t dlimb_t;
+#define BF_RAW_EXP_MIN INT32_MIN
+#define BF_RAW_EXP_MAX INT32_MAX
+
+#define LIMB_DIGITS 9
+#define BF_DEC_BASE 1000000000U
+
+#endif
+
+/* in bits */
+/* minimum number of bits for the exponent */
+#define BF_EXP_BITS_MIN 3
+/* maximum number of bits for the exponent */
+#define BF_EXP_BITS_MAX (LIMB_BITS - 3)
+/* extended range for exponent, used internally */
+#define BF_EXT_EXP_BITS_MAX (BF_EXP_BITS_MAX + 1)
+/* minimum possible precision */
+#define BF_PREC_MIN 2
+/* minimum possible precision */
+#define BF_PREC_MAX (((limb_t)1 << (LIMB_BITS - 2)) - 2)
+/* some operations support infinite precision */
+#define BF_PREC_INF (BF_PREC_MAX + 1) /* infinite precision */
+
+#if LIMB_BITS == 64
+#define BF_CHKSUM_MOD (UINT64_C(975620677) * UINT64_C(9795002197))
+#else
+#define BF_CHKSUM_MOD 975620677U
+#endif
+
+#define BF_EXP_ZERO BF_RAW_EXP_MIN
+#define BF_EXP_INF (BF_RAW_EXP_MAX - 1)
+#define BF_EXP_NAN BF_RAW_EXP_MAX
+
+/* +/-zero is represented with expn = BF_EXP_ZERO and len = 0,
+   +/-infinity is represented with expn = BF_EXP_INF and len = 0,
+   NaN is represented with expn = BF_EXP_NAN and len = 0 (sign is ignored)
+ */
+typedef struct {
+    struct bf_context_t *ctx;
+    int sign;
+    slimb_t expn;
+    limb_t len;
+    limb_t *tab;
+} bf_t;
+
+typedef struct {
+    /* must be kept identical to bf_t */
+    struct bf_context_t *ctx;
+    int sign;
+    slimb_t expn;
+    limb_t len;
+    limb_t *tab;
+} bfdec_t;
+
+typedef enum {
+    BF_RNDN, /* round to nearest, ties to even */
+    BF_RNDZ, /* round to zero */
+    BF_RNDD, /* round to -inf (the code relies on (BF_RNDD xor BF_RNDU) = 1) */
+    BF_RNDU, /* round to +inf */
+    BF_RNDNA, /* round to nearest, ties away from zero */
+    BF_RNDA, /* round away from zero */
+    BF_RNDF, /* faithful rounding (nondeterministic, either RNDD or RNDU,
+                inexact flag is always set)  */
+} bf_rnd_t;
+
+/* allow subnormal numbers. Only available if the number of exponent
+   bits is <= BF_EXP_BITS_USER_MAX and prec != BF_PREC_INF. */
+#define BF_FLAG_SUBNORMAL (1 << 3)
+/* 'prec' is the precision after the radix point instead of the whole
+   mantissa. Can only be used with bf_round() and
+   bfdec_[add|sub|mul|div|sqrt|round](). */
+#define BF_FLAG_RADPNT_PREC (1 << 4)
+
+#define BF_RND_MASK 0x7
+#define BF_EXP_BITS_SHIFT 5
+#define BF_EXP_BITS_MASK 0x3f
+
+/* shortcut for bf_set_exp_bits(BF_EXT_EXP_BITS_MAX) */
+#define BF_FLAG_EXT_EXP (BF_EXP_BITS_MASK << BF_EXP_BITS_SHIFT)
+
+/* contains the rounding mode and number of exponents bits */
+typedef uint32_t bf_flags_t;
+
+typedef void *bf_realloc_func_t(void *opaque, void *ptr, size_t size);
+
+typedef struct {
+    bf_t val;
+    limb_t prec;
+} BFConstCache;
+
+typedef struct bf_context_t {
+    void *realloc_opaque;
+    bf_realloc_func_t *realloc_func;
+    BFConstCache log2_cache;
+    BFConstCache pi_cache;
+    struct BFNTTState *ntt_state;
+} bf_context_t;
+
+static inline int bf_get_exp_bits(bf_flags_t flags)
+{
+    int e;
+    e = (flags >> BF_EXP_BITS_SHIFT) & BF_EXP_BITS_MASK;
+    if (e == BF_EXP_BITS_MASK)
+        return BF_EXP_BITS_MAX + 1;
+    else
+        return BF_EXP_BITS_MAX - e;
+}
+
+static inline bf_flags_t bf_set_exp_bits(int n)
+{
+    return ((BF_EXP_BITS_MAX - n) & BF_EXP_BITS_MASK) << BF_EXP_BITS_SHIFT;
+}
+
+/* returned status */
+#define BF_ST_INVALID_OP  (1 << 0)
+#define BF_ST_DIVIDE_ZERO (1 << 1)
+#define BF_ST_OVERFLOW    (1 << 2)
+#define BF_ST_UNDERFLOW   (1 << 3)
+#define BF_ST_INEXACT     (1 << 4)
+/* indicate that a memory allocation error occured. NaN is returned */
+#define BF_ST_MEM_ERROR   (1 << 5)
+
+#define BF_RADIX_MAX 36 /* maximum radix for bf_atof() and bf_ftoa() */
+
+static inline slimb_t bf_max(slimb_t a, slimb_t b)
+{
+    if (a > b)
+        return a;
+    else
+        return b;
+}
+
+static inline slimb_t bf_min(slimb_t a, slimb_t b)
+{
+    if (a < b)
+        return a;
+    else
+        return b;
+}
+
+void bf_context_init(bf_context_t *s, bf_realloc_func_t *realloc_func,
+                     void *realloc_opaque);
+void bf_context_end(bf_context_t *s);
+/* free memory allocated for the bf cache data */
+void bf_clear_cache(bf_context_t *s);
+
+static inline void *bf_realloc(bf_context_t *s, void *ptr, size_t size)
+{
+    return s->realloc_func(s->realloc_opaque, ptr, size);
+}
+
+/* 'size' must be != 0 */
+static inline void *bf_malloc(bf_context_t *s, size_t size)
+{
+    return bf_realloc(s, NULL, size);
+}
+
+static inline void bf_free(bf_context_t *s, void *ptr)
+{
+    /* must test ptr otherwise equivalent to malloc(0) */
+    if (ptr)
+        bf_realloc(s, ptr, 0);
+}
+
+void bf_init(bf_context_t *s, bf_t *r);
+
+static inline void bf_delete(bf_t *r)
+{
+    bf_context_t *s = r->ctx;
+    /* we accept to delete a zeroed bf_t structure */
+    if (s && r->tab) {
+        bf_realloc(s, r->tab, 0);
+    }
+}
+
+static inline void bf_neg(bf_t *r)
+{
+    r->sign ^= 1;
+}
+
+static inline int bf_is_finite(const bf_t *a)
+{
+    return (a->expn < BF_EXP_INF);
+}
+
+static inline int bf_is_nan(const bf_t *a)
+{
+    return (a->expn == BF_EXP_NAN);
+}
+
+static inline int bf_is_zero(const bf_t *a)
+{
+    return (a->expn == BF_EXP_ZERO);
+}
+
+static inline void bf_memcpy(bf_t *r, const bf_t *a)
+{
+    *r = *a;
+}
+
+int bf_set_ui(bf_t *r, uint64_t a);
+int bf_set_si(bf_t *r, int64_t a);
+void bf_set_nan(bf_t *r);
+void bf_set_zero(bf_t *r, int is_neg);
+void bf_set_inf(bf_t *r, int is_neg);
+int bf_set(bf_t *r, const bf_t *a);
+void bf_move(bf_t *r, bf_t *a);
+int bf_get_float64(const bf_t *a, double *pres, bf_rnd_t rnd_mode);
+int bf_set_float64(bf_t *a, double d);
+
+int bf_cmpu(const bf_t *a, const bf_t *b);
+int bf_cmp_full(const bf_t *a, const bf_t *b);
+int bf_cmp(const bf_t *a, const bf_t *b);
+static inline int bf_cmp_eq(const bf_t *a, const bf_t *b)
+{
+    return bf_cmp(a, b) == 0;
+}
+
+static inline int bf_cmp_le(const bf_t *a, const bf_t *b)
+{
+    return bf_cmp(a, b) <= 0;
+}
+
+static inline int bf_cmp_lt(const bf_t *a, const bf_t *b)
+{
+    return bf_cmp(a, b) < 0;
+}
+
+int bf_add(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags);
+int bf_sub(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags);
+int bf_add_si(bf_t *r, const bf_t *a, int64_t b1, limb_t prec, bf_flags_t flags);
+int bf_mul(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags);
+int bf_mul_ui(bf_t *r, const bf_t *a, uint64_t b1, limb_t prec, bf_flags_t flags);
+int bf_mul_si(bf_t *r, const bf_t *a, int64_t b1, limb_t prec,
+              bf_flags_t flags);
+int bf_mul_2exp(bf_t *r, slimb_t e, limb_t prec, bf_flags_t flags);
+int bf_div(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags);
+#define BF_DIVREM_EUCLIDIAN BF_RNDF
+int bf_divrem(bf_t *q, bf_t *r, const bf_t *a, const bf_t *b,
+              limb_t prec, bf_flags_t flags, int rnd_mode);
+int bf_rem(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec,
+           bf_flags_t flags, int rnd_mode);
+int bf_remquo(slimb_t *pq, bf_t *r, const bf_t *a, const bf_t *b, limb_t prec,
+              bf_flags_t flags, int rnd_mode);
+/* round to integer with infinite precision */
+int bf_rint(bf_t *r, int rnd_mode);
+int bf_round(bf_t *r, limb_t prec, bf_flags_t flags);
+int bf_sqrtrem(bf_t *r, bf_t *rem1, const bf_t *a);
+int bf_sqrt(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+slimb_t bf_get_exp_min(const bf_t *a);
+int bf_logic_or(bf_t *r, const bf_t *a, const bf_t *b);
+int bf_logic_xor(bf_t *r, const bf_t *a, const bf_t *b);
+int bf_logic_and(bf_t *r, const bf_t *a, const bf_t *b);
+
+/* additional flags for bf_atof */
+/* do not accept hex radix prefix (0x or 0X) if radix = 0 or radix = 16 */
+#define BF_ATOF_NO_HEX       (1 << 16)
+/* accept binary (0b or 0B) or octal (0o or 0O) radix prefix if radix = 0 */
+#define BF_ATOF_BIN_OCT      (1 << 17)
+/* Do not parse NaN or Inf */
+#define BF_ATOF_NO_NAN_INF   (1 << 18)
+/* return the exponent separately */
+#define BF_ATOF_EXPONENT       (1 << 19)
+
+int bf_atof(bf_t *a, const char *str, const char **pnext, int radix,
+            limb_t prec, bf_flags_t flags);
+/* this version accepts prec = BF_PREC_INF and returns the radix
+   exponent */
+int bf_atof2(bf_t *r, slimb_t *pexponent,
+             const char *str, const char **pnext, int radix,
+             limb_t prec, bf_flags_t flags);
+int bf_mul_pow_radix(bf_t *r, const bf_t *T, limb_t radix,
+                     slimb_t expn, limb_t prec, bf_flags_t flags);
+
+
+/* Conversion of floating point number to string. Return a null
+   terminated string or NULL if memory error. *plen contains its
+   length if plen != NULL.  The exponent letter is "e" for base 10,
+   "p" for bases 2, 8, 16 with a binary exponent and "@" for the other
+   bases. */
+
+#define BF_FTOA_FORMAT_MASK (3 << 16)
+
+/* fixed format: prec significant digits rounded with (flags &
+   BF_RND_MASK). Exponential notation is used if too many zeros are
+   needed.*/
+#define BF_FTOA_FORMAT_FIXED (0 << 16)
+/* fractional format: prec digits after the decimal point rounded with
+   (flags & BF_RND_MASK) */
+#define BF_FTOA_FORMAT_FRAC  (1 << 16)
+/* free format:
+
+   For binary radices with bf_ftoa() and for bfdec_ftoa(): use the minimum
+   number of digits to represent 'a'. The precision and the rounding
+   mode are ignored.
+
+   For the non binary radices with bf_ftoa(): use as many digits as
+   necessary so that bf_atof() return the same number when using
+   precision 'prec', rounding to nearest and the subnormal
+   configuration of 'flags'. The result is meaningful only if 'a' is
+   already rounded to 'prec' bits. If the subnormal flag is set, the
+   exponent in 'flags' must also be set to the desired exponent range.
+*/
+#define BF_FTOA_FORMAT_FREE  (2 << 16)
+/* same as BF_FTOA_FORMAT_FREE but uses the minimum number of digits
+   (takes more computation time). Identical to BF_FTOA_FORMAT_FREE for
+   binary radices with bf_ftoa() and for bfdec_ftoa(). */
+#define BF_FTOA_FORMAT_FREE_MIN (3 << 16)
+
+/* force exponential notation for fixed or free format */
+#define BF_FTOA_FORCE_EXP    (1 << 20)
+/* add 0x prefix for base 16, 0o prefix for base 8 or 0b prefix for
+   base 2 if non zero value */
+#define BF_FTOA_ADD_PREFIX   (1 << 21)
+/* return "Infinity" instead of "Inf" and add a "+" for positive
+   exponents */
+#define BF_FTOA_JS_QUIRKS    (1 << 22)
+
+char *bf_ftoa(size_t *plen, const bf_t *a, int radix, limb_t prec,
+              bf_flags_t flags);
+
+/* modulo 2^n instead of saturation. NaN and infinity return 0 */
+#define BF_GET_INT_MOD (1 << 0)
+int bf_get_int32(int *pres, const bf_t *a, int flags);
+int bf_get_int64(int64_t *pres, const bf_t *a, int flags);
+int bf_get_uint64(uint64_t *pres, const bf_t *a);
+
+/* the following functions are exported for testing only. */
+void mp_print_str(const char *str, const limb_t *tab, limb_t n);
+void bf_print_str(const char *str, const bf_t *a);
+int bf_resize(bf_t *r, limb_t len);
+int bf_get_fft_size(int *pdpl, int *pnb_mods, limb_t len);
+int bf_normalize_and_round(bf_t *r, limb_t prec1, bf_flags_t flags);
+int bf_can_round(const bf_t *a, slimb_t prec, bf_rnd_t rnd_mode, slimb_t k);
+slimb_t bf_mul_log2_radix(slimb_t a1, unsigned int radix, int is_inv,
+                          int is_ceil1);
+int mp_mul(bf_context_t *s, limb_t *result,
+           const limb_t *op1, limb_t op1_size,
+           const limb_t *op2, limb_t op2_size);
+limb_t mp_add(limb_t *res, const limb_t *op1, const limb_t *op2,
+              limb_t n, limb_t carry);
+limb_t mp_add_ui(limb_t *tab, limb_t b, size_t n);
+int mp_sqrtrem(bf_context_t *s, limb_t *tabs, limb_t *taba, limb_t n);
+int mp_recip(bf_context_t *s, limb_t *tabr, const limb_t *taba, limb_t n);
+limb_t bf_isqrt(limb_t a);
+
+/* transcendental functions */
+int bf_const_log2(bf_t *T, limb_t prec, bf_flags_t flags);
+int bf_const_pi(bf_t *T, limb_t prec, bf_flags_t flags);
+int bf_exp(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_log(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+#define BF_POW_JS_QUIRKS (1 << 16) /* (+/-1)^(+/-Inf) = NaN, 1^NaN = NaN */
+int bf_pow(bf_t *r, const bf_t *x, const bf_t *y, limb_t prec, bf_flags_t flags);
+int bf_cos(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_sin(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_tan(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_atan(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_atan2(bf_t *r, const bf_t *y, const bf_t *x,
+             limb_t prec, bf_flags_t flags);
+int bf_asin(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+int bf_acos(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags);
+
+/* decimal floating point */
+
+static inline void bfdec_init(bf_context_t *s, bfdec_t *r)
+{
+    bf_init(s, (bf_t *)r);
+}
+static inline void bfdec_delete(bfdec_t *r)
+{
+    bf_delete((bf_t *)r);
+}
+
+static inline void bfdec_neg(bfdec_t *r)
+{
+    r->sign ^= 1;
+}
+
+static inline int bfdec_is_finite(const bfdec_t *a)
+{
+    return (a->expn < BF_EXP_INF);
+}
+
+static inline int bfdec_is_nan(const bfdec_t *a)
+{
+    return (a->expn == BF_EXP_NAN);
+}
+
+static inline int bfdec_is_zero(const bfdec_t *a)
+{
+    return (a->expn == BF_EXP_ZERO);
+}
+
+static inline void bfdec_memcpy(bfdec_t *r, const bfdec_t *a)
+{
+    bf_memcpy((bf_t *)r, (const bf_t *)a);
+}
+
+int bfdec_set_ui(bfdec_t *r, uint64_t a);
+int bfdec_set_si(bfdec_t *r, int64_t a);
+
+static inline void bfdec_set_nan(bfdec_t *r)
+{
+    bf_set_nan((bf_t *)r);
+}
+static inline void bfdec_set_zero(bfdec_t *r, int is_neg)
+{
+    bf_set_zero((bf_t *)r, is_neg);
+}
+static inline void bfdec_set_inf(bfdec_t *r, int is_neg)
+{
+    bf_set_inf((bf_t *)r, is_neg);
+}
+static inline int bfdec_set(bfdec_t *r, const bfdec_t *a)
+{
+    return bf_set((bf_t *)r, (bf_t *)a);
+}
+static inline void bfdec_move(bfdec_t *r, bfdec_t *a)
+{
+    bf_move((bf_t *)r, (bf_t *)a);
+}
+static inline int bfdec_cmpu(const bfdec_t *a, const bfdec_t *b)
+{
+    return bf_cmpu((const bf_t *)a, (const bf_t *)b);
+}
+static inline int bfdec_cmp_full(const bfdec_t *a, const bfdec_t *b)
+{
+    return bf_cmp_full((const bf_t *)a, (const bf_t *)b);
+}
+static inline int bfdec_cmp(const bfdec_t *a, const bfdec_t *b)
+{
+    return bf_cmp((const bf_t *)a, (const bf_t *)b);
+}
+static inline int bfdec_cmp_eq(const bfdec_t *a, const bfdec_t *b)
+{
+    return bfdec_cmp(a, b) == 0;
+}
+static inline int bfdec_cmp_le(const bfdec_t *a, const bfdec_t *b)
+{
+    return bfdec_cmp(a, b) <= 0;
+}
+static inline int bfdec_cmp_lt(const bfdec_t *a, const bfdec_t *b)
+{
+    return bfdec_cmp(a, b) < 0;
+}
+
+int bfdec_add(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec,
+              bf_flags_t flags);
+int bfdec_sub(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec,
+              bf_flags_t flags);
+int bfdec_add_si(bfdec_t *r, const bfdec_t *a, int64_t b1, limb_t prec,
+                 bf_flags_t flags);
+int bfdec_mul(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec,
+              bf_flags_t flags);
+int bfdec_mul_si(bfdec_t *r, const bfdec_t *a, int64_t b1, limb_t prec,
+                 bf_flags_t flags);
+int bfdec_div(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec,
+              bf_flags_t flags);
+int bfdec_divrem(bfdec_t *q, bfdec_t *r, const bfdec_t *a, const bfdec_t *b,
+                 limb_t prec, bf_flags_t flags, int rnd_mode);
+int bfdec_rem(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec,
+              bf_flags_t flags, int rnd_mode);
+int bfdec_rint(bfdec_t *r, int rnd_mode);
+int bfdec_sqrt(bfdec_t *r, const bfdec_t *a, limb_t prec, bf_flags_t flags);
+int bfdec_round(bfdec_t *r, limb_t prec, bf_flags_t flags);
+int bfdec_get_int32(int *pres, const bfdec_t *a);
+int bfdec_pow_ui(bfdec_t *r, const bfdec_t *a, limb_t b);
+
+char *bfdec_ftoa(size_t *plen, const bfdec_t *a, limb_t prec, bf_flags_t flags);
+int bfdec_atof(bfdec_t *r, const char *str, const char **pnext,
+               limb_t prec, bf_flags_t flags);
+
+/* the following functions are exported for testing only. */
+extern const limb_t mp_pow_dec[LIMB_DIGITS + 1];
+void bfdec_print_str(const char *str, const bfdec_t *a);
+static inline int bfdec_resize(bfdec_t *r, limb_t len)
+{
+    return bf_resize((bf_t *)r, len);
+}
+int bfdec_normalize_and_round(bfdec_t *r, limb_t prec1, bf_flags_t flags);
+
+#endif /* LIBBF_H */

src/quickjs/libregexp-opcode.h

@@ -0,0 +1,57 @@
+/*
+ * Regular Expression Engine
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifdef DEF
+
+DEF(invalid, 1) /* never used */
+DEF(char, 3)
+DEF(char32, 5)
+DEF(dot, 1)
+DEF(any, 1) /* same as dot but match any character including line terminator */
+DEF(line_start, 1)
+DEF(line_end, 1)
+DEF(goto, 5)
+DEF(split_goto_first, 5)
+DEF(split_next_first, 5)
+DEF(match, 1)
+DEF(save_start, 2) /* save start position */
+DEF(save_end, 2) /* save end position, must come after saved_start */
+DEF(save_reset, 3) /* reset save positions */
+DEF(loop, 5) /* decrement the top the stack and goto if != 0 */
+DEF(push_i32, 5) /* push integer on the stack */
+DEF(drop, 1)
+DEF(word_boundary, 1)
+DEF(not_word_boundary, 1)
+DEF(back_reference, 2)
+DEF(backward_back_reference, 2) /* must come after back_reference */
+DEF(range, 3) /* variable length */
+DEF(range32, 3) /* variable length */
+DEF(lookahead, 5)
+DEF(negative_lookahead, 5)
+DEF(push_char_pos, 1) /* push the character position on the stack */
+DEF(check_advance, 1) /* pop one stack element and check that it is different from the character position */
+DEF(prev, 1) /* go to the previous char */
+DEF(simple_greedy_quant, 17)
+
+#endif /* DEF */

src/quickjs/libregexp.h

@@ -0,0 +1,92 @@
+/*
+ * Regular Expression Engine
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef LIBREGEXP_H
+#define LIBREGEXP_H
+
+#include <stddef.h>
+
+#include "libunicode.h"
+
+#define LRE_BOOL  int       /* for documentation purposes */
+
+#define LRE_FLAG_GLOBAL     (1 << 0)
+#define LRE_FLAG_IGNORECASE (1 << 1)
+#define LRE_FLAG_MULTILINE  (1 << 2)
+#define LRE_FLAG_DOTALL     (1 << 3)
+#define LRE_FLAG_UNICODE    (1 << 4)
+#define LRE_FLAG_STICKY     (1 << 5)
+#define LRE_FLAG_INDICES    (1 << 6) /* Unused by libregexp, just recorded. */
+#define LRE_FLAG_NAMED_GROUPS (1 << 7) /* named groups are present in the regexp */
+
+uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size,
+                     const char *buf, size_t buf_len, int re_flags,
+                     void *opaque);
+int lre_get_capture_count(const uint8_t *bc_buf);
+int lre_get_flags(const uint8_t *bc_buf);
+const char *lre_get_groupnames(const uint8_t *bc_buf);
+int lre_exec(uint8_t **capture,
+             const uint8_t *bc_buf, const uint8_t *cbuf, int cindex, int clen,
+             int cbuf_type, void *opaque);
+
+int lre_parse_escape(const uint8_t **pp, int allow_utf16);
+LRE_BOOL lre_is_space(int c);
+
+/* must be provided by the user */
+LRE_BOOL lre_check_stack_overflow(void *opaque, size_t alloca_size);
+void *lre_realloc(void *opaque, void *ptr, size_t size);
+
+/* JS identifier test */
+extern uint32_t const lre_id_start_table_ascii[4];
+extern uint32_t const lre_id_continue_table_ascii[4];
+
+static inline int lre_js_is_ident_first(int c)
+{
+    if ((uint32_t)c < 128) {
+        return (lre_id_start_table_ascii[c >> 5] >> (c & 31)) & 1;
+    } else {
+#ifdef CONFIG_ALL_UNICODE
+        return lre_is_id_start(c);
+#else
+        return !lre_is_space(c);
+#endif
+    }
+}
+
+static inline int lre_js_is_ident_next(int c)
+{
+    if ((uint32_t)c < 128) {
+        return (lre_id_continue_table_ascii[c >> 5] >> (c & 31)) & 1;
+    } else {
+        /* ZWNJ and ZWJ are accepted in identifiers */
+#ifdef CONFIG_ALL_UNICODE
+        return lre_is_id_continue(c) || c == 0x200C || c == 0x200D;
+#else
+        return !lre_is_space(c) || c == 0x200C || c == 0x200D;
+#endif
+    }
+}
+
+#undef LRE_BOOL
+
+#endif /* LIBREGEXP_H */

src/quickjs/libunicode.h

@@ -0,0 +1,127 @@
+/*
+ * Unicode utilities
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef LIBUNICODE_H
+#define LIBUNICODE_H
+
+#include <inttypes.h>
+
+#define LRE_BOOL  int       /* for documentation purposes */
+
+/* define it to include all the unicode tables (40KB larger) */
+#define CONFIG_ALL_UNICODE
+
+#define LRE_CC_RES_LEN_MAX 3
+
+typedef enum {
+    UNICODE_NFC,
+    UNICODE_NFD,
+    UNICODE_NFKC,
+    UNICODE_NFKD,
+} UnicodeNormalizationEnum;
+
+int lre_case_conv(uint32_t *res, uint32_t c, int conv_type);
+int lre_canonicalize(uint32_t c, LRE_BOOL is_unicode);
+LRE_BOOL lre_is_cased(uint32_t c);
+LRE_BOOL lre_is_case_ignorable(uint32_t c);
+
+/* char ranges */
+
+typedef struct {
+    int len; /* in points, always even */
+    int size;
+    uint32_t *points; /* points sorted by increasing value */
+    void *mem_opaque;
+    void *(*realloc_func)(void *opaque, void *ptr, size_t size);
+} CharRange;
+
+typedef enum {
+    CR_OP_UNION,
+    CR_OP_INTER,
+    CR_OP_XOR,
+} CharRangeOpEnum;
+
+void cr_init(CharRange *cr, void *mem_opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size));
+void cr_free(CharRange *cr);
+int cr_realloc(CharRange *cr, int size);
+int cr_copy(CharRange *cr, const CharRange *cr1);
+
+static inline int cr_add_point(CharRange *cr, uint32_t v)
+{
+    if (cr->len >= cr->size) {
+        if (cr_realloc(cr, cr->len + 1))
+            return -1;
+    }
+    cr->points[cr->len++] = v;
+    return 0;
+}
+
+static inline int cr_add_interval(CharRange *cr, uint32_t c1, uint32_t c2)
+{
+    if ((cr->len + 2) > cr->size) {
+        if (cr_realloc(cr, cr->len + 2))
+            return -1;
+    }
+    cr->points[cr->len++] = c1;
+    cr->points[cr->len++] = c2;
+    return 0;
+}
+
+int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len);
+
+static inline int cr_union_interval(CharRange *cr, uint32_t c1, uint32_t c2)
+{
+    uint32_t b_pt[2];
+    b_pt[0] = c1;
+    b_pt[1] = c2 + 1;
+    return cr_union1(cr, b_pt, 2);
+}
+
+int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,
+          const uint32_t *b_pt, int b_len, int op);
+
+int cr_invert(CharRange *cr);
+
+int cr_regexp_canonicalize(CharRange *cr, LRE_BOOL is_unicode);
+
+#ifdef CONFIG_ALL_UNICODE
+
+LRE_BOOL lre_is_id_start(uint32_t c);
+LRE_BOOL lre_is_id_continue(uint32_t c);
+
+int unicode_normalize(uint32_t **pdst, const uint32_t *src, int src_len,
+                      UnicodeNormalizationEnum n_type,
+                      void *opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size));
+
+/* Unicode character range functions */
+
+int unicode_script(CharRange *cr,
+                   const char *script_name, LRE_BOOL is_ext);
+int unicode_general_category(CharRange *cr, const char *gc_name);
+int unicode_prop(CharRange *cr, const char *prop_name);
+
+#endif /* CONFIG_ALL_UNICODE */
+
+#undef LRE_BOOL
+
+#endif /* LIBUNICODE_H */

src/quickjs/libutf.c

@@ -0,0 +1,153 @@
+#include <libutf.h>
+
+#include <assert.h>
+#include <stdlib.h>
+
+LibutfC16Type libutf_c16_type(uint_least16_t c16) {
+    if (0xD800 != (0xF800 & c16)) {
+        return LIBUTF_UTF16_NOT_SURROGATE;
+    }
+    return (c16 & 0x0400) ? LIBUTF_UTF16_SURROGATE_LOW : LIBUTF_UTF16_SURROGATE_HIGH;
+}
+
+LibutfC8Type libutf_c8_type(char c) {
+    static const LibutfC8Type lookup_table[256] = {
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+           -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+           -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+           -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+           -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+            2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+            2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+            3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+            4, 4, 4, 4, 4, 4, 4, 4,-2,-2,-2,-2,-2,-2,-2,-2,
+    };
+    return lookup_table[(unsigned char) c];
+}
+
+bool libutf_c32_to_c8(uint_least32_t c32, int *length, char c8[4]) {
+    if (!libutf_c32_is_valid(c32)) {
+        *length = 3;
+        c8[0] = (char) 0xEF;
+        c8[1] = (char) 0xBF;
+        c8[2] = (char) 0xBD;
+        return false;
+    }
+    if (c32 <= 0x007F) {
+        c8[0] = c32;
+        *length = 1;
+        return true;
+    } else if (c32 <= 0x07FF) {
+        c8[0] = (char) (0xC0 | (c32 >> 6));
+        c8[1] = (char) (0x80 | (c32 & 0x3F));
+        *length = 2;
+        return true;
+    } else if (c32 <= 0xFFFF) {
+        c8[0] = (char) (0xE0 | (c32 >> 12));
+        c8[1] = (char) (0x80 | ((c32 >> 6) & 0x3F));
+        c8[2] = (char) (0x80 | (c32 & 0x3F));
+        *length = 3;
+        return true;
+    }
+    assert(c32 <= 0x10FFFF);
+    c8[0] = (char) (0xF0 | (c32 >> 18));
+    c8[1] = (char) (0x80 | ((c32 >> 12) & 0x3F));
+    c8[2] = (char) (0x80 | ((c32 >> 6) & 0x3F));
+    c8[3] = (char) (0x80 | (c32 & 0x3F));
+    *length = 4;
+    return true;
+}
+
+bool libutf_c8_to_c32(const char *c8, uint_least32_t *c32) {
+    if (!c8 || !c32) {
+        return false;
+    }
+    unsigned char c = *c8;
+    *c32 = 0xFFFD;
+    LibutfC8Type type = libutf_c8_type(c);
+    if (type < 0) {
+        return false;
+    }
+    int len = type;
+    uint_least32_t result;
+    switch (len) {
+    case 1:
+        *c32 = c;
+        return true;
+    case 2:
+        result = c & 0x1F;
+        break;
+    case 3:
+        result = c & 0x0F;
+        break;
+    case 4:
+        result = c & 0x07;
+        break;
+    default:
+        abort();
+    }
+    for (int i = 1; i < len; ++i) {
+        c = *++c8;
+        if ((0xC0 & c) != 0x80) {
+            return false;
+        }
+        result = (result << 6) + (c & 0x3F);
+    }
+    if (!libutf_c32_is_valid(result)) {
+        return false;
+    }
+    char c8seq[4];
+    int n;
+    bool ok = libutf_c32_to_c8(result, &n, c8seq);
+    assert(ok);
+    if (n < (int) len) {
+        return false;
+    }
+    *c32 = result;
+    return true;
+}
+
+bool libutf_c32_to_c16(uint_least32_t c32, int *length, uint_least16_t c16[2]) {
+    if (!libutf_c32_is_valid(c32)) {
+        c16[0] = 0xFFFD;
+        c16[1] = 0;
+        *length = 1;
+        return false;
+    }
+    if (c32 < 0x10000) {
+        c16[0] = c32;
+        c16[1] = 0;
+        *length = 1;
+    } else {
+        c32 -= 0x10000;
+        c16[0] = 0xD800 | (c32 >> 10);
+        c16[1] = 0xDC00 | (c32 & 0x3FF);
+        *length = 2;
+    }
+    return true;
+}
+
+bool libutf_c16_to_c32(uint_least16_t c16[2], uint_least32_t *c32) {
+    LibutfC16Type type = libutf_c16_type(c16[0]);
+    if (LIBUTF_UTF16_NOT_SURROGATE == type) {
+        *c32 = c16[0];
+        return true;
+    }
+    if (LIBUTF_UTF16_SURROGATE_HIGH != type || LIBUTF_UTF16_SURROGATE_LOW != libutf_c16_type(c16[1])) {
+        *c32 = 0xFFFD;
+        return false;
+    }
+    *c32 = 0x10000 + ((c16[0] & 0x3FF) << 10) + (c16[1] & 0x3ff);
+    return true;
+}
+
+bool libutf_c32_is_valid(uint_least32_t c32) {
+    return c32 < 0xD800 || (0xDFFF < c32 && c32 < 0x110000);
+}

src/quickjs/libutf.h

@@ -0,0 +1,109 @@
+/* LIBrary of UTF-(8,16,32) helper utility functions.
+ *
+ * Terminology:
+ * code point = number of a Unicode character
+ * code unit
+ *   = 1 byte  for UTF-8
+ *   = 2 bytes for UTF-16
+ *   = 4 bytes for UTF-32
+ * sequence = sequence of code units encoding SINGLE Unicode character
+ *
+ * Unicode has code points 0x000000..0x10FFFF except for the codes 0xD800..0xDFFF which are
+ * reserved for UTF-16 surrogate pairs.
+ *
+ * UTF-32 is the simplest Unicode encoding. It encodes Unicode code points as is. Thus we may use
+ * UTF-32 as a synonym for Unicode code point.
+ * + O(1) indexing
+ * - the text is 2-4 times bigger than in other encodings
+ *
+ * UTF-16 is variable width encoding. Each Unicode code point is represented as either single
+ * code unit (two-byte number) or two code units which are called surrogate pairs.
+ * - ASCII-only texts will be twice as big as in ASCII
+ * - O(n) indexing
+ *
+ * UTF-8 is a variable width encoding. Each Unicode code point is represented as a sequence
+ * of up to 4 bytes. The first byte determines the length of the sequence.
+ * + is a superset of ASCII
+ * + ASCII-only texts will have the same size
+ * - O(n) indexing
+ * - single Unicode code point can have different representations in
+ *   UTF-8, only the shortest one is considered valid
+ */
+
+#ifndef LIBUTF_H
+#define LIBUTF_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// UTF-8 code unit type
+typedef enum {
+    LIBUTF_UTF8_OVERLONG   = -2,
+    LIBUTF_UTF8_TRAILING   = -1,
+    LIBUTF_UTF8_ONE_BYTE   = 1,
+    LIBUTF_UTF8_TWO_BYTE   = 2,
+    LIBUTF_UTF8_THREE_BYTE = 3,
+    LIBUTF_UTF8_FOUR_BYTE  = 4,
+} LibutfC8Type;
+
+// UTF-16 code unit type
+typedef enum {
+    LIBUTF_UTF16_SURROGATE_LOW  = -1,
+    LIBUTF_UTF16_NOT_SURROGATE  = 1,
+    LIBUTF_UTF16_SURROGATE_HIGH = 2,
+} LibutfC16Type;
+
+/* Determine type of UTF-16 code unit. type = length of UTF-16 sequence starting with the specified code unit
+ * or negative value if sequence cannot start with this code unit. */
+LibutfC16Type libutf_c16_type(uint_least16_t c16);
+
+/* Determine type of UTF-8 sequence based on the first byte. type = length of UTF-8 sequence starting
+ * with this byte or negative value if the byte cannot be first. UTF-8 may be up to 4 bytes long.
+ * Common idiom for using this function:
+ * >>> LibutfC8Type type = libutf_c8_type(c);
+ * >>> if (type < 0) {
+ * >>>     return ERROR_CODE;
+ * >>> }
+ * >>> int length = type;
+ * */
+LibutfC8Type libutf_c8_type(char c);
+
+/* Convert Unicode code point into UTF-8 sequence. If c32 is not a valid
+ * Unicode code point c8 will be filled with UTF-8 representation of special
+ * replacement character U+FFFD, *length will be set to its length and false will be returned.
+ * c32    -- UTF-32 Unicode code point
+ * length -- where to put length of the UTF-8 sequence.
+ * c8     -- where to put UTF-8 sequence. Make sure string has enough space.
+ * result -- true if c32 is a valid Unicode code point or false otherwise. */
+bool libutf_c32_to_c8(uint_least32_t c32, int *length, char c8[4]);
+
+/* Convert UTF-8 sequence into a UTF-32 Unicode code point. If c8 does not
+ * point to a valid UTF-8 sequence c32 will be filled with special replacement
+ * character U+FFFD and false will be returned.
+ * c8     -- pointer to UTF-8 sequence.
+ * c32    -- where to save UTF-32 Unicode code point.
+ * result -- true if c8 points to a valid UTF-8 sequence or false otherwise. */
+bool libutf_c8_to_c32(const char *c8, uint_least32_t *c32);
+
+/* Convert UTF-32 Unicode code point into UTF-16 sequence. If c32 is not a valid
+ * Unicode code point c16 will be filled with UTF-16 representation of special
+ * replacement character U+FFFD and false will be returned.
+ * c32    -- UTF-32 Unicode code point.
+ * length -- were to put length of UTF-16 sequence.
+ * c16    -- where to put UTF-16 sequence (c16[0] -- high surrogate, c16[1] -- low surrogate)
+ * result -- true if c32 is a valid Unicode code point or false otherwise. */
+bool libutf_c32_to_c16(uint_least32_t c32, int *length, uint_least16_t c16[2]);
+
+/* Construct UTF-32 Unicode code point from UTF-16 surrogate pair. high
+ * must be high surrogate and low must be low surrogate otherwise *c32 will
+ * be filled with special replacement character U+FFFD and false will be returned.
+ * c16    -- where to put UTF-16 sequence (c32[0] -- high surrogate, c32[1] -- low surrogate)
+ * result -- true if c16 points to a valid UTF-16 sequence (single not surrogate
+ * or valid surrogate pair) or false otherwise. */
+bool libutf_c16_to_c32(uint_least16_t c16[2], uint_least32_t *c32);
+
+/* Check whether given value is a valid Unicode code point i.e. below 0x110000 and is not
+ * a UTF-16 surrogate. */
+bool libutf_c32_is_valid(uint_least32_t c32);
+
+#endif

src/quickjs/list.h

@@ -0,0 +1,99 @@
+/*
+ * Linux klist like system
+ *
+ * Copyright (c) 2016-2017 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef LIST_H
+#define LIST_H
+
+#ifndef NULL
+#include <stddef.h>
+#endif
+
+struct list_head {
+    struct list_head *prev;
+    struct list_head *next;
+};
+
+#define LIST_HEAD_INIT(el) { &(el), &(el) }
+
+/* return the pointer of type 'type *' containing 'el' as field 'member' */
+#define list_entry(el, type, member) container_of(el, type, member)
+
+static inline void init_list_head(struct list_head *head)
+{
+    head->prev = head;
+    head->next = head;
+}
+
+/* insert 'el' between 'prev' and 'next' */
+static inline void __list_add(struct list_head *el,
+                              struct list_head *prev, struct list_head *next)
+{
+    prev->next = el;
+    el->prev = prev;
+    el->next = next;
+    next->prev = el;
+}
+
+/* add 'el' at the head of the list 'head' (= after element head) */
+static inline void list_add(struct list_head *el, struct list_head *head)
+{
+    __list_add(el, head, head->next);
+}
+
+/* add 'el' at the end of the list 'head' (= before element head) */
+static inline void list_add_tail(struct list_head *el, struct list_head *head)
+{
+    __list_add(el, head->prev, head);
+}
+
+static inline void list_del(struct list_head *el)
+{
+    struct list_head *prev, *next;
+    prev = el->prev;
+    next = el->next;
+    prev->next = next;
+    next->prev = prev;
+    el->prev = NULL; /* fail safe */
+    el->next = NULL; /* fail safe */
+}
+
+static inline int list_empty(struct list_head *el)
+{
+    return el->next == el;
+}
+
+#define list_for_each(el, head) \
+  for(el = (head)->next; el != (head); el = el->next)
+
+#define list_for_each_safe(el, el1, head)                \
+    for(el = (head)->next, el1 = el->next; el != (head); \
+        el = el1, el1 = el->next)
+
+#define list_for_each_prev(el, head) \
+  for(el = (head)->prev; el != (head); el = el->prev)
+
+#define list_for_each_prev_safe(el, el1, head)           \
+    for(el = (head)->prev, el1 = el->prev; el != (head); \
+        el = el1, el1 = el->prev)
+
+#endif /* LIST_H */

src/quickjs/qjs.c

@@ -0,0 +1,552 @@
+/*
+ * QuickJS stand alone interpreter
+ *
+ * Copyright (c) 2017-2021 Fabrice Bellard
+ * Copyright (c) 2017-2021 Charlie Gordon
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <inttypes.h>
+#include <string.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <time.h>
+#if defined(__APPLE__)
+#include <malloc/malloc.h>
+#elif defined(__linux__)
+#include <malloc.h>
+#elif defined(__FreeBSD__)
+#include <malloc_np.h>
+#endif
+
+#include "cutils.h"
+#include "quickjs-libc.h"
+
+#ifdef CONFIG_BIGNUM
+static int bignum_ext;
+#endif
+
+static int eval_buf(JSContext *ctx, const void *buf, int buf_len,
+                    const char *filename, int eval_flags)
+{
+    JSValue val;
+    int ret;
+
+    if ((eval_flags & JS_EVAL_TYPE_MASK) == JS_EVAL_TYPE_MODULE) {
+        /* for the modules, we compile then run to be able to set
+           import.meta */
+        val = JS_Eval(ctx, buf, buf_len, filename,
+                      eval_flags | JS_EVAL_FLAG_COMPILE_ONLY);
+        if (!JS_IsException(val)) {
+            js_module_set_import_meta(ctx, val, TRUE, TRUE);
+            val = JS_EvalFunction(ctx, val);
+        }
+        val = js_std_await(ctx, val);
+    } else {
+        val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
+    }
+    if (JS_IsException(val)) {
+        js_std_dump_error(ctx);
+        ret = -1;
+    } else {
+        ret = 0;
+    }
+    JS_FreeValue(ctx, val);
+    return ret;
+}
+
+static int eval_file(JSContext *ctx, const char *filename, int module)
+{
+    uint8_t *buf;
+    int ret, eval_flags;
+    size_t buf_len;
+
+    buf = js_load_file(ctx, &buf_len, filename);
+    if (!buf) {
+        perror(filename);
+        exit(1);
+    }
+
+    if (module < 0) {
+        module = (has_suffix(filename, ".mjs") ||
+                  JS_DetectModule((const char *)buf, buf_len));
+    }
+    if (module)
+        eval_flags = JS_EVAL_TYPE_MODULE;
+    else
+        eval_flags = JS_EVAL_TYPE_GLOBAL;
+    ret = eval_buf(ctx, buf, buf_len, filename, eval_flags);
+    js_free(ctx, buf);
+    return ret;
+}
+
+/* also used to initialize the worker context */
+static JSContext *JS_NewCustomContext(JSRuntime *rt)
+{
+    JSContext *ctx;
+    ctx = JS_NewContext(rt);
+    if (!ctx)
+        return NULL;
+#ifdef CONFIG_BIGNUM
+    if (bignum_ext) {
+        JS_AddIntrinsicBigFloat(ctx);
+        JS_AddIntrinsicBigDecimal(ctx);
+        JS_AddIntrinsicOperators(ctx);
+        JS_EnableBignumExt(ctx, TRUE);
+    }
+#endif
+    /* system modules */
+    js_init_module_std(ctx, "std");
+    js_init_module_os(ctx, "os");
+    return ctx;
+}
+
+#if defined(__APPLE__)
+#define MALLOC_OVERHEAD  0
+#else
+#define MALLOC_OVERHEAD  8
+#endif
+
+struct trace_malloc_data {
+    uint8_t *base;
+};
+
+static inline unsigned long long js_trace_malloc_ptr_offset(uint8_t *ptr,
+                                                struct trace_malloc_data *dp)
+{
+    return ptr - dp->base;
+}
+
+/* default memory allocation functions with memory limitation */
+static size_t js_trace_malloc_usable_size(const void *ptr)
+{
+#if defined(__APPLE__)
+    return malloc_size(ptr);
+#elif defined(_WIN32)
+    return _msize((void *)ptr);
+#elif defined(EMSCRIPTEN)
+    return 0;
+#elif defined(__linux__)
+    return malloc_usable_size((void *)ptr);
+#else
+    /* change this to `return 0;` if compilation fails */
+    return malloc_usable_size((void *)ptr);
+#endif
+}
+
+static void
+#ifdef _WIN32
+/* mingw printf is used */
+__attribute__((format(gnu_printf, 2, 3)))
+#else
+__attribute__((format(printf, 2, 3)))
+#endif
+    js_trace_malloc_printf(JSMallocState *s, const char *fmt, ...)
+{
+    va_list ap;
+    int c;
+
+    va_start(ap, fmt);
+    while ((c = *fmt++) != '\0') {
+        if (c == '%') {
+            /* only handle %p and %zd */
+            if (*fmt == 'p') {
+                uint8_t *ptr = va_arg(ap, void *);
+                if (ptr == NULL) {
+                    printf("NULL");
+                } else {
+                    printf("H%+06lld.%zd",
+                           js_trace_malloc_ptr_offset(ptr, s->opaque),
+                           js_trace_malloc_usable_size(ptr));
+                }
+                fmt++;
+                continue;
+            }
+            if (fmt[0] == 'z' && fmt[1] == 'd') {
+                size_t sz = va_arg(ap, size_t);
+                printf("%zd", sz);
+                fmt += 2;
+                continue;
+            }
+        }
+        putc(c, stdout);
+    }
+    va_end(ap);
+}
+
+static void js_trace_malloc_init(struct trace_malloc_data *s)
+{
+    free(s->base = malloc(8));
+}
+
+static void *js_trace_malloc(JSMallocState *s, size_t size)
+{
+    void *ptr;
+
+    /* Do not allocate zero bytes: behavior is platform dependent */
+    assert(size != 0);
+
+    if (unlikely(s->malloc_size + size > s->malloc_limit))
+        return NULL;
+    ptr = malloc(size);
+    js_trace_malloc_printf(s, "A %zd -> %p\n", size, ptr);
+    if (ptr) {
+        s->malloc_count++;
+        s->malloc_size += js_trace_malloc_usable_size(ptr) + MALLOC_OVERHEAD;
+    }
+    return ptr;
+}
+
+static void js_trace_free(JSMallocState *s, void *ptr)
+{
+    if (!ptr)
+        return;
+
+    js_trace_malloc_printf(s, "F %p\n", ptr);
+    s->malloc_count--;
+    s->malloc_size -= js_trace_malloc_usable_size(ptr) + MALLOC_OVERHEAD;
+    free(ptr);
+}
+
+static void *js_trace_realloc(JSMallocState *s, void *ptr, size_t size)
+{
+    size_t old_size;
+
+    if (!ptr) {
+        if (size == 0)
+            return NULL;
+        return js_trace_malloc(s, size);
+    }
+    old_size = js_trace_malloc_usable_size(ptr);
+    if (size == 0) {
+        js_trace_malloc_printf(s, "R %zd %p\n", size, ptr);
+        s->malloc_count--;
+        s->malloc_size -= old_size + MALLOC_OVERHEAD;
+        free(ptr);
+        return NULL;
+    }
+    if (s->malloc_size + size - old_size > s->malloc_limit)
+        return NULL;
+
+    js_trace_malloc_printf(s, "R %zd %p", size, ptr);
+
+    ptr = realloc(ptr, size);
+    js_trace_malloc_printf(s, " -> %p\n", ptr);
+    if (ptr) {
+        s->malloc_size += js_trace_malloc_usable_size(ptr) - old_size;
+    }
+    return ptr;
+}
+
+static const JSMallocFunctions trace_mf = {
+    js_trace_malloc,
+    js_trace_free,
+    js_trace_realloc,
+    js_trace_malloc_usable_size,
+};
+
+#define PROG_NAME "qjs"
+
+void help(void)
+{
+    printf("QuickJS lite\n"
+           "usage: " PROG_NAME " [options] [file [args]]\n"
+           "-h  --help         list options\n"
+           "-e  --eval EXPR    evaluate EXPR\n"
+           "-i  --interactive  go to interactive mode\n"
+           "-m  --module       load as ES6 module (default=autodetect)\n"
+           "    --script       load as ES6 script (default=autodetect)\n"
+           "-I  --include file include an additional file\n"
+           "    --std          make 'std' and 'os' available to the loaded script\n"
+#ifdef CONFIG_BIGNUM
+           "    --bignum       enable the bignum extensions (BigFloat, BigDecimal)\n"
+           "    --qjscalc      load the QJSCalc runtime (default if invoked as qjscalc)\n"
+#endif
+           "-T  --trace        trace memory allocation\n"
+           "-d  --dump         dump the memory usage stats\n"
+           "    --memory-limit n       limit the memory usage to 'n' bytes\n"
+           "    --stack-size n         limit the stack size to 'n' bytes\n"
+           "    --unhandled-rejection  dump unhandled promise rejections\n"
+           "-q  --quit         just instantiate the interpreter and quit\n");
+    exit(1);
+}
+
+int main(int argc, char **argv)
+{
+    JSRuntime *rt;
+    JSContext *ctx;
+    struct trace_malloc_data trace_data = { NULL };
+    int optind;
+    char *expr = NULL;
+    int interactive = 0;
+    int dump_memory = 0;
+    int trace_memory = 0;
+    int empty_run = 0;
+    int module = -1;
+    int load_std = 0;
+    int dump_unhandled_promise_rejection = 0;
+    size_t memory_limit = 0;
+    char *include_list[32];
+    int i, include_count = 0;
+#ifdef CONFIG_BIGNUM
+    int load_jscalc;
+#endif
+    size_t stack_size = 0;
+
+#ifdef CONFIG_BIGNUM
+    /* load jscalc runtime if invoked as 'qjscalc' */
+    {
+        const char *p, *exename;
+        exename = argv[0];
+        p = strrchr(exename, '/');
+        if (p)
+            exename = p + 1;
+        load_jscalc = !strcmp(exename, "qjscalc");
+    }
+#endif
+
+    /* cannot use getopt because we want to pass the command line to
+       the script */
+    optind = 1;
+    while (optind < argc && *argv[optind] == '-') {
+        char *arg = argv[optind] + 1;
+        const char *longopt = "";
+        /* a single - is not an option, it also stops argument scanning */
+        if (!*arg)
+            break;
+        optind++;
+        if (*arg == '-') {
+            longopt = arg + 1;
+            arg += strlen(arg);
+            /* -- stops argument scanning */
+            if (!*longopt)
+                break;
+        }
+        for (; *arg || *longopt; longopt = "") {
+            char opt = *arg;
+            if (opt)
+                arg++;
+            if (opt == 'h' || opt == '?' || !strcmp(longopt, "help")) {
+                help();
+                continue;
+            }
+            if (opt == 'e' || !strcmp(longopt, "eval")) {
+                if (*arg) {
+                    expr = arg;
+                    break;
+                }
+                if (optind < argc) {
+                    expr = argv[optind++];
+                    break;
+                }
+                fprintf(stderr, "qjs: missing expression for -e\n");
+                exit(2);
+            }
+            if (opt == 'I' || !strcmp(longopt, "include")) {
+                if (optind >= argc) {
+                    fprintf(stderr, "expecting filename");
+                    exit(1);
+                }
+                if (include_count >= countof(include_list)) {
+                    fprintf(stderr, "too many included files");
+                    exit(1);
+                }
+                include_list[include_count++] = argv[optind++];
+                continue;
+            }
+            if (opt == 'i' || !strcmp(longopt, "interactive")) {
+                interactive++;
+                continue;
+            }
+            if (opt == 'm' || !strcmp(longopt, "module")) {
+                module = 1;
+                continue;
+            }
+            if (!strcmp(longopt, "script")) {
+                module = 0;
+                continue;
+            }
+            if (opt == 'd' || !strcmp(longopt, "dump")) {
+                dump_memory++;
+                continue;
+            }
+            if (opt == 'T' || !strcmp(longopt, "trace")) {
+                trace_memory++;
+                continue;
+            }
+            if (!strcmp(longopt, "std")) {
+                load_std = 1;
+                continue;
+            }
+            if (!strcmp(longopt, "unhandled-rejection")) {
+                dump_unhandled_promise_rejection = 1;
+                continue;
+            }
+#ifdef CONFIG_BIGNUM
+            if (!strcmp(longopt, "bignum")) {
+                bignum_ext = 1;
+                continue;
+            }
+            if (!strcmp(longopt, "qjscalc")) {
+                load_jscalc = 1;
+                continue;
+            }
+#endif
+            if (opt == 'q' || !strcmp(longopt, "quit")) {
+                empty_run++;
+                continue;
+            }
+            if (!strcmp(longopt, "memory-limit")) {
+                if (optind >= argc) {
+                    fprintf(stderr, "expecting memory limit");
+                    exit(1);
+                }
+                memory_limit = (size_t)strtod(argv[optind++], NULL);
+                continue;
+            }
+            if (!strcmp(longopt, "stack-size")) {
+                if (optind >= argc) {
+                    fprintf(stderr, "expecting stack size");
+                    exit(1);
+                }
+                stack_size = (size_t)strtod(argv[optind++], NULL);
+                continue;
+            }
+            if (opt) {
+                fprintf(stderr, "qjs: unknown option '-%c'\n", opt);
+            } else {
+                fprintf(stderr, "qjs: unknown option '--%s'\n", longopt);
+            }
+            help();
+        }
+    }
+
+#ifdef CONFIG_BIGNUM
+    if (load_jscalc)
+        bignum_ext = 1;
+#endif
+
+    if (trace_memory) {
+        js_trace_malloc_init(&trace_data);
+        rt = JS_NewRuntime2(&trace_mf, &trace_data);
+    } else {
+        rt = JS_NewRuntime();
+    }
+    if (!rt) {
+        fprintf(stderr, "qjs: cannot allocate JS runtime\n");
+        exit(2);
+    }
+    if (memory_limit != 0)
+        JS_SetMemoryLimit(rt, memory_limit);
+    if (stack_size != 0)
+        JS_SetMaxStackSize(rt, stack_size);
+    js_std_set_worker_new_context_func(JS_NewCustomContext);
+    js_std_init_handlers(rt);
+    ctx = JS_NewCustomContext(rt);
+    if (!ctx) {
+        fprintf(stderr, "qjs: cannot allocate JS context\n");
+        exit(2);
+    }
+
+    /* loader for ES6 modules */
+    JS_SetModuleLoaderFunc(rt, NULL, js_module_loader, NULL);
+
+    if (dump_unhandled_promise_rejection) {
+        JS_SetHostPromiseRejectionTracker(rt, js_std_promise_rejection_tracker,
+                                          NULL);
+    }
+
+    if (!empty_run) {
+        js_std_add_helpers(ctx, argc - optind, argv + optind);
+
+        /* make 'std' and 'os' visible to non module code */
+        if (load_std) {
+            const char *str = "import * as std from 'std';\n"
+                "import * as os from 'os';\n"
+                "globalThis.std = std;\n"
+                "globalThis.os = os;\n";
+            eval_buf(ctx, str, strlen(str), "<input>", JS_EVAL_TYPE_MODULE);
+        }
+
+        for(i = 0; i < include_count; i++) {
+            if (eval_file(ctx, include_list[i], module))
+                goto fail;
+        }
+
+        if (expr) {
+            if (eval_buf(ctx, expr, strlen(expr), "<cmdline>", 0))
+                goto fail;
+        } else
+        if (optind >= argc) {
+            /* interactive mode */
+            interactive = 1;
+        } else {
+            const char *filename;
+            filename = argv[optind];
+            if (eval_file(ctx, filename, module))
+                goto fail;
+        }
+        js_std_loop(ctx);
+    }
+
+    if (dump_memory) {
+        JSMemoryUsage stats;
+        JS_ComputeMemoryUsage(rt, &stats);
+        JS_DumpMemoryUsage(stdout, &stats, rt);
+    }
+    js_std_free_handlers(rt);
+    JS_FreeContext(ctx);
+    JS_FreeRuntime(rt);
+
+    if (empty_run && dump_memory) {
+        clock_t t[5];
+        double best[5];
+        int i, j;
+        for (i = 0; i < 100; i++) {
+            t[0] = clock();
+            rt = JS_NewRuntime();
+            t[1] = clock();
+            ctx = JS_NewContext(rt);
+            t[2] = clock();
+            JS_FreeContext(ctx);
+            t[3] = clock();
+            JS_FreeRuntime(rt);
+            t[4] = clock();
+            for (j = 4; j > 0; j--) {
+                double ms = 1000.0 * (t[j] - t[j - 1]) / CLOCKS_PER_SEC;
+                if (i == 0 || best[j] > ms)
+                    best[j] = ms;
+            }
+        }
+        printf("\nInstantiation times (ms): %.3f = %.3f+%.3f+%.3f+%.3f\n",
+               best[1] + best[2] + best[3] + best[4],
+               best[1], best[2], best[3], best[4]);
+    }
+    return 0;
+ fail:
+    js_std_free_handlers(rt);
+    JS_FreeContext(ctx);
+    JS_FreeRuntime(rt);
+    return 1;
+}

src/quickjs/qjsc.c

@@ -0,0 +1,761 @@
+/*
+ * QuickJS command line compiler
+ *
+ * Copyright (c) 2018-2021 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <inttypes.h>
+#include <string.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+#if !defined(_WIN32)
+#include <sys/wait.h>
+#endif
+
+#include "cutils.h"
+#include "quickjs-libc.h"
+
+typedef struct {
+    char *name;
+    char *short_name;
+    int flags;
+} namelist_entry_t;
+
+typedef struct namelist_t {
+    namelist_entry_t *array;
+    int count;
+    int size;
+} namelist_t;
+
+typedef struct {
+    const char *option_name;
+    const char *init_name;
+} FeatureEntry;
+
+static namelist_t cname_list;
+static namelist_t cmodule_list;
+static namelist_t init_module_list;
+static uint64_t feature_bitmap;
+static FILE *outfile;
+static BOOL byte_swap;
+static BOOL dynamic_export;
+static const char *c_ident_prefix = "qjsc_";
+
+#define FE_ALL (-1)
+
+static const FeatureEntry feature_list[] = {
+    { "date", "Date" },
+    { "eval", "Eval" },
+    { "string-normalize", "StringNormalize" },
+    { "regexp", "RegExp" },
+    { "json", "JSON" },
+    { "proxy", "Proxy" },
+    { "map", "MapSet" },
+    { "typedarray", "TypedArrays" },
+    { "promise", "Promise" },
+#define FE_MODULE_LOADER 9
+    { "module-loader", NULL },
+    { "bigint", "BigInt" },
+};
+
+void namelist_add(namelist_t *lp, const char *name, const char *short_name,
+                  int flags)
+{
+    namelist_entry_t *e;
+    if (lp->count == lp->size) {
+        size_t newsize = lp->size + (lp->size >> 1) + 4;
+        namelist_entry_t *a =
+            realloc(lp->array, sizeof(lp->array[0]) * newsize);
+        /* XXX: check for realloc failure */
+        lp->array = a;
+        lp->size = newsize;
+    }
+    e =  &lp->array[lp->count++];
+    e->name = strdup(name);
+    if (short_name)
+        e->short_name = strdup(short_name);
+    else
+        e->short_name = NULL;
+    e->flags = flags;
+}
+
+void namelist_free(namelist_t *lp)
+{
+    while (lp->count > 0) {
+        namelist_entry_t *e = &lp->array[--lp->count];
+        free(e->name);
+        free(e->short_name);
+    }
+    free(lp->array);
+    lp->array = NULL;
+    lp->size = 0;
+}
+
+namelist_entry_t *namelist_find(namelist_t *lp, const char *name)
+{
+    int i;
+    for(i = 0; i < lp->count; i++) {
+        namelist_entry_t *e = &lp->array[i];
+        if (!strcmp(e->name, name))
+            return e;
+    }
+    return NULL;
+}
+
+static void get_c_name(char *buf, size_t buf_size, const char *file)
+{
+    const char *p, *r;
+    size_t len, i;
+    int c;
+    char *q;
+
+    p = strrchr(file, '/');
+    if (!p)
+        p = file;
+    else
+        p++;
+    r = strrchr(p, '.');
+    if (!r)
+        len = strlen(p);
+    else
+        len = r - p;
+    pstrcpy(buf, buf_size, c_ident_prefix);
+    q = buf + strlen(buf);
+    for(i = 0; i < len; i++) {
+        c = p[i];
+        if (!((c >= '0' && c <= '9') ||
+              (c >= 'A' && c <= 'Z') ||
+              (c >= 'a' && c <= 'z'))) {
+            c = '_';
+        }
+        if ((q - buf) < buf_size - 1)
+            *q++ = c;
+    }
+    *q = '\0';
+}
+
+static void dump_hex(FILE *f, const uint8_t *buf, size_t len)
+{
+    size_t i, col;
+    col = 0;
+    for(i = 0; i < len; i++) {
+        fprintf(f, " 0x%02x,", buf[i]);
+        if (++col == 8) {
+            fprintf(f, "\n");
+            col = 0;
+        }
+    }
+    if (col != 0)
+        fprintf(f, "\n");
+}
+
+static void output_object_code(JSContext *ctx,
+                               FILE *fo, JSValueConst obj, const char *c_name,
+                               BOOL load_only)
+{
+    uint8_t *out_buf;
+    size_t out_buf_len;
+    int flags;
+    flags = JS_WRITE_OBJ_BYTECODE;
+    if (byte_swap)
+        flags |= JS_WRITE_OBJ_BSWAP;
+    out_buf = JS_WriteObject(ctx, &out_buf_len, obj, flags);
+    if (!out_buf) {
+        js_std_dump_error(ctx);
+        exit(1);
+    }
+
+    namelist_add(&cname_list, c_name, NULL, load_only);
+
+    fprintf(fo, "const uint32_t %s_size = %u;\n\n",
+            c_name, (unsigned int)out_buf_len);
+    fprintf(fo, "const uint8_t %s[%u] = {\n",
+            c_name, (unsigned int)out_buf_len);
+    dump_hex(fo, out_buf, out_buf_len);
+    fprintf(fo, "};\n\n");
+
+    js_free(ctx, out_buf);
+}
+
+static int js_module_dummy_init(JSContext *ctx, JSModuleDef *m)
+{
+    /* should never be called when compiling JS code */
+    abort();
+}
+
+static void find_unique_cname(char *cname, size_t cname_size)
+{
+    char cname1[1024];
+    int suffix_num;
+    size_t len, max_len;
+    assert(cname_size >= 32);
+    /* find a C name not matching an existing module C name by
+       adding a numeric suffix */
+    len = strlen(cname);
+    max_len = cname_size - 16;
+    if (len > max_len)
+        cname[max_len] = '\0';
+    suffix_num = 1;
+    for(;;) {
+        snprintf(cname1, sizeof(cname1), "%s_%d", cname, suffix_num);
+        if (!namelist_find(&cname_list, cname1))
+            break;
+        suffix_num++;
+    }
+    pstrcpy(cname, cname_size, cname1);
+}
+
+JSModuleDef *jsc_module_loader(JSContext *ctx,
+                              const char *module_name, void *opaque)
+{
+    JSModuleDef *m;
+    namelist_entry_t *e;
+
+    /* check if it is a declared C or system module */
+    e = namelist_find(&cmodule_list, module_name);
+    if (e) {
+        /* add in the static init module list */
+        namelist_add(&init_module_list, e->name, e->short_name, 0);
+        /* create a dummy module */
+        m = JS_NewCModule(ctx, module_name, js_module_dummy_init);
+    } else if (has_suffix(module_name, ".so")) {
+        fprintf(stderr, "Warning: binary module '%s' will be dynamically loaded\n", module_name);
+        /* create a dummy module */
+        m = JS_NewCModule(ctx, module_name, js_module_dummy_init);
+        /* the resulting executable will export its symbols for the
+           dynamic library */
+        dynamic_export = TRUE;
+    } else {
+        size_t buf_len;
+        uint8_t *buf;
+        JSValue func_val;
+        char cname[1024];
+
+        buf = js_load_file(ctx, &buf_len, module_name);
+        if (!buf) {
+            JS_ThrowReferenceError(ctx, "could not load module filename '%s'",
+                                   module_name);
+            return NULL;
+        }
+
+        /* compile the module */
+        func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name,
+                           JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY);
+        js_free(ctx, buf);
+        if (JS_IsException(func_val))
+            return NULL;
+        get_c_name(cname, sizeof(cname), module_name);
+        if (namelist_find(&cname_list, cname)) {
+            find_unique_cname(cname, sizeof(cname));
+        }
+        output_object_code(ctx, outfile, func_val, cname, TRUE);
+
+        /* the module is already referenced, so we must free it */
+        m = JS_VALUE_GET_PTR(func_val);
+        JS_FreeValue(ctx, func_val);
+    }
+    return m;
+}
+
+static void compile_file(JSContext *ctx, FILE *fo,
+                         const char *filename,
+                         const char *c_name1,
+                         int module)
+{
+    uint8_t *buf;
+    char c_name[1024];
+    int eval_flags;
+    JSValue obj;
+    size_t buf_len;
+
+    buf = js_load_file(ctx, &buf_len, filename);
+    if (!buf) {
+        fprintf(stderr, "Could not load '%s'\n", filename);
+        exit(1);
+    }
+    eval_flags = JS_EVAL_FLAG_COMPILE_ONLY;
+    if (module < 0) {
+        module = (has_suffix(filename, ".mjs") ||
+                  JS_DetectModule((const char *)buf, buf_len));
+    }
+    if (module)
+        eval_flags |= JS_EVAL_TYPE_MODULE;
+    else
+        eval_flags |= JS_EVAL_TYPE_GLOBAL;
+    obj = JS_Eval(ctx, (const char *)buf, buf_len, filename, eval_flags);
+    if (JS_IsException(obj)) {
+        js_std_dump_error(ctx);
+        exit(1);
+    }
+    js_free(ctx, buf);
+    if (c_name1) {
+        pstrcpy(c_name, sizeof(c_name), c_name1);
+    } else {
+        get_c_name(c_name, sizeof(c_name), filename);
+    }
+    output_object_code(ctx, fo, obj, c_name, FALSE);
+    JS_FreeValue(ctx, obj);
+}
+
+static const char main_c_template1[] =
+    "int main(int argc, char **argv)\n"
+    "{\n"
+    "  JSRuntime *rt;\n"
+    "  JSContext *ctx;\n"
+    "  rt = JS_NewRuntime();\n"
+    "  js_std_set_worker_new_context_func(JS_NewCustomContext);\n"
+    "  js_std_init_handlers(rt);\n"
+    ;
+
+static const char main_c_template2[] =
+    "  js_std_loop(ctx);\n"
+    "  js_std_free_handlers(rt);\n"
+    "  JS_FreeContext(ctx);\n"
+    "  JS_FreeRuntime(rt);\n"
+    "  return 0;\n"
+    "}\n";
+
+#define PROG_NAME "qjsc"
+
+void help(void)
+{
+    printf("QuickJS Compiler version " CONFIG_VERSION "\n"
+           "usage: " PROG_NAME " [options] [files]\n"
+           "\n"
+           "options are:\n"
+           "-c          only output bytecode to a C file\n"
+           "-e          output main() and bytecode to a C file (default = executable output)\n"
+           "-o output   set the output filename\n"
+           "-N cname    set the C name of the generated data\n"
+           "-m          compile as Javascript module (default=autodetect)\n"
+           "-D module_name         compile a dynamically loaded module or worker\n"
+           "-M module_name[,cname] add initialization code for an external C module\n"
+           "-x          byte swapped output\n"
+           "-p prefix   set the prefix of the generated C names\n"
+           "-S n        set the maximum stack size to 'n' bytes (default=%d)\n",
+           JS_DEFAULT_STACK_SIZE);
+#ifdef CONFIG_LTO
+    {
+        int i;
+        printf("-flto       use link time optimization\n");
+        printf("-fbignum    enable bignum extensions\n");
+        printf("-fno-[");
+        for(i = 0; i < countof(feature_list); i++) {
+            if (i != 0)
+                printf("|");
+            printf("%s", feature_list[i].option_name);
+        }
+        printf("]\n"
+               "            disable selected language features (smaller code size)\n");
+    }
+#endif
+    exit(1);
+}
+
+#if defined(CONFIG_CC) && !defined(_WIN32)
+
+int exec_cmd(char **argv)
+{
+    int pid, status, ret;
+
+    pid = fork();
+    if (pid == 0) {
+        execvp(argv[0], argv);
+        exit(1);
+    }
+
+    for(;;) {
+        ret = waitpid(pid, &status, 0);
+        if (ret == pid && WIFEXITED(status))
+            break;
+    }
+    return WEXITSTATUS(status);
+}
+
+static int output_executable(const char *out_filename, const char *cfilename,
+                             BOOL use_lto, BOOL verbose, const char *exename)
+{
+    const char *argv[64];
+    const char **arg, *bn_suffix, *lto_suffix;
+    char libjsname[1024];
+    char exe_dir[1024], inc_dir[1024], lib_dir[1024], buf[1024], *p;
+    int ret;
+
+    /* get the directory of the executable */
+    pstrcpy(exe_dir, sizeof(exe_dir), exename);
+    p = strrchr(exe_dir, '/');
+    if (p) {
+        *p = '\0';
+    } else {
+        pstrcpy(exe_dir, sizeof(exe_dir), ".");
+    }
+
+    /* if 'quickjs.h' is present at the same path as the executable, we
+       use it as include and lib directory */
+    snprintf(buf, sizeof(buf), "%s/quickjs.h", exe_dir);
+    if (access(buf, R_OK) == 0) {
+        pstrcpy(inc_dir, sizeof(inc_dir), exe_dir);
+        pstrcpy(lib_dir, sizeof(lib_dir), exe_dir);
+    } else {
+        snprintf(inc_dir, sizeof(inc_dir), "%s/include/quickjs", CONFIG_PREFIX);
+        snprintf(lib_dir, sizeof(lib_dir), "%s/lib/quickjs", CONFIG_PREFIX);
+    }
+
+    lto_suffix = "";
+    bn_suffix = "";
+
+    arg = argv;
+    *arg++ = CONFIG_CC;
+    *arg++ = "-O2";
+#ifdef CONFIG_LTO
+    if (use_lto) {
+        *arg++ = "-flto";
+        lto_suffix = ".lto";
+    }
+#endif
+    /* XXX: use the executable path to find the includes files and
+       libraries */
+    *arg++ = "-D";
+    *arg++ = "_GNU_SOURCE";
+    *arg++ = "-I";
+    *arg++ = inc_dir;
+    *arg++ = "-o";
+    *arg++ = out_filename;
+    if (dynamic_export)
+        *arg++ = "-rdynamic";
+    *arg++ = cfilename;
+    snprintf(libjsname, sizeof(libjsname), "%s/libquickjs%s%s.a",
+             lib_dir, bn_suffix, lto_suffix);
+    *arg++ = libjsname;
+    *arg++ = "-lm";
+    *arg++ = "-ldl";
+    *arg++ = "-lpthread";
+    *arg = NULL;
+
+    if (verbose) {
+        for(arg = argv; *arg != NULL; arg++)
+            printf("%s ", *arg);
+        printf("\n");
+    }
+
+    ret = exec_cmd((char **)argv);
+    unlink(cfilename);
+    return ret;
+}
+#else
+static int output_executable(const char *out_filename, const char *cfilename,
+                             BOOL use_lto, BOOL verbose, const char *exename)
+{
+    fprintf(stderr, "Executable output is not supported for this target\n");
+    exit(1);
+    return 0;
+}
+#endif
+
+
+typedef enum {
+    OUTPUT_C,
+    OUTPUT_C_MAIN,
+    OUTPUT_EXECUTABLE,
+} OutputTypeEnum;
+
+int main(int argc, char **argv)
+{
+    int c, i, verbose;
+    const char *out_filename, *cname;
+    char cfilename[1024];
+    FILE *fo;
+    JSRuntime *rt;
+    JSContext *ctx;
+    BOOL use_lto;
+    int module;
+    OutputTypeEnum output_type;
+    size_t stack_size;
+#ifdef CONFIG_BIGNUM
+    BOOL bignum_ext = FALSE;
+#endif
+    namelist_t dynamic_module_list;
+
+    out_filename = NULL;
+    output_type = OUTPUT_EXECUTABLE;
+    cname = NULL;
+    feature_bitmap = FE_ALL;
+    module = -1;
+    byte_swap = FALSE;
+    verbose = 0;
+    use_lto = FALSE;
+    stack_size = 0;
+    memset(&dynamic_module_list, 0, sizeof(dynamic_module_list));
+
+    /* add system modules */
+    namelist_add(&cmodule_list, "std", "std", 0);
+    namelist_add(&cmodule_list, "os", "os", 0);
+
+    for(;;) {
+        c = getopt(argc, argv, "ho:cN:f:mxevM:p:S:D:");
+        if (c == -1)
+            break;
+        switch(c) {
+        case 'h':
+            help();
+        case 'o':
+            out_filename = optarg;
+            break;
+        case 'c':
+            output_type = OUTPUT_C;
+            break;
+        case 'e':
+            output_type = OUTPUT_C_MAIN;
+            break;
+        case 'N':
+            cname = optarg;
+            break;
+        case 'f':
+            {
+                const char *p;
+                p = optarg;
+                if (!strcmp(optarg, "lto")) {
+                    use_lto = TRUE;
+                } else if (strstart(p, "no-", &p)) {
+                    use_lto = TRUE;
+                    for(i = 0; i < countof(feature_list); i++) {
+                        if (!strcmp(p, feature_list[i].option_name)) {
+                            feature_bitmap &= ~((uint64_t)1 << i);
+                            break;
+                        }
+                    }
+                    if (i == countof(feature_list))
+                        goto bad_feature;
+                } else
+#ifdef CONFIG_BIGNUM
+                if (!strcmp(optarg, "bignum")) {
+                    bignum_ext = TRUE;
+                } else
+#endif
+                {
+                bad_feature:
+                    fprintf(stderr, "unsupported feature: %s\n", optarg);
+                    exit(1);
+                }
+            }
+            break;
+        case 'm':
+            module = 1;
+            break;
+        case 'M':
+            {
+                char *p;
+                char path[1024];
+                char cname[1024];
+                pstrcpy(path, sizeof(path), optarg);
+                p = strchr(path, ',');
+                if (p) {
+                    *p = '\0';
+                    pstrcpy(cname, sizeof(cname), p + 1);
+                } else {
+                    get_c_name(cname, sizeof(cname), path);
+                }
+                namelist_add(&cmodule_list, path, cname, 0);
+            }
+            break;
+        case 'D':
+            namelist_add(&dynamic_module_list, optarg, NULL, 0);
+            break;
+        case 'x':
+            byte_swap = TRUE;
+            break;
+        case 'v':
+            verbose++;
+            break;
+        case 'p':
+            c_ident_prefix = optarg;
+            break;
+        case 'S':
+            stack_size = (size_t)strtod(optarg, NULL);
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (optind >= argc)
+        help();
+
+    if (!out_filename) {
+        if (output_type == OUTPUT_EXECUTABLE) {
+            out_filename = "a.out";
+        } else {
+            out_filename = "out.c";
+        }
+    }
+
+    if (output_type == OUTPUT_EXECUTABLE) {
+#if defined(_WIN32) || defined(__ANDROID__)
+        /* XXX: find a /tmp directory ? */
+        snprintf(cfilename, sizeof(cfilename), "out%d.c", getpid());
+#else
+        snprintf(cfilename, sizeof(cfilename), "/tmp/out%d.c", getpid());
+#endif
+    } else {
+        pstrcpy(cfilename, sizeof(cfilename), out_filename);
+    }
+
+    fo = fopen(cfilename, "w");
+    if (!fo) {
+        perror(cfilename);
+        exit(1);
+    }
+    outfile = fo;
+
+    rt = JS_NewRuntime();
+    ctx = JS_NewContext(rt);
+#ifdef CONFIG_BIGNUM
+    if (bignum_ext) {
+        JS_AddIntrinsicBigFloat(ctx);
+        JS_AddIntrinsicBigDecimal(ctx);
+        JS_AddIntrinsicOperators(ctx);
+        JS_EnableBignumExt(ctx, TRUE);
+    }
+#endif
+
+    /* loader for ES6 modules */
+    JS_SetModuleLoaderFunc(rt, NULL, jsc_module_loader, NULL);
+
+    fprintf(fo, "/* File generated automatically by the QuickJS compiler. */\n"
+            "\n"
+            );
+
+    if (output_type != OUTPUT_C) {
+        fprintf(fo, "#include \"quickjs-libc.h\"\n"
+                "\n"
+                );
+    } else {
+        fprintf(fo, "#include <inttypes.h>\n"
+                "\n"
+                );
+    }
+
+    for(i = optind; i < argc; i++) {
+        const char *filename = argv[i];
+        compile_file(ctx, fo, filename, cname, module);
+        cname = NULL;
+    }
+
+    for(i = 0; i < dynamic_module_list.count; i++) {
+        if (!jsc_module_loader(ctx, dynamic_module_list.array[i].name, NULL)) {
+            fprintf(stderr, "Could not load dynamic module '%s'\n",
+                    dynamic_module_list.array[i].name);
+            exit(1);
+        }
+    }
+
+    if (output_type != OUTPUT_C) {
+        fprintf(fo,
+                "static JSContext *JS_NewCustomContext(JSRuntime *rt)\n"
+                "{\n"
+                "  JSContext *ctx = JS_NewContextRaw(rt);\n"
+                "  if (!ctx)\n"
+                "    return NULL;\n");
+        /* add the basic objects */
+        fprintf(fo, "  JS_AddIntrinsicBaseObjects(ctx);\n");
+        for(i = 0; i < countof(feature_list); i++) {
+            if ((feature_bitmap & ((uint64_t)1 << i)) &&
+                feature_list[i].init_name) {
+                fprintf(fo, "  JS_AddIntrinsic%s(ctx);\n",
+                        feature_list[i].init_name);
+            }
+        }
+#ifdef CONFIG_BIGNUM
+        if (bignum_ext) {
+            fprintf(fo,
+                    "  JS_AddIntrinsicBigFloat(ctx);\n"
+                    "  JS_AddIntrinsicBigDecimal(ctx);\n"
+                    "  JS_AddIntrinsicOperators(ctx);\n"
+                    "  JS_EnableBignumExt(ctx, 1);\n");
+        }
+#endif
+        /* add the precompiled modules (XXX: could modify the module
+           loader instead) */
+        for(i = 0; i < init_module_list.count; i++) {
+            namelist_entry_t *e = &init_module_list.array[i];
+            /* initialize the static C modules */
+
+            fprintf(fo,
+                    "  {\n"
+                    "    extern JSModuleDef *js_init_module_%s(JSContext *ctx, const char *name);\n"
+                    "    js_init_module_%s(ctx, \"%s\");\n"
+                    "  }\n",
+                    e->short_name, e->short_name, e->name);
+        }
+        for(i = 0; i < cname_list.count; i++) {
+            namelist_entry_t *e = &cname_list.array[i];
+            if (e->flags) {
+                fprintf(fo, "  js_std_eval_binary(ctx, %s, %s_size, 1);\n",
+                        e->name, e->name);
+            }
+        }
+        fprintf(fo,
+                "  return ctx;\n"
+                "}\n\n");
+
+        fputs(main_c_template1, fo);
+
+        if (stack_size != 0) {
+            fprintf(fo, "  JS_SetMaxStackSize(rt, %u);\n",
+                    (unsigned int)stack_size);
+        }
+
+        /* add the module loader if necessary */
+        if (feature_bitmap & (1 << FE_MODULE_LOADER)) {
+            fprintf(fo, "  JS_SetModuleLoaderFunc(rt, NULL, js_module_loader, NULL);\n");
+        }
+
+        fprintf(fo,
+                "  ctx = JS_NewCustomContext(rt);\n"
+                "  js_std_add_helpers(ctx, argc, argv);\n");
+
+        for(i = 0; i < cname_list.count; i++) {
+            namelist_entry_t *e = &cname_list.array[i];
+            if (!e->flags) {
+                fprintf(fo, "  js_std_eval_binary(ctx, %s, %s_size, 0);\n",
+                        e->name, e->name);
+            }
+        }
+        fputs(main_c_template2, fo);
+    }
+
+    JS_FreeContext(ctx);
+    JS_FreeRuntime(rt);
+
+    fclose(fo);
+
+    if (output_type == OUTPUT_EXECUTABLE) {
+        return output_executable(out_filename, cfilename, use_lto, verbose,
+                                 argv[0]);
+    }
+    namelist_free(&cname_list);
+    namelist_free(&cmodule_list);
+    namelist_free(&init_module_list);
+    return 0;
+}

src/quickjs/qsort_r.c

@@ -0,0 +1,87 @@
+#include <sys/types.h>
+#include <stdlib.h>
+
+static void
+exch(char* base, size_t size, size_t a, size_t b) {
+  char* x = base + a * size;
+  char* y = base + b * size;
+  while(size) {
+    char z = *x;
+    *x = *y;
+    *y = z;
+    --size;
+    ++x;
+    ++y;
+  }
+}
+
+#define RAND
+
+/* Quicksort with 3-way partitioning, ala Sedgewick */
+/* Blame him for the scary variable names */
+/* http://www.cs.princeton.edu/~rs/talks/QuicksortIsOptimal.pdf */
+static void
+quicksort(char* base, size_t size, ssize_t l, ssize_t r, int (*compar)(const void*, const void*, void*), void* ptr) {
+  ssize_t i = l - 1, j = r, p = l - 1, q = r, k;
+  char* v = base + r * size;
+  if(r <= l)
+    return;
+
+#ifdef RAND
+  /*
+     We chose the rightmost element in the array to be sorted as pivot,
+     which is OK if the data is random, but which is horrible if the
+     data is already sorted.  Try to improve by exchanging it with a
+     random other pivot.
+   */
+  exch(base, size, l + (rand() % (r - l)), r);
+#elif defined MID
+  /*
+     We chose the rightmost element in the array to be sorted as pivot,
+     which is OK if the data is random, but which is horrible if the
+     data is already sorted.  Try to improve by chosing the middle
+     element instead.
+   */
+  exch(base, size, l + (r - l) / 2, r);
+#endif
+
+  for(;;) {
+    while(++i != r && compar(base + i * size, v, ptr) < 0)
+      ;
+    while(compar(v, base + (--j) * size, ptr) < 0)
+      if(j == l)
+        break;
+    if(i >= j)
+      break;
+    exch(base, size, i, j);
+    if(compar(base + i * size, v, ptr) == 0)
+      exch(base, size, ++p, i);
+    if(compar(v, base + j * size, ptr) == 0)
+      exch(base, size, j, --q);
+  }
+  exch(base, size, i, r);
+  j = i - 1;
+  ++i;
+  for(k = l; k < p; k++, j--)
+    exch(base, size, k, j);
+  for(k = r - 1; k > q; k--, i++)
+    exch(base, size, i, k);
+  quicksort(base, size, l, j, compar, ptr);
+  quicksort(base, size, i, r, compar, ptr);
+}
+
+void
+quicksort_r(void* base, size_t nmemb, size_t size, int (*compar)(const void*, const void*, void*), void* ptr) {
+  /* check for integer overflows */
+  if(nmemb >= (((size_t)-1) >> 1) || size >= (((size_t)-1) >> 1))
+    return;
+#if 0
+  if (sizeof(size_t) < sizeof(unsigned long long)) {
+    if ((unsigned long long)size * nmemb > (size_t)-1) return;
+  } else {
+    if (size*nmemb/nmemb != size) return;
+  }
+#endif
+  if(nmemb > 1)
+    quicksort(base, size, 0, nmemb - 1, compar, ptr);
+}

src/quickjs/quickjs-atom.h

@@ -0,0 +1,273 @@
+/*
+ * QuickJS atom definitions
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ * Copyright (c) 2017-2018 Charlie Gordon
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifdef DEF
+
+/* Note: first atoms are considered as keywords in the parser */
+DEF(null, "null") /* must be first */
+DEF(false, "false")
+DEF(true, "true")
+DEF(if, "if")
+DEF(else, "else")
+DEF(return, "return")
+DEF(var, "var")
+DEF(this, "this")
+DEF(delete, "delete")
+DEF(void, "void")
+DEF(typeof, "typeof")
+DEF(new, "new")
+DEF(in, "in")
+DEF(instanceof, "instanceof")
+DEF(do, "do")
+DEF(while, "while")
+DEF(for, "for")
+DEF(break, "break")
+DEF(continue, "continue")
+DEF(switch, "switch")
+DEF(case, "case")
+DEF(default, "default")
+DEF(throw, "throw")
+DEF(try, "try")
+DEF(catch, "catch")
+DEF(finally, "finally")
+DEF(function, "function")
+DEF(debugger, "debugger")
+DEF(with, "with")
+/* FutureReservedWord */
+DEF(class, "class")
+DEF(const, "const")
+DEF(enum, "enum")
+DEF(export, "export")
+DEF(extends, "extends")
+DEF(import, "import")
+DEF(super, "super")
+/* FutureReservedWords when parsing strict mode code */
+DEF(implements, "implements")
+DEF(interface, "interface")
+DEF(let, "let")
+DEF(package, "package")
+DEF(private, "private")
+DEF(protected, "protected")
+DEF(public, "public")
+DEF(static, "static")
+DEF(yield, "yield")
+DEF(await, "await")
+
+/* empty string */
+DEF(empty_string, "")
+/* identifiers */
+DEF(length, "length")
+DEF(fileName, "fileName")
+DEF(lineNumber, "lineNumber")
+DEF(message, "message")
+DEF(cause, "cause")
+DEF(errors, "errors")
+DEF(stack, "stack")
+DEF(name, "name")
+DEF(toString, "toString")
+DEF(toLocaleString, "toLocaleString")
+DEF(valueOf, "valueOf")
+DEF(eval, "eval")
+DEF(prototype, "prototype")
+DEF(constructor, "constructor")
+DEF(configurable, "configurable")
+DEF(writable, "writable")
+DEF(enumerable, "enumerable")
+DEF(value, "value")
+DEF(get, "get")
+DEF(set, "set")
+DEF(of, "of")
+DEF(__proto__, "__proto__")
+DEF(undefined, "undefined")
+DEF(number, "number")
+DEF(boolean, "boolean")
+DEF(string, "string")
+DEF(object, "object")
+DEF(symbol, "symbol")
+DEF(integer, "integer")
+DEF(unknown, "unknown")
+DEF(arguments, "arguments")
+DEF(callee, "callee")
+DEF(caller, "caller")
+DEF(_eval_, "<eval>")
+DEF(_ret_, "<ret>")
+DEF(_var_, "<var>")
+DEF(_arg_var_, "<arg_var>")
+DEF(_with_, "<with>")
+DEF(lastIndex, "lastIndex")
+DEF(target, "target")
+DEF(index, "index")
+DEF(input, "input")
+DEF(defineProperties, "defineProperties")
+DEF(apply, "apply")
+DEF(join, "join")
+DEF(concat, "concat")
+DEF(split, "split")
+DEF(construct, "construct")
+DEF(getPrototypeOf, "getPrototypeOf")
+DEF(setPrototypeOf, "setPrototypeOf")
+DEF(isExtensible, "isExtensible")
+DEF(preventExtensions, "preventExtensions")
+DEF(has, "has")
+DEF(deleteProperty, "deleteProperty")
+DEF(defineProperty, "defineProperty")
+DEF(getOwnPropertyDescriptor, "getOwnPropertyDescriptor")
+DEF(ownKeys, "ownKeys")
+DEF(add, "add")
+DEF(done, "done")
+DEF(next, "next")
+DEF(values, "values")
+DEF(source, "source")
+DEF(flags, "flags")
+DEF(global, "global")
+DEF(unicode, "unicode")
+DEF(raw, "raw")
+DEF(new_target, "new.target")
+DEF(this_active_func, "this.active_func")
+DEF(home_object, "<home_object>")
+DEF(computed_field, "<computed_field>")
+DEF(static_computed_field, "<static_computed_field>") /* must come after computed_fields */
+DEF(class_fields_init, "<class_fields_init>")
+DEF(brand, "<brand>")
+DEF(hash_constructor, "#constructor")
+DEF(as, "as")
+DEF(from, "from")
+DEF(meta, "meta")
+DEF(_default_, "*default*")
+DEF(_star_, "*")
+DEF(Module, "Module")
+DEF(then, "then")
+DEF(resolve, "resolve")
+DEF(reject, "reject")
+DEF(promise, "promise")
+DEF(proxy, "proxy")
+DEF(revoke, "revoke")
+DEF(async, "async")
+DEF(exec, "exec")
+DEF(groups, "groups")
+DEF(indices, "indices")
+DEF(status, "status")
+DEF(reason, "reason")
+DEF(globalThis, "globalThis")
+DEF(bigint, "bigint")
+#ifdef CONFIG_BIGNUM
+DEF(bigfloat, "bigfloat")
+DEF(bigdecimal, "bigdecimal")
+DEF(roundingMode, "roundingMode")
+DEF(maximumSignificantDigits, "maximumSignificantDigits")
+DEF(maximumFractionDigits, "maximumFractionDigits")
+#endif
+/* the following 3 atoms are only used with CONFIG_ATOMICS */
+DEF(not_equal, "not-equal")
+DEF(timed_out, "timed-out")
+DEF(ok, "ok")
+/* */
+DEF(toJSON, "toJSON")
+/* class names */
+DEF(Object, "Object")
+DEF(Array, "Array")
+DEF(Error, "Error")
+DEF(Number, "Number")
+DEF(String, "String")
+DEF(Boolean, "Boolean")
+DEF(Symbol, "Symbol")
+DEF(Arguments, "Arguments")
+DEF(Math, "Math")
+DEF(JSON, "JSON")
+DEF(Date, "Date")
+DEF(Function, "Function")
+DEF(GeneratorFunction, "GeneratorFunction")
+DEF(ForInIterator, "ForInIterator")
+DEF(RegExp, "RegExp")
+DEF(ArrayBuffer, "ArrayBuffer")
+DEF(SharedArrayBuffer, "SharedArrayBuffer")
+/* must keep same order as class IDs for typed arrays */
+DEF(Uint8ClampedArray, "Uint8ClampedArray")
+DEF(Int8Array, "Int8Array")
+DEF(Uint8Array, "Uint8Array")
+DEF(Int16Array, "Int16Array")
+DEF(Uint16Array, "Uint16Array")
+DEF(Int32Array, "Int32Array")
+DEF(Uint32Array, "Uint32Array")
+DEF(BigInt64Array, "BigInt64Array")
+DEF(BigUint64Array, "BigUint64Array")
+DEF(Float32Array, "Float32Array")
+DEF(Float64Array, "Float64Array")
+DEF(DataView, "DataView")
+DEF(BigInt, "BigInt")
+#ifdef CONFIG_BIGNUM
+DEF(BigFloat, "BigFloat")
+DEF(BigFloatEnv, "BigFloatEnv")
+DEF(BigDecimal, "BigDecimal")
+DEF(OperatorSet, "OperatorSet")
+DEF(Operators, "Operators")
+#endif
+DEF(Map, "Map")
+DEF(Set, "Set") /* Map + 1 */
+DEF(WeakMap, "WeakMap") /* Map + 2 */
+DEF(WeakSet, "WeakSet") /* Map + 3 */
+DEF(Map_Iterator, "Map Iterator")
+DEF(Set_Iterator, "Set Iterator")
+DEF(Array_Iterator, "Array Iterator")
+DEF(String_Iterator, "String Iterator")
+DEF(RegExp_String_Iterator, "RegExp String Iterator")
+DEF(Generator, "Generator")
+DEF(Proxy, "Proxy")
+DEF(Promise, "Promise")
+DEF(PromiseResolveFunction, "PromiseResolveFunction")
+DEF(PromiseRejectFunction, "PromiseRejectFunction")
+DEF(AsyncFunction, "AsyncFunction")
+DEF(AsyncFunctionResolve, "AsyncFunctionResolve")
+DEF(AsyncFunctionReject, "AsyncFunctionReject")
+DEF(AsyncGeneratorFunction, "AsyncGeneratorFunction")
+DEF(AsyncGenerator, "AsyncGenerator")
+DEF(EvalError, "EvalError")
+DEF(RangeError, "RangeError")
+DEF(ReferenceError, "ReferenceError")
+DEF(SyntaxError, "SyntaxError")
+DEF(TypeError, "TypeError")
+DEF(URIError, "URIError")
+DEF(InternalError, "InternalError")
+/* private symbols */
+DEF(Private_brand, "<brand>")
+/* symbols */
+DEF(Symbol_toPrimitive, "Symbol.toPrimitive")
+DEF(Symbol_iterator, "Symbol.iterator")
+DEF(Symbol_match, "Symbol.match")
+DEF(Symbol_matchAll, "Symbol.matchAll")
+DEF(Symbol_replace, "Symbol.replace")
+DEF(Symbol_search, "Symbol.search")
+DEF(Symbol_split, "Symbol.split")
+DEF(Symbol_toStringTag, "Symbol.toStringTag")
+DEF(Symbol_isConcatSpreadable, "Symbol.isConcatSpreadable")
+DEF(Symbol_hasInstance, "Symbol.hasInstance")
+DEF(Symbol_species, "Symbol.species")
+DEF(Symbol_unscopables, "Symbol.unscopables")
+DEF(Symbol_asyncIterator, "Symbol.asyncIterator")
+#ifdef CONFIG_BIGNUM
+DEF(Symbol_operatorSet, "Symbol.operatorSet")
+#endif
+
+#endif /* DEF */

src/quickjs/quickjs-config.h

@@ -0,0 +1,8 @@
+#ifndef __QUICKJS_CONFIG_H
+#define __QUICKJS_CONFIG_H
+
+#ifdef _WIN32
+#define _WIN32_WINNT 0x0600
+#endif
+
+#endif	// __QUICKJS_CONFIG_H

src/quickjs/quickjs-internal.c

@@ -0,0 +1,645 @@
+#include "utils.h"
+#include "char-utils.h"
+#include "vector.h"
+#include "quickjs-internal.h"
+
+JSValue
+js_std_file(JSContext* ctx, FILE* f) {
+  JSClassID class_id = js_class_find(ctx, "FILE");
+  JSValue obj, proto = JS_GetClassProto(ctx, class_id);
+  JSSTDFile* file;
+
+  file = js_malloc(ctx, sizeof(JSSTDFile));
+  *file = (JSSTDFile){0, TRUE, FALSE};
+
+  file->f = f;
+
+  obj = JS_NewObjectProtoClass(ctx, proto, class_id);
+
+  JS_SetOpaque(obj, file);
+
+  return obj;
+}
+
+struct list_head*
+js_modules_list(JSContext* ctx) {
+  return &ctx->loaded_modules;
+}
+
+JSModuleDef**
+js_modules_vector(JSContext* ctx) {
+  struct list_head* el;
+  Vector vec = VECTOR(ctx);
+  JSModuleDef* m;
+
+  list_for_each(el, js_modules_list(ctx)) {
+    m = list_entry(el, JSModuleDef, link);
+
+    vector_push(&vec, m);
+  }
+
+  m = NULL;
+  vector_push(&vec, m);
+
+  return vector_begin(&vec);
+}
+
+JSValue
+js_modules_entries(JSContext* ctx, JSValueConst this_val, int magic) {
+  struct list_head* el;
+  JSValue ret = JS_NewArray(ctx);
+  uint32_t i = 0;
+  list_for_each(el, &ctx->loaded_modules) {
+    JSModuleDef* m = list_entry(el, JSModuleDef, link);
+    // const char* name = module_namecstr(ctx, m);
+    JSValue entry = JS_NewArray(ctx);
+    JS_SetPropertyUint32(ctx, entry, 0, JS_AtomToValue(ctx, m->module_name));
+    JS_SetPropertyUint32(ctx, entry, 1, magic ? module_entry(ctx, m) : module_value(ctx, m));
+
+    if(1 /*str[0] != '<'*/)
+      JS_SetPropertyUint32(ctx, ret, i++, entry);
+    else
+      JS_FreeValue(ctx, entry);
+
+    // JS_FreeCString(ctx, name);
+  }
+
+  return ret;
+}
+
+JSValue
+js_modules_object(JSContext* ctx, JSValueConst this_val, int magic) {
+  struct list_head* it;
+  JSValue obj = JS_NewObject(ctx);
+
+  list_for_each(it, &ctx->loaded_modules) {
+    JSModuleDef* m = list_entry(it, JSModuleDef, link);
+    const char* name = module_namecstr(ctx, m);
+    JSValue entry = magic ? module_entry(ctx, m) : module_value(ctx, m);
+
+    if(1 /*str[0] != '<'*/)
+      JS_SetPropertyStr(ctx, obj, basename(name), entry);
+    else
+      JS_FreeValue(ctx, entry);
+
+    JS_FreeCString(ctx, name);
+  }
+
+  return obj;
+}
+
+JSModuleDef*
+js_module_find_fwd(JSContext* ctx, const char* name, JSModuleDef* start) {
+  struct list_head* el;
+
+  for(el = start ? &start->link : ctx->loaded_modules.next; el != &ctx->loaded_modules; el = el->next)
+  /*list_for_each(el, &ctx->loaded_modules)*/ {
+    JSModuleDef* m = list_entry(el, JSModuleDef, link);
+    const char* str = module_namecstr(ctx, m);
+    BOOL match = !strcmp(str, name);
+    JS_FreeCString(ctx, str);
+
+    if(match)
+      return m;
+  }
+
+  return 0;
+}
+
+int
+js_module_index(JSContext* ctx, JSModuleDef* m) {
+  struct list_head* el;
+  int i = 0;
+
+  list_for_each(el, &ctx->loaded_modules) {
+
+    if(m == list_entry(el, JSModuleDef, link))
+      return i;
+    ++i;
+  }
+
+  return -1;
+}
+
+JSModuleDef*
+js_module_find_rev(JSContext* ctx, const char* name, JSModuleDef* start) {
+  struct list_head* el;
+
+  for(el = start ? &start->link : ctx->loaded_modules.prev; el != &ctx->loaded_modules; el = el->prev) /*list_for_each_prev(el, &ctx->loaded_modules)*/ {
+    JSModuleDef* m = list_entry(el, JSModuleDef, link);
+    const char* str = module_namecstr(ctx, m);
+    BOOL match = !strcmp(str, name);
+    JS_FreeCString(ctx, str);
+
+    if(match)
+      return m;
+  }
+
+  return 0;
+}
+
+int
+js_module_indexof(JSContext* ctx, JSModuleDef* def) {
+  int i = 0;
+  struct list_head* el;
+
+  list_for_each(el, &ctx->loaded_modules) {
+    JSModuleDef* m = list_entry(el, JSModuleDef, link);
+
+    if(m == def)
+      return i;
+
+    ++i;
+  }
+
+  return -1;
+}
+
+JSModuleDef*
+js_module_at(JSContext* ctx, int index) {
+  int i = 0;
+  struct list_head* el;
+
+  if(index >= 0) {
+    list_for_each(el, &ctx->loaded_modules) {
+      JSModuleDef* m = list_entry(el, JSModuleDef, link);
+
+      if(index == i)
+        return m;
+
+      ++i;
+    }
+  } else {
+    index = -(index + 1);
+
+    list_for_each_prev(el, &ctx->loaded_modules) {
+      JSModuleDef* m = list_entry(el, JSModuleDef, link);
+
+      if(index == i)
+        return m;
+
+      ++i;
+    }
+  }
+
+  return 0;
+}
+
+void
+module_make_object(JSContext* ctx, JSModuleDef* m, JSValueConst obj) {
+  JSValue tmp;
+  char buf[FMT_XLONG + 2];
+  strcpy(buf, "0x");
+
+  if(!js_has_propertystr(ctx, obj, "name"))
+    JS_SetPropertyStr(ctx, obj, "name", module_name(ctx, m));
+
+  JS_DefinePropertyValueStr(ctx, obj, "resolved", JS_NewBool(ctx, m->resolved), 0);
+  JS_DefinePropertyValueStr(ctx, obj, "funcCreated", JS_NewBool(ctx, m->func_created), 0);
+  JS_DefinePropertyValueStr(ctx, obj, "instantiated", JS_NewBool(ctx, m->instantiated), 0);
+  JS_DefinePropertyValueStr(ctx, obj, "evaluated", JS_NewBool(ctx, m->evaluated), 0);
+
+  if(!JS_IsUndefined((tmp = module_ns(ctx, m))))
+    JS_DefinePropertyValueStr(ctx, obj, "ns", tmp, 0);
+
+  if(!JS_IsUndefined((tmp = module_exports(ctx, m))))
+    JS_DefinePropertyValueStr(ctx, obj, "exports", tmp, 0);
+
+  if(!JS_IsUndefined((tmp = module_imports(ctx, m))))
+    JS_DefinePropertyValueStr(ctx, obj, "imports", tmp, 0);
+
+  if(!JS_IsUndefined((tmp = module_reqmodules(ctx, m))))
+    JS_SetPropertyStr(ctx, obj, "reqModules", tmp);
+
+  if(m->init_func) {
+    JS_SetPropertyStr(ctx, obj, "native", JS_NewBool(ctx, m->init_func != NULL));
+  }
+
+  if(!JS_IsUndefined((tmp = module_func(ctx, m)))) {
+    if(m->init_func)
+      JS_DefinePropertyValueStr(ctx, obj, "initFunc", tmp, 0);
+    else
+      JS_SetPropertyStr(ctx, obj, "func", tmp);
+  }
+
+  if(!js_is_null_or_undefined((tmp = JS_DupValue(ctx, m->meta_obj))))
+    JS_SetPropertyStr(ctx, obj, "metaObj", tmp);
+
+  if(!js_is_null_or_undefined((tmp = JS_DupValue(ctx, m->eval_exception))))
+    JS_SetPropertyStr(ctx, obj, "evalException", tmp);
+
+  {
+    JSAtom atom = js_symbol_static_atom(ctx, "toStringTag");
+    JS_DefinePropertyValue(ctx, obj, atom, JS_NewString(ctx, "Module"), 0);
+    JS_FreeAtom(ctx, atom);
+  }
+
+  {
+
+    buf[2 + fmt_xlonglong0(&buf[2], (long long)(uintptr_t)m, __SIZEOF_POINTER__ * 2)] = 0;
+
+    JS_DefinePropertyValueStr(ctx, obj, "address", JS_NewString(ctx, buf), 0);
+  }
+}
+
+JSValue
+module_object(JSContext* ctx, JSModuleDef* m) {
+  JSValue obj = JS_NewObject(ctx);
+  module_make_object(ctx, m, obj);
+  return obj;
+}
+
+int
+module_exports_get(JSContext* ctx, JSModuleDef* m, BOOL rename_default, JSValueConst exports) {
+  JSAtom def = JS_NewAtom(ctx, "default");
+  int i;
+
+  for(i = 0; i < m->export_entries_count; i++) {
+    JSExportEntry* entry = &m->export_entries[i];
+    JSVarRef* ref = entry->u.local.var_ref;
+    JSValue val = JS_UNDEFINED;
+    JSAtom name = entry->export_name;
+
+    if(ref) {
+      val = JS_DupValue(ctx, ref->pvalue ? *ref->pvalue : ref->value);
+
+      if(rename_default && name == def)
+        name = m->module_name;
+    }
+
+    JS_SetProperty(ctx, exports, name, val);
+  }
+
+  JS_FreeAtom(ctx, def);
+  return i;
+}
+
+JSValue
+module_imports(JSContext* ctx, JSModuleDef* m) {
+  JSValue obj = m->import_entries_count > 0 ? JS_NewArray(ctx) : JS_UNDEFINED;
+  int i;
+
+  for(i = 0; i < m->import_entries_count; i++) {
+    JSImportEntry* entry = &m->import_entries[i];
+    JSAtom name = entry->import_name;
+    /*JSReqModuleEntry* req_module = &m->req_module_entries[entry->req_module_idx];
+    JSAtom module_name = req_module->module_name;*/
+
+    JSValue import_value = JS_NewArray(ctx);
+
+    JS_SetPropertyUint32(ctx, import_value, 0, JS_AtomToValue(ctx, name));
+    JS_SetPropertyUint32(ctx, import_value, 1, JS_NewUint32(ctx, entry->req_module_idx));
+    JS_SetPropertyUint32(ctx, obj, i, import_value);
+  }
+
+  return obj;
+}
+
+JSValue
+module_reqmodules(JSContext* ctx, JSModuleDef* m) {
+  JSValue obj = m->req_module_entries_count > 0 ? JS_NewArray(ctx) : JS_UNDEFINED;
+  int i;
+
+  for(i = 0; i < m->req_module_entries_count; i++) {
+    JSReqModuleEntry* req_module = &m->req_module_entries[i];
+    JSAtom module_name = req_module->module_name;
+    JSModuleDef* module = req_module->module;
+
+    JSValue req_module_value = JS_NewArray(ctx);
+
+    JS_SetPropertyUint32(ctx, req_module_value, 0, JS_AtomToValue(ctx, module_name));
+    JS_SetPropertyUint32(ctx, req_module_value, 1, JS_NewInt32(ctx, js_module_index(ctx, module)));
+    JS_SetPropertyUint32(ctx, obj, i, req_module_value);
+  }
+
+  return obj;
+}
+
+JSValue
+module_default_export(JSContext* ctx, JSModuleDef* m) {
+  JSAtom def = JS_NewAtom(ctx, "default");
+  JSValue ret = JS_UNDEFINED;
+  int i;
+
+  for(i = 0; i < m->export_entries_count; i++) {
+    JSExportEntry* entry = &m->export_entries[i];
+    JSVarRef* ref = entry->u.local.var_ref;
+    JSAtom name = entry->export_name;
+
+    if(ref) {
+
+      if(name == def) {
+        ret = JS_DupValue(ctx, ref->pvalue ? *ref->pvalue : ref->value);
+        break;
+      }
+    }
+  }
+
+  JS_FreeAtom(ctx, def);
+  return ret;
+}
+
+JSValue
+module_ns(JSContext* ctx, JSModuleDef* m) {
+  return JS_DupValue(ctx, m->module_ns);
+}
+
+JSValue
+module_exception(JSContext* ctx, JSModuleDef* m) {
+  return m->eval_has_exception ? JS_DupValue(ctx, m->eval_exception) : JS_NULL;
+}
+
+JSValue
+module_meta_obj(JSContext* ctx, JSModuleDef* m) {
+  return JS_DupValue(ctx, m->meta_obj);
+}
+
+static JSValue
+call_module_func(JSContext* ctx, JSValueConst this_val, int argc, JSValueConst* argv, int magic, JSValue* data) {
+  union {
+    JSModuleInitFunc* init_func;
+    int32_t i[2];
+  } u;
+
+  u.i[0] = JS_VALUE_GET_INT(data[0]);
+  u.i[1] = JS_VALUE_GET_INT(data[1]);
+
+  if(argc >= 1) {
+    JSModuleDef* m;
+
+    if((m = js_module_def(ctx, argv[0])))
+      return JS_NewInt32(ctx, u.init_func(ctx, m));
+  }
+
+  return JS_ThrowTypeError(ctx, "argument 1 module expected");
+}
+
+JSValue
+module_func(JSContext* ctx, JSModuleDef* m) {
+  JSValue func = JS_UNDEFINED;
+
+  if(JS_IsFunction(ctx, m->func_obj)) {
+    func = JS_DupValue(ctx, m->func_obj);
+  } else if(m->init_func) {
+    union {
+      JSModuleInitFunc* init_func;
+      int32_t i[2];
+    } u = {m->init_func};
+
+    JSValueConst data[2] = {
+        JS_MKVAL(JS_TAG_INT, u.i[0]),
+        JS_MKVAL(JS_TAG_INT, u.i[1]),
+    };
+
+    func = JS_NewCFunctionData(ctx, call_module_func, 1, 0, 2, data);
+  }
+
+  return func;
+}
+
+JSValue
+module_name(JSContext* ctx, JSModuleDef* m) {
+
+  if(m->module_name < (size_t)JS_GetRuntime(ctx)->atom_count)
+    return JS_AtomToValue(ctx, m->module_name);
+
+  return JS_UNDEFINED;
+}
+
+const char*
+module_namecstr(JSContext* ctx, JSModuleDef* m) {
+  return JS_AtomToCString(ctx, m->module_name);
+}
+
+JSValue
+module_exports_find(JSContext* ctx, JSModuleDef* m, JSAtom atom) {
+  int i;
+
+  for(i = 0; i < m->export_entries_count; i++) {
+    JSExportEntry* entry = &m->export_entries[i];
+
+    if(entry->export_name == atom) {
+      JSVarRef* ref = entry->u.local.var_ref;
+      JSValue export = ref ? JS_DupValue(ctx, ref->pvalue ? *ref->pvalue : ref->value) : JS_UNDEFINED;
+      return export;
+    }
+  }
+
+  return JS_UNDEFINED;
+}
+
+JSModuleDef*
+module_next(JSContext* ctx, JSModuleDef* m) {
+  return m->link.next != js_modules_list(ctx) ? list_entry(m->link.next, JSModuleDef, link) : 0;
+}
+
+JSModuleDef*
+module_prev(JSContext* ctx, JSModuleDef* m) {
+  return m->link.prev != js_modules_list(ctx) ? list_entry(m->link.prev, JSModuleDef, link) : 0;
+}
+
+JSModuleDef*
+module_last(JSContext* ctx) {
+  struct list_head* list = js_modules_list(ctx);
+
+  return list_empty(list) ? 0 : list_entry(list->prev, JSModuleDef, link);
+}
+
+void
+module_rename(JSContext* ctx, JSModuleDef* m, JSAtom name) {
+  JS_FreeAtom(ctx, m->module_name);
+  m->module_name = name;
+}
+
+static void
+js_arraybuffer_freestring(JSRuntime* rt, void* opaque, void* ptr) {
+  JSString* jstr = opaque;
+  JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_STRING, jstr));
+}
+
+JSValue
+js_arraybuffer_fromstring(JSContext* ctx, JSValueConst str) {
+  JSString* jstr;
+
+  if(!JS_IsString(str))
+    return JS_ThrowTypeError(ctx, "Not a string");
+
+  JS_DupValue(ctx, str);
+  jstr = JS_VALUE_GET_PTR(str);
+
+  return JS_NewArrayBuffer(ctx, jstr->u.str8, jstr->len, js_arraybuffer_freestring, jstr, FALSE);
+}
+
+void*
+js_sab_alloc(void* opaque, size_t size) {
+  JSSABHeader* sab;
+
+  if(!(sab = malloc(sizeof(JSSABHeader) + size)))
+    return 0;
+
+  sab->ref_count = 1;
+  return sab->buf;
+}
+
+void
+js_sab_free(void* opaque, void* ptr) {
+  JSSABHeader* sab;
+  int ref_count;
+  sab = (JSSABHeader*)((uint8_t*)ptr - sizeof(JSSABHeader));
+  ref_count = atomic_add_int(&sab->ref_count, -1);
+  assert(ref_count >= 0);
+
+  if(ref_count == 0)
+    free(sab);
+}
+
+void
+js_sab_dup(void* opaque, void* ptr) {
+  JSSABHeader* sab;
+  sab = (JSSABHeader*)((uint8_t*)ptr - sizeof(JSSABHeader));
+  atomic_add_int(&sab->ref_count, 1);
+}
+
+JSValueConst
+js_cstring_value(const char* ptr) {
+  return JS_MKPTR(JS_TAG_STRING, (JSString*)(void*)(ptr - offsetof(JSString, u)));
+}
+
+char*
+js_cstring_dup(JSContext* ctx, const char* str) {
+  /* purposely removing constness */
+  JSString* p = (JSString*)(void*)(str - offsetof(JSString, u));
+  JS_DupValue(ctx, JS_MKPTR(JS_TAG_STRING, p));
+  return (char*)str;
+}
+
+size_t
+js_cstring_len(JSValueConst v) {
+  JSString* p;
+
+  if(JS_IsString(v)) {
+    p = JS_VALUE_GET_PTR(v);
+    return p->len;
+  }
+
+  return 0;
+}
+
+char*
+js_cstring_ptr(JSValueConst v) {
+  JSString* p;
+
+  if(JS_IsString(v)) {
+    p = JS_VALUE_GET_PTR(v);
+    return (char*)p->u.str8;
+  }
+
+  return 0;
+}
+
+const char*
+js_class_name(JSContext* ctx, JSClassID id) {
+  JSAtom atom = JS_GetRuntime(ctx)->class_array[id].class_name;
+  return JS_AtomToCString(ctx, atom);
+}
+
+JSAtom
+js_class_atom(JSContext* ctx, JSClassID id) {
+  JSAtom atom = 0;
+
+  if(id > 0 && id < (JSClassID)JS_GetRuntime(ctx)->class_count)
+    atom = JS_GetRuntime(ctx)->class_array[id].class_name;
+  return atom;
+}
+
+JSClassID
+js_class_find(JSContext* ctx, const char* name) {
+  JSAtom atom = JS_NewAtom(ctx, name);
+  JSRuntime* rt = JS_GetRuntime(ctx);
+  int i, n = rt->class_count;
+
+  for(i = 0; i < n; i++)
+
+    if(rt->class_array[i].class_name == atom)
+      return i;
+
+  return -1;
+}
+
+JSClassID
+js_class_id(JSContext* ctx, int id) {
+  return JS_GetRuntime(ctx)->class_array[id].class_id;
+}
+
+JSValue
+js_opcode_array(JSContext* ctx, const JSOpCode* opcode) {
+  JSValue ret = JS_NewArray(ctx);
+  JS_SetPropertyUint32(ctx, ret, 0, JS_NewUint32(ctx, opcode->size));
+  JS_SetPropertyUint32(ctx, ret, 1, JS_NewUint32(ctx, opcode->n_pop));
+  JS_SetPropertyUint32(ctx, ret, 2, JS_NewUint32(ctx, opcode->n_push));
+  JS_SetPropertyUint32(ctx, ret, 3, JS_NewUint32(ctx, opcode->fmt));
+  JS_SetPropertyUint32(ctx, ret, 4, JS_NewString(ctx, opcode->name));
+  return ret;
+}
+
+JSValue
+js_opcode_object(JSContext* ctx, const struct JSOpCode* opcode) {
+  JSValue ret = JS_NewObject(ctx);
+  JS_SetPropertyStr(ctx, ret, "size", JS_NewUint32(ctx, opcode->size));
+  JS_SetPropertyStr(ctx, ret, "n_pop", JS_NewUint32(ctx, opcode->n_pop));
+  JS_SetPropertyStr(ctx, ret, "n_push", JS_NewUint32(ctx, opcode->n_push));
+  JS_SetPropertyStr(ctx, ret, "fmt", JS_NewUint32(ctx, opcode->fmt));
+  JS_SetPropertyStr(ctx, ret, "name", JS_NewString(ctx, opcode->name));
+  return ret;
+}
+
+JSValue
+js_get_bytecode(JSContext* ctx, JSValueConst value) {
+  JSValue ret = JS_UNDEFINED;
+
+  if(JS_IsFunction(ctx, value)) {
+    JSObject* obj = JS_VALUE_GET_OBJ(value);
+    JSFunctionBytecode* fnbc;
+
+    if((fnbc = obj->u.func.function_bytecode)) {
+      ret = JS_NewArrayBufferCopy(ctx, fnbc->byte_code_buf, fnbc->byte_code_len);
+    }
+  }
+
+  return ret;
+}
+
+JSValue
+js_opcode_list(JSContext* ctx, BOOL as_object) {
+  JSValue ret = JS_NewArray(ctx);
+  size_t i, j, len = countof(js_opcodes);
+
+  for(i = 0, j = 0; i < len; i++) {
+
+    if(i >= OP_TEMP_START && i < OP_TEMP_END)
+      continue;
+
+    JS_SetPropertyUint32(ctx, ret, j++, (as_object ? js_opcode_object : js_opcode_array)(ctx, &js_opcodes[i]));
+  }
+
+  return ret;
+}
+
+#ifdef HAVE_JS_DEBUGGER_BUILD_BACKTRACE
+JSValue js_debugger_build_backtrace(JSContext* ctx, const uint8_t* cur_pc);
+
+JSValue
+js_stack_get(JSContext* ctx) {
+  return js_debugger_build_backtrace(ctx, ctx->rt->current_stack_frame->cur_pc);
+}
+#endif
+
+#define SHORT_OPCODES 1
+
+const JSOpCode js_opcodes[OP_COUNT + (OP_TEMP_END - OP_TEMP_START)] = {
+#define FMT(f)
+#define DEF(id, size, n_pop, n_push, f) {size, n_pop, n_push, OP_FMT_##f, #id},
+#include <quickjs-opcode.h>
+#undef DEF
+#undef FMT
+};

src/quickjs/quickjs-libc.h

@@ -0,0 +1,62 @@
+/*
+ * QuickJS C library
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef QUICKJS_LIBC_H
+#define QUICKJS_LIBC_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "quickjs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JSModuleDef *js_init_module_std(JSContext *ctx, const char *module_name);
+JSModuleDef *js_init_module_os(JSContext *ctx, const char *module_name);
+void js_std_add_helpers(JSContext *ctx, int argc, char **argv);
+void js_std_loop(JSContext *ctx);
+JSValue js_std_await(JSContext *ctx, JSValue obj);
+void js_std_init_handlers(JSRuntime *rt);
+void js_std_free_handlers(JSRuntime *rt);
+void js_std_dump_error(JSContext *ctx);
+uint8_t *js_load_file(JSContext *ctx, size_t *pbuf_len, const char *filename);
+int js_module_set_import_meta(JSContext *ctx, JSValueConst func_val,
+                              JS_BOOL use_realpath, JS_BOOL is_main);
+JSModuleDef *js_module_loader(JSContext *ctx,
+                              const char *module_name, void *opaque);
+
+int js_std_eval_binary_bare(JSContext *ctx, const uint8_t *buf, size_t buf_len);
+void js_std_eval_binary(JSContext *ctx, const uint8_t *buf, size_t buf_len,
+                        int flags);
+void js_std_promise_rejection_tracker(JSContext *ctx, JSValueConst promise,
+                                      JSValueConst reason,
+                                      JS_BOOL is_handled, void *opaque);
+void js_std_set_worker_new_context_func(JSContext *(*func)(JSRuntime *rt));
+
+#ifdef __cplusplus
+} /* extern "C" { */
+#endif
+
+#endif /* QUICKJS_LIBC_H */

src/quickjs/quickjs-opcode.h

@@ -0,0 +1,372 @@
+/*
+ * QuickJS opcode definitions
+ *
+ * Copyright (c) 2017-2018 Fabrice Bellard
+ * Copyright (c) 2017-2018 Charlie Gordon
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifdef FMT
+FMT(none)
+FMT(none_int)
+FMT(none_loc)
+FMT(none_arg)
+FMT(none_var_ref)
+FMT(u8)
+FMT(i8)
+FMT(loc8)
+FMT(const8)
+FMT(label8)
+FMT(u16)
+FMT(i16)
+FMT(label16)
+FMT(npop)
+FMT(npopx)
+FMT(npop_u16)
+FMT(loc)
+FMT(arg)
+FMT(var_ref)
+FMT(u32)
+FMT(i32)
+FMT(const)
+FMT(label)
+FMT(atom)
+FMT(atom_u8)
+FMT(atom_u16)
+FMT(atom_label_u8)
+FMT(atom_label_u16)
+FMT(label_u16)
+#undef FMT
+#endif /* FMT */
+
+#ifdef DEF
+
+#ifndef def
+#define def(id, size, n_pop, n_push, f) DEF(id, size, n_pop, n_push, f)
+#endif
+
+DEF(invalid, 1, 0, 0, none) /* never emitted */
+
+/* push values */
+DEF(       push_i32, 5, 0, 1, i32)
+DEF(     push_const, 5, 0, 1, const)
+DEF(       fclosure, 5, 0, 1, const) /* must follow push_const */
+DEF(push_atom_value, 5, 0, 1, atom)
+DEF( private_symbol, 5, 0, 1, atom)
+DEF(      undefined, 1, 0, 1, none)
+DEF(           null, 1, 0, 1, none)
+DEF(      push_this, 1, 0, 1, none) /* only used at the start of a function */
+DEF(     push_false, 1, 0, 1, none)
+DEF(      push_true, 1, 0, 1, none)
+DEF(         object, 1, 0, 1, none)
+DEF( special_object, 2, 0, 1, u8) /* only used at the start of a function */
+DEF(           rest, 3, 0, 1, u16) /* only used at the start of a function */
+
+DEF(           drop, 1, 1, 0, none) /* a -> */
+DEF(            nip, 1, 2, 1, none) /* a b -> b */
+DEF(           nip1, 1, 3, 2, none) /* a b c -> b c */
+DEF(            dup, 1, 1, 2, none) /* a -> a a */
+DEF(           dup1, 1, 2, 3, none) /* a b -> a a b */
+DEF(           dup2, 1, 2, 4, none) /* a b -> a b a b */
+DEF(           dup3, 1, 3, 6, none) /* a b c -> a b c a b c */
+DEF(        insert2, 1, 2, 3, none) /* obj a -> a obj a (dup_x1) */
+DEF(        insert3, 1, 3, 4, none) /* obj prop a -> a obj prop a (dup_x2) */
+DEF(        insert4, 1, 4, 5, none) /* this obj prop a -> a this obj prop a */
+DEF(          perm3, 1, 3, 3, none) /* obj a b -> a obj b */
+DEF(          perm4, 1, 4, 4, none) /* obj prop a b -> a obj prop b */
+DEF(          perm5, 1, 5, 5, none) /* this obj prop a b -> a this obj prop b */
+DEF(           swap, 1, 2, 2, none) /* a b -> b a */
+DEF(          swap2, 1, 4, 4, none) /* a b c d -> c d a b */
+DEF(          rot3l, 1, 3, 3, none) /* x a b -> a b x */
+DEF(          rot3r, 1, 3, 3, none) /* a b x -> x a b */
+DEF(          rot4l, 1, 4, 4, none) /* x a b c -> a b c x */
+DEF(          rot5l, 1, 5, 5, none) /* x a b c d -> a b c d x */
+
+DEF(call_constructor, 3, 2, 1, npop) /* func new.target args -> ret. arguments are not counted in n_pop */
+DEF(           call, 3, 1, 1, npop) /* arguments are not counted in n_pop */
+DEF(      tail_call, 3, 1, 0, npop) /* arguments are not counted in n_pop */
+DEF(    call_method, 3, 2, 1, npop) /* arguments are not counted in n_pop */
+DEF(tail_call_method, 3, 2, 0, npop) /* arguments are not counted in n_pop */
+DEF(     array_from, 3, 0, 1, npop) /* arguments are not counted in n_pop */
+DEF(          apply, 3, 3, 1, u16)
+DEF(         return, 1, 1, 0, none)
+DEF(   return_undef, 1, 0, 0, none)
+DEF(check_ctor_return, 1, 1, 2, none)
+DEF(     check_ctor, 1, 0, 0, none)
+DEF(    check_brand, 1, 2, 2, none) /* this_obj func -> this_obj func */
+DEF(      add_brand, 1, 2, 0, none) /* this_obj home_obj -> */
+DEF(   return_async, 1, 1, 0, none)
+DEF(          throw, 1, 1, 0, none)
+DEF(    throw_error, 6, 0, 0, atom_u8)
+DEF(           eval, 5, 1, 1, npop_u16) /* func args... -> ret_val */
+DEF(     apply_eval, 3, 2, 1, u16) /* func array -> ret_eval */
+DEF(         regexp, 1, 2, 1, none) /* create a RegExp object from the pattern and a
+                                       bytecode string */
+DEF(      get_super, 1, 1, 1, none)
+DEF(         import, 1, 1, 1, none) /* dynamic module import */
+
+DEF(      check_var, 5, 0, 1, atom) /* check if a variable exists */
+DEF(  get_var_undef, 5, 0, 1, atom) /* push undefined if the variable does not exist */
+DEF(        get_var, 5, 0, 1, atom) /* throw an exception if the variable does not exist */
+DEF(        put_var, 5, 1, 0, atom) /* must come after get_var */
+DEF(   put_var_init, 5, 1, 0, atom) /* must come after put_var. Used to initialize a global lexical variable */
+DEF( put_var_strict, 5, 2, 0, atom) /* for strict mode variable write */
+
+DEF(  get_ref_value, 1, 2, 3, none)
+DEF(  put_ref_value, 1, 3, 0, none)
+
+DEF(     define_var, 6, 0, 0, atom_u8)
+DEF(check_define_var, 6, 0, 0, atom_u8)
+DEF(    define_func, 6, 1, 0, atom_u8)
+DEF(      get_field, 5, 1, 1, atom)
+DEF(     get_field2, 5, 1, 2, atom)
+DEF(      put_field, 5, 2, 0, atom)
+DEF( get_private_field, 1, 2, 1, none) /* obj prop -> value */
+DEF( put_private_field, 1, 3, 0, none) /* obj value prop -> */
+DEF(define_private_field, 1, 3, 1, none) /* obj prop value -> obj */
+DEF(   get_array_el, 1, 2, 1, none)
+DEF(  get_array_el2, 1, 2, 2, none) /* obj prop -> obj value */
+DEF(   put_array_el, 1, 3, 0, none)
+DEF(get_super_value, 1, 3, 1, none) /* this obj prop -> value */
+DEF(put_super_value, 1, 4, 0, none) /* this obj prop value -> */
+DEF(   define_field, 5, 2, 1, atom)
+DEF(       set_name, 5, 1, 1, atom)
+DEF(set_name_computed, 1, 2, 2, none)
+DEF(      set_proto, 1, 2, 1, none)
+DEF(set_home_object, 1, 2, 2, none)
+DEF(define_array_el, 1, 3, 2, none)
+DEF(         append, 1, 3, 2, none) /* append enumerated object, update length */
+DEF(copy_data_properties, 2, 3, 3, u8)
+DEF(  define_method, 6, 2, 1, atom_u8)
+DEF(define_method_computed, 2, 3, 1, u8) /* must come after define_method */
+DEF(   define_class, 6, 2, 2, atom_u8) /* parent ctor -> ctor proto */
+DEF(   define_class_computed, 6, 3, 3, atom_u8) /* field_name parent ctor -> field_name ctor proto (class with computed name) */
+
+DEF(        get_loc, 3, 0, 1, loc)
+DEF(        put_loc, 3, 1, 0, loc) /* must come after get_loc */
+DEF(        set_loc, 3, 1, 1, loc) /* must come after put_loc */
+DEF(        get_arg, 3, 0, 1, arg)
+DEF(        put_arg, 3, 1, 0, arg) /* must come after get_arg */
+DEF(        set_arg, 3, 1, 1, arg) /* must come after put_arg */
+DEF(    get_var_ref, 3, 0, 1, var_ref)
+DEF(    put_var_ref, 3, 1, 0, var_ref) /* must come after get_var_ref */
+DEF(    set_var_ref, 3, 1, 1, var_ref) /* must come after put_var_ref */
+DEF(set_loc_uninitialized, 3, 0, 0, loc)
+DEF(  get_loc_check, 3, 0, 1, loc)
+DEF(  put_loc_check, 3, 1, 0, loc) /* must come after get_loc_check */
+DEF(  put_loc_check_init, 3, 1, 0, loc)
+DEF(get_loc_checkthis, 3, 0, 1, loc)
+DEF(get_var_ref_check, 3, 0, 1, var_ref)
+DEF(put_var_ref_check, 3, 1, 0, var_ref) /* must come after get_var_ref_check */
+DEF(put_var_ref_check_init, 3, 1, 0, var_ref)
+DEF(      close_loc, 3, 0, 0, loc)
+DEF(       if_false, 5, 1, 0, label)
+DEF(        if_true, 5, 1, 0, label) /* must come after if_false */
+DEF(           goto, 5, 0, 0, label) /* must come after if_true */
+DEF(          catch, 5, 0, 1, label)
+DEF(          gosub, 5, 0, 0, label) /* used to execute the finally block */
+DEF(            ret, 1, 1, 0, none) /* used to return from the finally block */
+DEF(      nip_catch, 1, 2, 1, none) /* catch ... a -> a */
+
+DEF(      to_object, 1, 1, 1, none)
+//DEF(      to_string, 1, 1, 1, none)
+DEF(     to_propkey, 1, 1, 1, none)
+DEF(    to_propkey2, 1, 2, 2, none)
+
+DEF(   with_get_var, 10, 1, 0, atom_label_u8)     /* must be in the same order as scope_xxx */
+DEF(   with_put_var, 10, 2, 1, atom_label_u8)     /* must be in the same order as scope_xxx */
+DEF(with_delete_var, 10, 1, 0, atom_label_u8)     /* must be in the same order as scope_xxx */
+DEF(  with_make_ref, 10, 1, 0, atom_label_u8)     /* must be in the same order as scope_xxx */
+DEF(   with_get_ref, 10, 1, 0, atom_label_u8)     /* must be in the same order as scope_xxx */
+DEF(with_get_ref_undef, 10, 1, 0, atom_label_u8)
+
+DEF(   make_loc_ref, 7, 0, 2, atom_u16)
+DEF(   make_arg_ref, 7, 0, 2, atom_u16)
+DEF(make_var_ref_ref, 7, 0, 2, atom_u16)
+DEF(   make_var_ref, 5, 0, 2, atom)
+
+DEF(   for_in_start, 1, 1, 1, none)
+DEF(   for_of_start, 1, 1, 3, none)
+DEF(for_await_of_start, 1, 1, 3, none)
+DEF(    for_in_next, 1, 1, 3, none)
+DEF(    for_of_next, 2, 3, 5, u8)
+DEF(iterator_check_object, 1, 1, 1, none)
+DEF(iterator_get_value_done, 1, 1, 2, none)
+DEF( iterator_close, 1, 3, 0, none)
+DEF(  iterator_next, 1, 4, 4, none)
+DEF(  iterator_call, 2, 4, 5, u8)
+DEF(  initial_yield, 1, 0, 0, none)
+DEF(          yield, 1, 1, 2, none)
+DEF(     yield_star, 1, 1, 2, none)
+DEF(async_yield_star, 1, 1, 2, none)
+DEF(          await, 1, 1, 1, none)
+
+/* arithmetic/logic operations */
+DEF(            neg, 1, 1, 1, none)
+DEF(           plus, 1, 1, 1, none)
+DEF(            dec, 1, 1, 1, none)
+DEF(            inc, 1, 1, 1, none)
+DEF(       post_dec, 1, 1, 2, none)
+DEF(       post_inc, 1, 1, 2, none)
+DEF(        dec_loc, 2, 0, 0, loc8)
+DEF(        inc_loc, 2, 0, 0, loc8)
+DEF(        add_loc, 2, 1, 0, loc8)
+DEF(            not, 1, 1, 1, none)
+DEF(           lnot, 1, 1, 1, none)
+DEF(         typeof, 1, 1, 1, none)
+DEF(         delete, 1, 2, 1, none)
+DEF(     delete_var, 5, 0, 1, atom)
+
+DEF(            mul, 1, 2, 1, none)
+DEF(            div, 1, 2, 1, none)
+DEF(            mod, 1, 2, 1, none)
+DEF(            add, 1, 2, 1, none)
+DEF(            sub, 1, 2, 1, none)
+DEF(            pow, 1, 2, 1, none)
+DEF(            shl, 1, 2, 1, none)
+DEF(            sar, 1, 2, 1, none)
+DEF(            shr, 1, 2, 1, none)
+DEF(             lt, 1, 2, 1, none)
+DEF(            lte, 1, 2, 1, none)
+DEF(             gt, 1, 2, 1, none)
+DEF(            gte, 1, 2, 1, none)
+DEF(     instanceof, 1, 2, 1, none)
+DEF(             in, 1, 2, 1, none)
+DEF(             eq, 1, 2, 1, none)
+DEF(            neq, 1, 2, 1, none)
+DEF(      strict_eq, 1, 2, 1, none)
+DEF(     strict_neq, 1, 2, 1, none)
+DEF(            and, 1, 2, 1, none)
+DEF(            xor, 1, 2, 1, none)
+DEF(             or, 1, 2, 1, none)
+DEF(is_undefined_or_null, 1, 1, 1, none)
+DEF(     private_in, 1, 2, 1, none)
+#ifdef CONFIG_BIGNUM
+DEF(      mul_pow10, 1, 2, 1, none)
+DEF(       math_mod, 1, 2, 1, none)
+#endif
+/* must be the last non short and non temporary opcode */
+DEF(            nop, 1, 0, 0, none)
+
+/* temporary opcodes: never emitted in the final bytecode */
+
+def(    enter_scope, 3, 0, 0, u16)  /* emitted in phase 1, removed in phase 2 */
+def(    leave_scope, 3, 0, 0, u16)  /* emitted in phase 1, removed in phase 2 */
+
+def(          label, 5, 0, 0, label) /* emitted in phase 1, removed in phase 3 */
+
+/* the following opcodes must be in the same order as the 'with_x' and
+   get_var_undef, get_var and put_var opcodes */
+def(scope_get_var_undef, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def(  scope_get_var, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def(  scope_put_var, 7, 1, 0, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def(scope_delete_var, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def( scope_make_ref, 11, 0, 2, atom_label_u16) /* emitted in phase 1, removed in phase 2 */
+def(  scope_get_ref, 7, 0, 2, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def(scope_put_var_init, 7, 0, 2, atom_u16) /* emitted in phase 1, removed in phase 2 */
+def(scope_get_var_checkthis, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2, only used to return 'this' in derived class constructors */
+def(scope_get_private_field, 7, 1, 1, atom_u16) /* obj -> value, emitted in phase 1, removed in phase 2 */
+def(scope_get_private_field2, 7, 1, 2, atom_u16) /* obj -> obj value, emitted in phase 1, removed in phase 2 */
+def(scope_put_private_field, 7, 2, 0, atom_u16) /* obj value ->, emitted in phase 1, removed in phase 2 */
+def(scope_in_private_field, 7, 1, 1, atom_u16) /* obj -> res emitted in phase 1, removed in phase 2 */
+def(get_field_opt_chain, 5, 1, 1, atom) /* emitted in phase 1, removed in phase 2 */
+def(get_array_el_opt_chain, 1, 2, 1, none) /* emitted in phase 1, removed in phase 2 */
+def( set_class_name, 5, 1, 1, u32) /* emitted in phase 1, removed in phase 2 */
+
+def(       line_num, 5, 0, 0, u32) /* emitted in phase 1, removed in phase 3 */
+
+#if SHORT_OPCODES
+DEF(    push_minus1, 1, 0, 1, none_int)
+DEF(         push_0, 1, 0, 1, none_int)
+DEF(         push_1, 1, 0, 1, none_int)
+DEF(         push_2, 1, 0, 1, none_int)
+DEF(         push_3, 1, 0, 1, none_int)
+DEF(         push_4, 1, 0, 1, none_int)
+DEF(         push_5, 1, 0, 1, none_int)
+DEF(         push_6, 1, 0, 1, none_int)
+DEF(         push_7, 1, 0, 1, none_int)
+DEF(        push_i8, 2, 0, 1, i8)
+DEF(       push_i16, 3, 0, 1, i16)
+DEF(    push_const8, 2, 0, 1, const8)
+DEF(      fclosure8, 2, 0, 1, const8) /* must follow push_const8 */
+DEF(push_empty_string, 1, 0, 1, none)
+
+DEF(       get_loc8, 2, 0, 1, loc8)
+DEF(       put_loc8, 2, 1, 0, loc8)
+DEF(       set_loc8, 2, 1, 1, loc8)
+
+DEF(       get_loc0, 1, 0, 1, none_loc)
+DEF(       get_loc1, 1, 0, 1, none_loc)
+DEF(       get_loc2, 1, 0, 1, none_loc)
+DEF(       get_loc3, 1, 0, 1, none_loc)
+DEF(       put_loc0, 1, 1, 0, none_loc)
+DEF(       put_loc1, 1, 1, 0, none_loc)
+DEF(       put_loc2, 1, 1, 0, none_loc)
+DEF(       put_loc3, 1, 1, 0, none_loc)
+DEF(       set_loc0, 1, 1, 1, none_loc)
+DEF(       set_loc1, 1, 1, 1, none_loc)
+DEF(       set_loc2, 1, 1, 1, none_loc)
+DEF(       set_loc3, 1, 1, 1, none_loc)
+DEF(       get_arg0, 1, 0, 1, none_arg)
+DEF(       get_arg1, 1, 0, 1, none_arg)
+DEF(       get_arg2, 1, 0, 1, none_arg)
+DEF(       get_arg3, 1, 0, 1, none_arg)
+DEF(       put_arg0, 1, 1, 0, none_arg)
+DEF(       put_arg1, 1, 1, 0, none_arg)
+DEF(       put_arg2, 1, 1, 0, none_arg)
+DEF(       put_arg3, 1, 1, 0, none_arg)
+DEF(       set_arg0, 1, 1, 1, none_arg)
+DEF(       set_arg1, 1, 1, 1, none_arg)
+DEF(       set_arg2, 1, 1, 1, none_arg)
+DEF(       set_arg3, 1, 1, 1, none_arg)
+DEF(   get_var_ref0, 1, 0, 1, none_var_ref)
+DEF(   get_var_ref1, 1, 0, 1, none_var_ref)
+DEF(   get_var_ref2, 1, 0, 1, none_var_ref)
+DEF(   get_var_ref3, 1, 0, 1, none_var_ref)
+DEF(   put_var_ref0, 1, 1, 0, none_var_ref)
+DEF(   put_var_ref1, 1, 1, 0, none_var_ref)
+DEF(   put_var_ref2, 1, 1, 0, none_var_ref)
+DEF(   put_var_ref3, 1, 1, 0, none_var_ref)
+DEF(   set_var_ref0, 1, 1, 1, none_var_ref)
+DEF(   set_var_ref1, 1, 1, 1, none_var_ref)
+DEF(   set_var_ref2, 1, 1, 1, none_var_ref)
+DEF(   set_var_ref3, 1, 1, 1, none_var_ref)
+
+DEF(     get_length, 1, 1, 1, none)
+
+DEF(      if_false8, 2, 1, 0, label8)
+DEF(       if_true8, 2, 1, 0, label8) /* must come after if_false8 */
+DEF(          goto8, 2, 0, 0, label8) /* must come after if_true8 */
+DEF(         goto16, 3, 0, 0, label16)
+
+DEF(          call0, 1, 1, 1, npopx)
+DEF(          call1, 1, 1, 1, npopx)
+DEF(          call2, 1, 1, 1, npopx)
+DEF(          call3, 1, 1, 1, npopx)
+
+DEF(   is_undefined, 1, 1, 1, none)
+DEF(        is_null, 1, 1, 1, none)
+DEF(typeof_is_undefined, 1, 1, 1, none)
+DEF( typeof_is_function, 1, 1, 1, none)
+#endif
+
+#undef DEF
+#undef def
+#endif  /* DEF */

src/quickjs/quickjs-textcode.c

@@ -0,0 +1,689 @@
+#include "defines.h"
+#include "quickjs-textcode.h"
+#include "utils.h"
+#include "buffer-utils.h"
+#include "debug.h"
+#include <libutf.h>
+#include <libutf.h>
+#include "tutf8e.h"
+
+/**
+ * \addtogroup quickjs-textdecoder
+ * @{
+ */
+
+VISIBLE JSClassID js_decoder_class_id = 0, js_encoder_class_id = 0;
+VISIBLE JSValue textdecoder_proto = {{0}, JS_TAG_UNDEFINED}, textdecoder_ctor = {{0}, JS_TAG_UNDEFINED}, textencoder_proto = {{0}, JS_TAG_UNDEFINED},
+                textencoder_ctor = {{0}, JS_TAG_UNDEFINED};
+
+const TUTF8encoder* tutf8e_coders[] = {
+    /* 0, 0, 0, 0, 0, 0, 0, 0, */
+    &tutf8e_encoder_iso_8859_1,   &tutf8e_encoder_iso_8859_2,   &tutf8e_encoder_iso_8859_3,   &tutf8e_encoder_iso_8859_4,   &tutf8e_encoder_iso_8859_5,
+    &tutf8e_encoder_iso_8859_6,   &tutf8e_encoder_iso_8859_7,   &tutf8e_encoder_iso_8859_8,   &tutf8e_encoder_iso_8859_9,   &tutf8e_encoder_iso_8859_10,
+    &tutf8e_encoder_iso_8859_11,  &tutf8e_encoder_iso_8859_13,  &tutf8e_encoder_iso_8859_14,  &tutf8e_encoder_iso_8859_15,  &tutf8e_encoder_iso_8859_16,
+    &tutf8e_encoder_windows_1250, &tutf8e_encoder_windows_1251, &tutf8e_encoder_windows_1252, &tutf8e_encoder_windows_1253, &tutf8e_encoder_windows_1254,
+    &tutf8e_encoder_windows_1255, &tutf8e_encoder_windows_1256, &tutf8e_encoder_windows_1257, &tutf8e_encoder_windows_1258,
+};
+
+const char* const textcode_encodings[] = {
+    "unknown",      "UTF-8",        "UTF-16",       "UTF-32",       "unknown",      "UTF-8",        "UTF-16BE",     "UTF-32BE",
+    "ISO-8859-1",   "ISO-8859-2",   "ISO-8859-3",   "ISO-8859-4",   "ISO-8859-5",   "ISO-8859-6",   "ISO-8859-7",   "ISO-8859-8",
+    "ISO-8859-9",   "ISO-8859-10",  "ISO-8859-11",  "ISO-8859-13",  "ISO-8859-14",  "ISO-8859-15",  "ISO-8859-16",  "WINDOWS-1250",
+    "WINDOWS-1251", "WINDOWS-1252", "WINDOWS-1253", "WINDOWS-1254", "WINDOWS-1255", "WINDOWS-1256", "WINDOWS-1257", "WINDOWS-1258",
+};
+
+enum {
+  DECODER_DECODE,
+  DECODER_END,
+};
+enum {
+  DECODER_ENCODING,
+  DECODER_ENDIANNESS,
+  DECODER_BUFFERED,
+};
+
+static size_t
+textdecoder_try(const void* in, size_t len) {
+  const uint8_t *x, *y, *end;
+  size_t r = 0;
+  int ch;
+
+  x = in;
+  end = x + len;
+
+  while(x < end) {
+    y = x;
+
+    if((ch = unicode_from_utf8(x, end - x, &x)) == -1)
+      break;
+
+    r += x - y;
+  }
+
+  return r;
+}
+
+size_t
+textdecoder_length(TextDecoder* td) {
+  size_t len = 0, r;
+
+  r = textdecoder_try(ringbuffer_begin(&td->buffer), ringbuffer_continuous(&td->buffer));
+  len += r;
+
+  if(len == ringbuffer_continuous(&td->buffer))
+    if(td->buffer.head < td->buffer.tail) {
+      r = textdecoder_try(td->buffer.data, ringbuffer_head(&td->buffer));
+      len += r;
+    }
+
+  return len;
+}
+
+JSValue
+textdecoder_decode(TextDecoder* dec, JSContext* ctx) {
+  JSValue ret = JS_UNDEFINED;
+  DynBuf dbuf;
+  size_t i = 0, blen;
+  uint_least32_t cp;
+  char tmp[UTF8_CHAR_LEN_MAX];
+  int len = 0;
+  js_dbuf_init(ctx, &dbuf);
+  blen = ringbuffer_length(&dec->buffer);
+
+  if(blen)
+    switch(dec->encoding) {
+      case UTF8: {
+        size_t blen, rlen = ringbuffer_length(&dec->buffer);
+
+        if((blen = textdecoder_length(dec)) < rlen) {
+          ringbuffer_normalize(&dec->buffer);
+          blen = textdecoder_length(dec);
+        }
+
+        ret = JS_NewStringLen(ctx, (const char*)ringbuffer_begin(&dec->buffer), blen);
+        ringbuffer_skip(&dec->buffer, blen);
+        break;
+      }
+      case UTF16: {
+        uint_least16_t* ptr = ringbuffer_begin(&dec->buffer);
+        size_t n = blen & ~(0x1);
+
+        for(i = 0; i < n; ptr = ringbuffer_next(&dec->buffer, ptr), i += 2) {
+          uint_least16_t u16[2] = {uint16_get_endian(ptr, dec->endian), 0};
+          size_t ns = 2;
+
+          if(utf16_multiword(u16)) {
+            if(i + 2 >= n)
+              break;
+            u16[1] = uint16_get_endian(ptr + 1, dec->endian == BIG);
+            ns += 2;
+          }
+
+          if(!libutf_c16_to_c32(u16, &cp)) {
+            ret = JS_ThrowInternalError(ctx,
+                                        "%s: TextDecoder: not a valid utf-16 code at (%llu: 0x%04x, 0x%04x): %lu",
+                                        __func__,
+                                        (long long unsigned int)i,
+                                        (unsigned int)ptr[0],
+                                        (unsigned int)ptr[1],
+                                        (unsigned long)cp);
+            break;
+          }
+          len = unicode_to_utf8((void*)tmp, cp);
+
+          if(dbuf_put(&dbuf, (const void*)tmp, len))
+            return JS_EXCEPTION;
+        }
+
+        break;
+      }
+      case UTF32: {
+        const uint_least32_t* ptr = ringbuffer_begin(&dec->buffer);
+        size_t n = blen & ~(0x3);
+
+        for(i = 0; i < n; ptr = ringbuffer_next(&dec->buffer, ptr), i += 4) {
+          cp = uint32_get_endian(ptr, dec->endian);
+          if(!libutf_c32_to_c8(cp, &len, tmp)) {
+            ret = JS_ThrowInternalError(ctx,
+                                        "%s: TextDecoder: not a valid utf-32 code at (%llu: 0x%04x, 0x%04x): %lu",
+                                        __func__,
+                                        (long long unsigned int)i,
+                                        (unsigned int)ptr[0],
+                                        (unsigned int)ptr[1],
+                                        (unsigned long)cp);
+            break;
+          }
+          if(dbuf_put(&dbuf, (const void*)tmp, len))
+            return JS_EXCEPTION;
+        }
+        break;
+      }
+      default: {
+        TUTF8encoder encoder;
+
+        if((encoder = *tutf8e_coders[dec->encoding - 8])) {
+
+          const char* ptr = ringbuffer_begin(&dec->buffer);
+          size_t n = 0;
+
+          if(TUTF8E_OK == tutf8e_encoder_buffer_length(encoder, ptr, 0, blen, &n)) {
+
+            uint8_t* dst = dbuf_reserve(&dbuf, n);
+
+            if(TUTF8E_OK == tutf8e_encoder_buffer_encode(encoder, ptr, blen, 0, (char*)dst, &n)) {
+              dbuf.size += n;
+              i = blen;
+            }
+          }
+
+        } else {
+          ret = JS_ThrowInternalError(ctx, "%s: TextDecoder: unknown encoding: %s", __func__, textcode_encodings[dec->encoding]);
+        }
+        break;
+      }
+    }
+
+  ringbuffer_skip(&dec->buffer, i);
+
+  if(JS_IsUndefined(ret) && dbuf.size > 0)
+    ret = JS_NewStringLen(ctx, (const char*)dbuf.buf, dbuf.size);
+
+  dbuf_free(&dbuf);
+
+  return ret;
+}
+
+static JSValue
+js_decoder_get(JSContext* ctx, JSValueConst this_val, int magic) {
+  TextDecoder* dec;
+  JSValue ret = JS_UNDEFINED;
+  if(!(dec = js_decoder_data(ctx, this_val)))
+    return ret;
+  switch(magic) {
+    case DECODER_ENCODING: {
+      ret = JS_NewString(ctx, textcode_encodings[dec->type_code]);
+      break;
+    }
+
+    case DECODER_ENDIANNESS: {
+      ret = JS_NewBool(ctx, dec->endian == BIG);
+      break;
+    }
+
+    case DECODER_BUFFERED: {
+      ret = JS_NewUint32(ctx, ringbuffer_length(&dec->buffer));
+      break;
+    }
+  }
+  return ret;
+}
+
+static JSValue
+js_decoder_constructor(JSContext* ctx, JSValueConst new_target, int argc, JSValueConst argv[]) {
+  JSValue obj = JS_UNDEFINED;
+  JSValue proto;
+  TextDecoder* dec;
+
+  if(!(dec = js_mallocz(ctx, sizeof(TextDecoder))))
+    return JS_EXCEPTION;
+
+  /* using new_target to get the prototype is necessary when the class is extended. */
+  proto = JS_GetPropertyStr(ctx, new_target, "prototype");
+  if(JS_IsException(proto))
+    goto fail;
+  if(!JS_IsObject(proto))
+    proto = textdecoder_proto;
+
+  /* using new_target to get the prototype is necessary when the class is extended. */
+  obj = JS_NewObjectProtoClass(ctx, proto, js_decoder_class_id);
+  JS_FreeValue(ctx, proto);
+  if(JS_IsException(obj))
+    goto fail;
+
+  ringbuffer_init(&dec->buffer, ctx);
+
+  if(argc >= 1) {
+    const char* s = JS_ToCString(ctx, argv[0]);
+
+    if(s[case_finds(s, "utf32")] || s[case_finds(s, "utf-32")])
+      dec->encoding = UTF32;
+    else if(s[case_finds(s, "utf16")] || s[case_finds(s, "utf-16")])
+      dec->encoding = UTF16;
+    else if(s[case_finds(s, "utf8")] || s[case_finds(s, "utf-8")])
+      dec->encoding = UTF8;
+    else {
+      return JS_ThrowInternalError(ctx, "%s: TextDecoder: '%s' is invalid s", __func__, s);
+    }
+
+    if(s[case_finds(s, "be")] || s[case_finds(s, "be")])
+      dec->endian = BIG;
+
+    JS_FreeCString(ctx, s);
+  } else {
+    dec->encoding = UTF8;
+  }
+  JS_SetOpaque(obj, dec);
+
+  return obj;
+
+fail:
+  js_free(ctx, dec);
+  JS_FreeValue(ctx, obj);
+  return JS_EXCEPTION;
+}
+
+static JSValue
+js_decoder_decode(JSContext* ctx, JSValueConst this_val, int argc, JSValueConst argv[], int magic) {
+  TextDecoder* dec;
+  JSValue ret = JS_UNDEFINED;
+
+  if(!(dec = js_decoder_data(ctx, this_val)))
+    return JS_EXCEPTION;
+
+  switch(magic) {
+    case DECODER_END:
+    case DECODER_DECODE: {
+      InputBuffer in = js_input_chars(ctx, argv[0]);
+
+      // printf("js_decoder_decode (1) %s length=%zu in.size=%zu\n", magic == DECODER_DECODE ? "decode" :
+      // "end", ringbuffer_length(&dec->buffer), in.size);
+      if(ringbuffer_write(&dec->buffer, in.data, in.size) < 0)
+        return JS_ThrowInternalError(ctx, "%s: TextDecoder: ringbuffer %s failed", __func__, magic == DECODER_DECODE ? "decode" : "end");
+
+      if(ringbuffer_length(&dec->buffer) == 0)
+        ret = JS_NULL;
+      else
+        ret = textdecoder_decode(dec, ctx);
+
+      if(magic == DECODER_END)
+        ringbuffer_reset(&dec->buffer);
+      break;
+    }
+  }
+  return ret;
+}
+
+static JSValue
+js_decoder_inspect(JSContext* ctx, JSValueConst this_val, int argc, JSValueConst argv[]) {
+  TextDecoder* dec;
+
+  if(!(dec = js_decoder_data(ctx, this_val)))
+    return JS_EXCEPTION;
+
+  JSValue obj = JS_NewObjectClass(ctx, js_decoder_class_id);
+
+  JS_DefinePropertyValueStr(ctx, obj, "encoding", JS_NewString(ctx, textcode_encodings[dec->type_code]), JS_PROP_ENUMERABLE);
+  JS_DefinePropertyValueStr(ctx, obj, "buffered", JS_NewUint32(ctx, ringbuffer_length(&dec->buffer)), JS_PROP_ENUMERABLE);
+  return obj;
+}
+
+static void
+js_decoder_finalizer(JSRuntime* rt, JSValue val) {
+  TextDecoder* dec = JS_GetOpaque(val, js_decoder_class_id);
+  if(dec) {
+    ringbuffer_free(&dec->buffer);
+    js_free_rt(rt, dec);
+  }
+  // JS_FreeValueRT(rt, val);
+}
+
+static JSClassDef js_decoder_class = {
+    .class_name = "TextDecoder",
+    .finalizer = js_decoder_finalizer,
+};
+
+static const JSCFunctionListEntry js_decoder_funcs[] = {
+    JS_CFUNC_MAGIC_DEF("decode", 1, js_decoder_decode, DECODER_DECODE),
+    JS_CFUNC_MAGIC_DEF("end", 1, js_decoder_decode, DECODER_END),
+    JS_CGETSET_ENUMERABLE_DEF("encoding", js_decoder_get, 0, DECODER_ENCODING),
+    JS_CGETSET_MAGIC_DEF("endian", js_decoder_get, 0, DECODER_ENDIANNESS),
+    JS_CGETSET_MAGIC_DEF("buffered", js_decoder_get, 0, DECODER_BUFFERED),
+    JS_PROP_STRING_DEF("[Symbol.toStringTag]", "TextDecoder", JS_PROP_CONFIGURABLE),
+};
+
+enum {
+  ENCODER_ENCODE,
+  ENCODER_END,
+};
+enum {
+  ENCODER_ENCODING,
+  ENCODER_ENDIANNESS,
+  ENCODER_BUFFERED,
+};
+
+/*static size_t
+textencoder_try(const void* in, size_t len) {
+  const uint8_t *x, *y, *end;
+  size_t r = 0;
+  int ch;
+  for(x = in, end=x+len; x < end; ) {
+    y = x;
+    if((ch = unicode_from_utf8(x, end - x, &x)) == -1)
+      break;
+    r += x - y;
+  }
+  return r;
+}
+
+size_t
+textencoder_length(TextEncoder* td) {
+  size_t r = 0;
+
+  r += textencoder_try(ringbuffer_begin(&td->buffer), ringbuffer_continuous(&td->buffer));
+
+  if(td->buffer.head < td->buffer.tail)
+    r += textencoder_try(td->buffer.data, ringbuffer_head(&td->buffer));
+
+  return r;
+}*/
+
+JSValue
+textencoder_read(TextEncoder* te, JSContext* ctx) {
+  JSValue ret, buf;
+  int bits;
+  size_t len = ringbuffer_length(&te->buffer);
+
+  if(len > ringbuffer_continuous(&te->buffer))
+    ringbuffer_normalize(&te->buffer);
+
+  switch(te->encoding) {
+    case UTF8: bits = 8; break;
+    case UTF16: bits = 16; break;
+    case UTF32: bits = 32; break;
+    default: return JS_ThrowInternalError(ctx, "%s: TextEncoder: invalid encoding: %d", __func__, te->encoding);
+  }
+
+  buf = JS_NewArrayBufferCopy(ctx, ringbuffer_begin(&te->buffer), len);
+  ret = js_typedarray_new(ctx, bits, FALSE, FALSE, buf);
+  JS_FreeValue(ctx, buf);
+
+  te->buffer.tail += len;
+  return ret;
+}
+
+JSValue
+textencoder_encode(TextEncoder* enc, InputBuffer in, JSContext* ctx) {
+  JSValue ret = JS_UNDEFINED;
+  size_t i;
+  uint32_t cp;
+  uint8_t u8[UTF8_CHAR_LEN_MAX];
+  const uint8_t *ptr, *end, *next;
+
+  switch(enc->encoding) {
+    case UTF8: {
+      if(ringbuffer_write(&enc->buffer, in.data, in.size) < 0)
+        return JS_ThrowInternalError(ctx, "%s: TextEncoder: ringbuffer write failed", __func__);
+      break;
+    }
+
+    case UTF16: {
+      ptr = block_begin(&in.block);
+      end = block_end(&in.block);
+
+      for(i = 0; ptr < end; ptr = next, i++) {
+        cp = unicode_from_utf8(ptr, end - ptr, &next);
+
+        {
+          uint_least16_t u16[2];
+          int len;
+
+          if(!libutf_c32_to_c16(cp, &len, u16))
+            return JS_ThrowInternalError(ctx,
+                                         "%s: TextEncoder: not a valid code point at (%llu) [%llu]: %lu",
+                                         __func__,
+                                         (long long unsigned int)i,
+                                         (long long unsigned int)(end - ptr),
+                                         (unsigned long)cp);
+
+          for(int j = 0; j < len; j++)
+            uint16_put_endian(u8 + j * 2, u16[j], enc->endian);
+
+          if(ringbuffer_append(&enc->buffer, u8, len * sizeof(uint16_t), ctx) < 0)
+            return JS_ThrowInternalError(ctx, "TextEncoder: ringbuffer write failed");
+        }
+      }
+
+      break;
+    }
+
+    case UTF32: {
+      ptr = block_begin(&in.block);
+      end = block_end(&in.block);
+
+      for(i = 0; ptr < end; ptr = next, i++) {
+        cp = unicode_from_utf8(ptr, end - ptr, &next);
+        if(cp == 0xffffffff)
+          return JS_ThrowInternalError(
+              ctx, "%s: TextEncoder: not a valid code point at (%llu): %lu", __func__, (long long unsigned int)(ptr - in.block.base), (long unsigned int)cp);
+        /*cp = 0;
+        if(!libutf_c8_to_c32(ptr, &cp))
+          return JS_ThrowInternalError(ctx, "No a valid code point at (%zu) [%zu]: %" PRIu32, i, end - ptr,
+        cp); next = ptr + libutf_c8_type(ptr); if(next == ptr) break;*/
+
+        uint32_put_endian(u8, cp, enc->endian);
+
+        if(ringbuffer_append(&enc->buffer, u8, sizeof(cp), ctx) < 0)
+          return JS_ThrowInternalError(ctx, "%s: TextEncoder: ringbuffer write failed", __func__);
+      }
+      break;
+    }
+    default: {
+      ret = JS_ThrowInternalError(ctx, "%s: TextEncoder: unknown encoding", __func__);
+      break;
+    }
+  }
+
+  return ret;
+}
+
+static JSValue
+js_encoder_get(JSContext* ctx, JSValueConst this_val, int magic) {
+  TextEncoder* enc;
+  JSValue ret = JS_UNDEFINED;
+  if(!(enc = js_encoder_data(ctx, this_val)))
+    return ret;
+  switch(magic) {
+    case ENCODER_ENCODING: {
+      ret = JS_NewString(ctx, textcode_encodings[enc->type_code]);
+      break;
+    }
+
+    case ENCODER_ENDIANNESS: {
+      ret = JS_NewBool(ctx, enc->endian == BIG);
+      break;
+    }
+
+    case ENCODER_BUFFERED: {
+      ret = JS_NewUint32(ctx, ringbuffer_length(&enc->buffer));
+      break;
+    }
+  }
+  return ret;
+}
+
+static JSValue
+js_encoder_constructor(JSContext* ctx, JSValueConst new_target, int argc, JSValueConst argv[]) {
+  JSValue obj = JS_UNDEFINED;
+  JSValue proto;
+  TextEncoder* enc;
+
+  if(!(enc = js_mallocz(ctx, sizeof(TextEncoder))))
+    return JS_EXCEPTION;
+
+  /* using new_target to get the prototype is necessary when the class is extended. */
+  proto = JS_GetPropertyStr(ctx, new_target, "prototype");
+  if(JS_IsException(proto))
+    goto fail;
+  if(!JS_IsObject(proto))
+    proto = textencoder_proto;
+
+  /* using new_target to get the prototype is necessary when the class is extended. */
+  obj = JS_NewObjectProtoClass(ctx, proto, js_encoder_class_id);
+  JS_FreeValue(ctx, proto);
+  if(JS_IsException(obj))
+    goto fail;
+
+  ringbuffer_init(&enc->buffer, ctx);
+
+  if(argc >= 1) {
+    const char* s = JS_ToCString(ctx, argv[0]);
+
+    if(s[case_finds(s, "utf32")] || s[case_finds(s, "utf-32")])
+      enc->encoding = UTF32;
+    else if(s[case_finds(s, "utf16")] || s[case_finds(s, "utf-16")])
+      enc->encoding = UTF16;
+    else if(s[case_finds(s, "utf8")] || s[case_finds(s, "utf-8")])
+      enc->encoding = UTF8;
+    else {
+      return JS_ThrowInternalError(ctx, "TextEncoder '%s' is invalid s", s);
+    }
+
+    if(enc->encoding > UTF8)
+      if(s[case_finds(s, "be")] || s[case_finds(s, "be")])
+        enc->endian = BIG;
+
+    JS_FreeCString(ctx, s);
+  } else {
+    enc->encoding = UTF8;
+  }
+  JS_SetOpaque(obj, enc);
+
+  return obj;
+
+fail:
+  js_free(ctx, enc);
+  JS_FreeValue(ctx, obj);
+  return JS_EXCEPTION;
+}
+
+static JSValue
+js_encoder_encode(JSContext* ctx, JSValueConst this_val, int argc, JSValueConst argv[], int magic) {
+  TextEncoder* enc;
+  JSValue ret = JS_UNDEFINED;
+
+  if(!(enc = js_encoder_data(ctx, this_val)))
+    return JS_EXCEPTION;
+
+  switch(magic) {
+    case ENCODER_END:
+    case ENCODER_ENCODE: {
+      InputBuffer in = js_input_chars(ctx, argv[0]);
+
+      ret = textencoder_encode(enc, in, ctx);
+
+      if(JS_IsException(ret))
+        break;
+
+      if(ringbuffer_length(&enc->buffer) == 0)
+        ret = JS_NULL;
+      else
+        ret = textencoder_read(enc, ctx);
+
+      if(magic == ENCODER_END)
+        ringbuffer_reset(&enc->buffer);
+      break;
+    }
+  }
+  return ret;
+}
+
+static JSValue
+js_encoder_inspect(JSContext* ctx, JSValueConst this_val, int argc, JSValueConst argv[]) {
+  TextEncoder* enc;
+
+  if(!(enc = js_encoder_data(ctx, this_val)))
+    return JS_EXCEPTION;
+
+  JSValue obj = JS_NewObjectClass(ctx, js_encoder_class_id);
+
+  JS_DefinePropertyValueStr(ctx, obj, "encoding", JS_NewString(ctx, textcode_encodings[enc->type_code]), JS_PROP_ENUMERABLE);
+  JS_DefinePropertyValueStr(ctx, obj, "buffered", JS_NewUint32(ctx, ringbuffer_length(&enc->buffer)), JS_PROP_ENUMERABLE);
+  return obj;
+}
+
+static void
+js_encoder_finalizer(JSRuntime* rt, JSValue val) {
+  TextEncoder* enc = JS_GetOpaque(val, js_encoder_class_id);
+  if(enc) {
+    ringbuffer_free(&enc->buffer);
+    js_free_rt(rt, enc);
+  }
+  // JS_FreeValueRT(rt, val);
+}
+
+static JSClassDef js_encoder_class = {
+    .class_name = "TextEncoder",
+    .finalizer = js_encoder_finalizer,
+};
+
+static const JSCFunctionListEntry js_encoder_funcs[] = {
+    JS_CFUNC_MAGIC_DEF("encode", 1, js_encoder_encode, ENCODER_ENCODE),
+    JS_CFUNC_MAGIC_DEF("end", 1, js_encoder_encode, ENCODER_END),
+    JS_CGETSET_ENUMERABLE_DEF("encoding", js_encoder_get, 0, ENCODER_ENCODING),
+    JS_CGETSET_MAGIC_DEF("endian", js_encoder_get, 0, ENCODER_ENDIANNESS),
+    JS_CGETSET_MAGIC_DEF("buffered", js_encoder_get, 0, ENCODER_BUFFERED),
+    JS_PROP_STRING_DEF("[Symbol.toStringTag]", "TextEncoder", JS_PROP_CONFIGURABLE),
+};
+
+extern
+void JS_NewGlobalCConstructor2(JSContext *ctx,
+                                      JSValue func_obj,
+                                      const char *name,
+                                      JSValueConst proto);
+int
+js_code_init_textdecoder(JSContext* ctx, JSModuleDef* m)
+{
+    // RH TODO: check if this (possibly being called twice or for some other reason) produces a mem leak
+    JS_NewClassID(&js_encoder_class_id);
+    JS_NewClass(JS_GetRuntime(ctx), js_encoder_class_id, &js_encoder_class);
+
+    textencoder_ctor = JS_NewCFunction2(ctx, js_encoder_constructor, "TextEncoder", 1, JS_CFUNC_constructor, 0);
+    textencoder_proto = JS_NewObject(ctx);
+
+    JS_SetPropertyFunctionList(ctx, textencoder_proto, js_encoder_funcs, countof(js_encoder_funcs));
+    JS_SetClassProto(ctx, js_encoder_class_id, textencoder_proto);
+    JS_SetConstructor(ctx, textencoder_ctor, textencoder_proto);
+
+    JS_NewGlobalCConstructor2(ctx, textencoder_ctor, "TextEncoder", textencoder_proto);
+
+
+    JS_NewClassID(&js_decoder_class_id);
+    JS_NewClass(JS_GetRuntime(ctx), js_decoder_class_id, &js_decoder_class);
+
+    textdecoder_ctor = JS_NewCFunction2(ctx, js_decoder_constructor, "TextDecoder", 1, JS_CFUNC_constructor, 0);
+    textdecoder_proto = JS_NewObject(ctx);
+
+    JS_SetPropertyFunctionList(ctx, textdecoder_proto, js_decoder_funcs, countof(js_decoder_funcs));
+    JS_SetClassProto(ctx, js_decoder_class_id, textdecoder_proto);
+
+    JS_SetConstructor(ctx, textdecoder_ctor, textdecoder_proto);
+    
+    JS_NewGlobalCConstructor2(ctx, textdecoder_ctor, "TextDecoder", textdecoder_proto);
+
+  return 0;
+}
+
+#ifdef JS_TEXTCODE_MODULE
+#define JS_INIT_MODULE js_init_module
+#else
+#define JS_INIT_MODULE js_init_module_textdecoder
+#endif
+
+VISIBLE JSModuleDef*
+JS_INIT_MODULE(JSContext* ctx, const char* module_name) {
+  JSModuleDef* m;
+
+  /*if((m = JS_NewCModule(ctx, module_name, js_code_init)))
+  {
+    JS_AddModuleExport(ctx, m, "TextDecoder");
+    JS_AddModuleExport(ctx, m, "TextEncoder");
+    //if(!strcmp(module_name, "textdecoder"))
+    //  JS_AddModuleExport(ctx, m, "default");
+  }*/
+
+  return m;
+}
+
+/**
+ * @}
+ */

src/quickjs/quickjs-textcode.h

@@ -0,0 +1,86 @@
+#ifndef QUICKJS_TEXTCODE_H
+#define QUICKJS_TEXTCODE_H
+
+#include "utils.h"
+#include "ringbuffer.h"
+
+/**
+ * \defgroup quickjs-textdecoder quickjs-textdecoder: String Decoder
+ * @{
+ */
+
+typedef enum utf_encoding { UNKNOWN = 0, UTF8, UTF16, UTF32 } UTFCharset;
+typedef enum text_encoding {
+  UTF16LE = 2,
+  UTF32LE = 3,
+  UTF16BE = 6,
+  UTF32BE = 7,
+  ISO_8859_1,
+  ISO_8859_2,
+  ISO_8859_3,
+  ISO_8859_4,
+  ISO_8859_5,
+  ISO_8859_6,
+  ISO_8859_7,
+  ISO_8859_8,
+  ISO_8859_9,
+  ISO_8859_10,
+  ISO_8859_11,
+  ISO_8859_13,
+  ISO_8859_14,
+  ISO_8859_15,
+  ISO_8859_16,
+  WINDOWS_1250,
+  WINDOWS_1251,
+  WINDOWS_1252,
+  WINDOWS_1253,
+  WINDOWS_1254,
+  WINDOWS_1255,
+  WINDOWS_1256,
+  WINDOWS_1257,
+  WINDOWS_1258,
+} TextEncoding;
+
+#define TextcodeType \
+  PACK union { \
+    struct { \
+      UTFCharset encoding : 2; \
+      Endian endian : 1; \
+    }; \
+    TextEncoding type_code; \
+  }; \
+  ENDPACK
+
+struct text_coder {
+  RingBuffer buffer;
+  TextcodeType
+};
+
+typedef struct text_coder TextEncoder;
+typedef struct text_coder TextDecoder;
+
+extern VISIBLE JSClassID js_decoder_class_id, js_encoder_class_id;
+extern VISIBLE JSValue textdecoder_proto, textdecoder_ctor, textencoder_proto, textencoder_ctor;
+extern const char* const textcode_encodings[];
+
+size_t textdecoder_length(TextDecoder*);
+JSValue textdecoder_read(TextDecoder*, JSContext* ctx);
+int js_code_init_textdecoder(JSContext*, JSModuleDef* m);
+size_t textencoder_length(TextEncoder*);
+JSValue textencoder_read(TextEncoder*, JSContext* ctx);
+int js_encoder_init(JSContext*, JSModuleDef* m);
+
+static inline TextDecoder*
+js_decoder_data(JSContext* ctx, JSValueConst value) {
+  return JS_GetOpaque(value, js_decoder_class_id);
+}
+
+static inline TextEncoder*
+js_encoder_data(JSContext* ctx, JSValueConst value) {
+  return JS_GetOpaque(value, js_encoder_class_id);
+}
+
+/**
+ * @}
+ */
+#endif /* defined(QUICKJS_TEXTCODE_H) */

src/quickjs/ringbuffer.c

@@ -0,0 +1,136 @@
+#include "ringbuffer.h"
+
+/**
+ * \addtogroup ringbuffer
+ * @{
+ */
+void
+ringbuffer_reset(RingBuffer* r) {
+  r->head = r->tail = 0;
+}
+
+void
+ringbuffer_queue(RingBuffer* r, uint8_t data) {
+  /* overwrite the oldest byte if the r is full */
+  if(ringbuffer_full(r)) {
+    r->tail = ((r->tail + 1) % r->size);
+  }
+
+  r->data[r->head] = data;
+  r->head = ((r->head + 1) % r->size);
+}
+
+BOOL
+ringbuffer_dequeue(RingBuffer* r, uint8_t* data) {
+  if(ringbuffer_empty(r))
+    return FALSE;
+
+  *data = r->data[r->tail];
+  r->tail = ((r->tail + 1) % r->size);
+  return TRUE;
+}
+
+ssize_t
+ringbuffer_write(RingBuffer* r, const void* x, size_t len) {
+  const uint8_t* p = x;
+  size_t i;
+
+  if(ringbuffer_avail(r) < len)
+    return -1;
+  //    ringbuffer_realloc(r, ringbuffer_length(r) + len);
+
+  for(i = 0; i < len; i++) {
+    assert(!ringbuffer_full(r));
+    ringbuffer_queue(r, p[i]);
+  }
+
+  return i;
+}
+
+ssize_t
+ringbuffer_append(RingBuffer* r, const void* x, size_t len, JSContext* ctx) {
+
+  if(!ringbuffer_reserve(r, ringbuffer_length(r) + len))
+    return -1;
+
+  return ringbuffer_write(r, x, len);
+}
+
+ssize_t
+ringbuffer_read(RingBuffer* r, void* x, size_t len) {
+  uint8_t* p = x;
+  size_t i;
+
+  if(ringbuffer_empty(r))
+    return -1;
+
+  for(i = 0; i < len; i++)
+    ringbuffer_dequeue(r, &p[i]);
+
+  return i;
+}
+
+uint8_t*
+ringbuffer_peek(RingBuffer* r, size_t index) {
+  if(index >= ringbuffer_length(r))
+    return 0;
+
+  return &r->data[(r->tail + index) % r->size];
+}
+
+void
+ringbuffer_normalize(RingBuffer* r) {
+  if(r->head < r->tail) {
+    size_t n = r->size - r->tail;
+    void* x = alloca(r->head);
+    memcpy(x, r->data, r->head);
+    memmove(r->data, &r->data[r->tail], n);
+    memcpy(&r->data[n], x, r->head);
+    r->tail = 0;
+    r->head += n;
+    return;
+  }
+
+  memcpy(r->data, &r->data[r->tail], ringbuffer_length(r));
+  r->head -= r->tail;
+  r->tail = 0;
+}
+
+BOOL
+ringbuffer_resize(RingBuffer* r, size_t newsize) {
+  ringbuffer_normalize(r);
+  if(newsize > r->size)
+    return vector_grow(&r->vec, 1, newsize);
+  else if(newsize < r->size)
+    return vector_shrink(&r->vec, 1, newsize);
+  return TRUE;
+}
+
+BOOL
+ringbuffer_allocate(RingBuffer* r, size_t size) {
+  ssize_t n = ringbuffer_length(r);
+
+  if((r->size - n) < size)
+    return ringbuffer_resize(r, ringbuffer_length(r) + size);
+
+  return TRUE;
+}
+
+uint8_t*
+ringbuffer_reserve(RingBuffer* rb, size_t min_bytes) {
+  ssize_t grow;
+  if((grow = min_bytes - ringbuffer_avail(rb)) > 0)
+    if(!ringbuffer_resize(rb, vector_size(&rb->vec, 1) + grow))
+      return 0;
+
+  if(ringbuffer_headroom(rb) < min_bytes)
+    ringbuffer_normalize(rb);
+
+  assert(ringbuffer_headroom(rb) >= min_bytes);
+
+  return ringbuffer_end(rb);
+}
+
+/**
+ * @}
+ */

src/quickjs/ringbuffer.h

@@ -0,0 +1,88 @@
+#ifndef RINGBUFFER_H
+#define RINGBUFFER_H
+
+#include "vector.h"
+#include "debug.h"
+
+/**
+ * \defgroup ringbuffer ringbuffer: Ring Buffer implementation
+ * @{
+ */
+typedef union ringbuffer {
+  struct {
+    uint8_t* data;
+    size_t size, capacity;
+    BOOL error;
+    DynBufReallocFunc* realloc_func;
+    void* opaque;
+    volatile uint32_t tail, head;
+  };
+  DynBuf dbuf;
+  Vector vec;
+
+} RingBuffer;
+
+#define RINGBUFFER_INIT() \
+  { \
+    { 0, 0, 0, 0, &ringbuffer_default_realloc, 0 } \
+  }
+
+#define ringbuffer_init(rb, ctx) \
+  do { \
+    vector_init(&(rb)->vec, ctx); \
+    vector_allocate(&(rb)->vec, 1, 1023); \
+  } while(0)
+#define ringbuffer_init_rt(rb, rt) \
+  do { \
+    vector_init_rt(&(rb)->vec, rt); \
+    vector_allocate(&(rb)->vec, 1, 1023); \
+  } while(0)
+#define RINGBUFFER(ctx) \
+  (RingBuffer) { \
+    { 0, 0, 0, 0, (DynBufReallocFunc*)&js_realloc, ctx, 0, 0 } \
+  }
+#define RINGBUFFER_RT(rt) \
+  (RingBuffer) { \
+    { 0, 0, 0, 0, (DynBufReallocFunc*)&js_realloc_rt, rt } \
+  }
+#define ringbuffer_free(rb) vector_free(&(rb)->vec)
+#define ringbuffer_begin(rb) (void*)&ringbuffer_tail(rb)
+#define ringbuffer_end(rb) (void*)&ringbuffer_head(rb)
+#define ringbuffer_head(rb) (rb)->data[(rb)->head]
+#define ringbuffer_tail(rb) (rb)->data[(rb)->tail]
+
+#define ringbuffer_empty(rb) ((rb)->tail == (rb)->head)
+#define ringbuffer_full(rb) ((rb)->size == (rb)->head - (rb)->tail)
+#define ringbuffer_wrapped(rb) ((rb)->head < (rb)->tail)
+#define ringbuffer_headroom(rb) ((rb)->size - (rb)->head)
+#define ringbuffer_avail(rb) ((rb)->size - ringbuffer_length(rb))
+#define ringbuffer_length(rb) (ringbuffer_wrapped(rb) ? ((rb)->size - (rb)->tail) + (rb)->head : (rb)->head - (rb)->tail)
+#define ringbuffer_continuous(rb) (ringbuffer_wrapped(rb) ? (rb)->size - (rb)->tail : (rb)->head - (rb)->tail)
+#define ringbuffer_is_continuous(rb) ((rb)->head >= (rb)->tail)
+//#define ringbuffer_skip(rb, n) ((rb)->tail += (n), (rb)->tail %= (rb)->size)
+#define ringbuffer_wrap(rb, idx) ((idx) % (rb)->size)
+#define ringbuffer_next(rb, ptr) (void*)(ringbuffer_wrap(rb, ((uint8_t*)(ptr + 1)) - (rb)->data) + (rb)->data)
+
+void ringbuffer_reset(RingBuffer*);
+void ringbuffer_queue(RingBuffer*, uint8_t data);
+BOOL ringbuffer_dequeue(RingBuffer*, uint8_t* data);
+ssize_t ringbuffer_write(RingBuffer*, const void* x, size_t len);
+ssize_t ringbuffer_read(RingBuffer*, void* x, size_t len);
+uint8_t* ringbuffer_peek(RingBuffer*, size_t index);
+void ringbuffer_normalize(RingBuffer*);
+BOOL ringbuffer_resize(RingBuffer*, size_t);
+BOOL ringbuffer_allocate(RingBuffer*, size_t);
+uint8_t* ringbuffer_reserve(RingBuffer* rb, size_t min_bytes);
+ssize_t ringbuffer_append(RingBuffer* r, const void* x, size_t len, JSContext* ctx);
+
+static inline uint8_t*
+ringbuffer_skip(RingBuffer* rb, size_t n) {
+  assert(ringbuffer_length(rb) >= n);
+  rb->tail = (rb->tail + n) % rb->size;
+  return ringbuffer_begin(rb);
+}
+
+/**
+ * @}
+ */
+#endif /* defined(RINGBUFFER_H) */

src/quickjs/test_fib.c

@@ -0,0 +1,69 @@
+/* File generated automatically by the QuickJS compiler. */
+
+#include "quickjs-libc.h"
+
+const uint32_t qjsc_test_fib_size = 160;
+
+const uint8_t qjsc_test_fib[160] = {
+ 0x43, 0x07, 0x28, 0x65, 0x78, 0x61, 0x6d, 0x70,
+ 0x6c, 0x65, 0x73, 0x2f, 0x74, 0x65, 0x73, 0x74,
+ 0x5f, 0x66, 0x69, 0x62, 0x2e, 0x6a, 0x73, 0x10,
+ 0x2e, 0x2f, 0x66, 0x69, 0x62, 0x2e, 0x73, 0x6f,
+ 0x06, 0x66, 0x69, 0x62, 0x0e, 0x63, 0x6f, 0x6e,
+ 0x73, 0x6f, 0x6c, 0x65, 0x06, 0x6c, 0x6f, 0x67,
+ 0x16, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57,
+ 0x6f, 0x72, 0x6c, 0x64, 0x10, 0x66, 0x69, 0x62,
+ 0x28, 0x31, 0x30, 0x29, 0x3d, 0x0d, 0xc6, 0x03,
+ 0x01, 0xc8, 0x03, 0x00, 0x00, 0x01, 0x00, 0xca,
+ 0x03, 0x00, 0x00, 0x0c, 0x20, 0x06, 0x01, 0xa2,
+ 0x01, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x32,
+ 0x00, 0xca, 0x03, 0x00, 0x0c, 0x08, 0xec, 0x02,
+ 0x29, 0x38, 0xe6, 0x00, 0x00, 0x00, 0x42, 0xe7,
+ 0x00, 0x00, 0x00, 0x04, 0xe8, 0x00, 0x00, 0x00,
+ 0x24, 0x01, 0x00, 0x0e, 0x38, 0xe6, 0x00, 0x00,
+ 0x00, 0x42, 0xe7, 0x00, 0x00, 0x00, 0x04, 0xe9,
+ 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0xbf, 0x0a,
+ 0xf1, 0x24, 0x02, 0x00, 0x0e, 0x06, 0x2e, 0xc6,
+ 0x03, 0x01, 0x05, 0x01, 0x00, 0x04, 0x0a, 0x62,
+};
+
+static JSContext *JS_NewCustomContext(JSRuntime *rt)
+{
+  JSContext *ctx = JS_NewContextRaw(rt);
+  if (!ctx)
+    return NULL;
+  JS_AddIntrinsicBaseObjects(ctx);
+  JS_AddIntrinsicDate(ctx);
+  JS_AddIntrinsicEval(ctx);
+  JS_AddIntrinsicStringNormalize(ctx);
+  JS_AddIntrinsicRegExp(ctx);
+  JS_AddIntrinsicJSON(ctx);
+  JS_AddIntrinsicProxy(ctx);
+  JS_AddIntrinsicMapSet(ctx);
+  JS_AddIntrinsicTypedArrays(ctx);
+  JS_AddIntrinsicPromise(ctx);
+  JS_AddIntrinsicBigInt(ctx);
+  {
+    extern JSModuleDef *js_init_module_fib(JSContext *ctx, const char *name);
+    js_init_module_fib(ctx, "examples/fib.so");
+  }
+  return ctx;
+}
+
+int main(int argc, char **argv)
+{
+  JSRuntime *rt;
+  JSContext *ctx;
+  rt = JS_NewRuntime();
+  js_std_set_worker_new_context_func(JS_NewCustomContext);
+  js_std_init_handlers(rt);
+  JS_SetModuleLoaderFunc(rt, NULL, js_module_loader, NULL);
+  ctx = JS_NewCustomContext(rt);
+  js_std_add_helpers(ctx, argc, argv);
+  js_std_eval_binary(ctx, qjsc_test_fib, qjsc_test_fib_size, 0);
+  js_std_loop(ctx);
+  js_std_free_handlers(rt);
+  JS_FreeContext(ctx);
+  JS_FreeRuntime(rt);
+  return 0;
+}

src/quickjs/tutf8e.c

@@ -0,0 +1,676 @@
+
+#include <tutf8e.h>
+
+#include <string.h>
+
+int tutf8e_string_length(const uint16_t *table, const char *input, const char *invalid, size_t *input_length, size_t *output_length)
+{
+  const size_t invalid_length = invalid ? strlen(invalid) : 0;
+
+  const unsigned char *i;
+  for (i = (const unsigned char *) input; *i; ++i, (*input_length)++) {
+    const uint16_t c = table[*i];
+    if (c<0x80) {
+      *output_length += 1;
+      continue;
+    }
+    if (c<0x800) {
+      *output_length += 2;
+      continue;
+    }
+    if (c<0xffff) {
+      *output_length += 3;
+      continue;
+    }
+    if (invalid) {
+      *output_length += invalid_length;
+    }
+    else {
+      return TUTF8E_INVALID;
+    }
+  }
+  return TUTF8E_OK;
+}
+
+int tutf8e_string_encode(const uint16_t *table, const char *input, const char *invalid, char *output, size_t *output_length)
+{
+  int ret;
+  size_t input_length = 0;
+  size_t encoded_length = 0;
+  if (!(ret = tutf8e_string_length(table, input, invalid, &input_length, &encoded_length)))
+  {
+    if (encoded_length+1 > *output_length) return TUTF8E_TOOLONG;
+    if (!(ret = tutf8e_buffer_encode(table, input, input_length, invalid, output, output_length)))
+    {
+      output[encoded_length] = 0;
+      return TUTF8E_OK;
+    }
+  }
+  return ret;
+}
+
+int tutf8e_buffer_length
+(
+  const uint16_t *table,
+  const char *input,
+  size_t input_length,
+  const char *invalid,
+  size_t *length
+)
+{
+  const size_t invalid_length = invalid ? strlen(invalid) : 0;
+
+  const unsigned char *i;
+  for (i = (const unsigned char *) input; input_length; ++i, --input_length) {
+    const uint16_t c = table[*i];
+    if (c<0x80) {
+      ++*length;
+      continue;
+    }
+    if (c<0x800) {
+      *length += 2;
+      continue;
+    }
+    if (c<0xffff) {
+      *length += 3;
+      continue;
+    }
+    if (invalid) {
+      *length += invalid_length;
+    }
+    else {
+      return TUTF8E_INVALID;
+    }
+  }
+  return TUTF8E_OK;
+}
+
+int tutf8e_buffer_encode
+(
+  const uint16_t *table,
+  const char *input,
+  size_t input_length,
+  const char *invalid,
+  char *output,
+  size_t *output_length
+)
+{
+  size_t invalid_length = invalid ? strlen(invalid) : 0;
+
+  size_t left = *output_length;
+  unsigned char *o = (unsigned char *) output;
+  const unsigned char *i;
+  for (i = (const unsigned char *) input; input_length; ++i, --input_length) {
+    const uint16_t c = table[*i];
+    if (c<0x80) {
+      if (left<1) return TUTF8E_TOOLONG;
+      *(o++) = c;
+      left -= 1;
+      continue;
+    }
+    if (c<0x800) {
+      if (left<2) return TUTF8E_TOOLONG;
+      *(o++) = 0xc0 | (c>>6);
+      *(o++) = 0x80 | (c&0x3f);
+      left -= 2;
+      continue;
+    }
+    if (c<0xffff) {
+      if (left<3) return TUTF8E_TOOLONG;
+      *(o++) = 0xe0 | (c>>12);
+      *(o++) = 0x80 | ((c>>6)&0x3f);
+      *(o++) = 0x80 | (c&0x3f);
+      left -= 3;
+      continue;
+    }
+    if (invalid)
+    {
+      if (left<invalid_length) return TUTF8E_TOOLONG;
+      if (invalid_length) {
+        memcpy(o, invalid, invalid_length);
+        o += invalid_length;
+        left -= invalid_length;
+      }
+    }
+    else {
+      return TUTF8E_INVALID;
+    }
+  }
+  *output_length -= left;
+  return TUTF8E_OK;
+}
+
+const uint16_t tutf8e_iso_8859_1_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff,
+};
+
+const uint16_t tutf8e_iso_8859_10_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x0128, 0x0136, 0x00a7, 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a,
+  0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b,
+  0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf,
+  0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+  0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef,
+  0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138,
+};
+
+const uint16_t tutf8e_iso_8859_11_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f,
+  0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f,
+  0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f,
+  0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e35, 0x0e36, 0x0e37, 0x0e38, 0x0e39, 0x0e3a, 0xffff, 0xffff, 0xffff, 0xffff, 0x0e3f,
+  0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f,
+  0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0xffff, 0xffff, 0xffff, 0xffff,
+};
+
+const uint16_t tutf8e_iso_8859_13_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6,
+  0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b,
+  0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df,
+  0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c,
+  0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019,
+};
+
+const uint16_t tutf8e_iso_8859_14_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178,
+  0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff,
+};
+
+const uint16_t tutf8e_iso_8859_15_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff,
+};
+
+const uint16_t tutf8e_iso_8859_16_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b,
+  0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c,
+  0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff,
+};
+
+const uint16_t tutf8e_iso_8859_2_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b,
+  0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c,
+  0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
+  0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
+  0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
+  0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9,
+};
+
+const uint16_t tutf8e_iso_8859_3_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0xffff, 0x0124, 0x00a7, 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0xffff, 0x017b,
+  0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0xffff, 0x017c,
+  0x00c0, 0x00c1, 0x00c2, 0xffff, 0x00c4, 0x010a, 0x0108, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0xffff, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0xffff, 0x00e4, 0x010b, 0x0109, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0xffff, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9,
+};
+
+const uint16_t tutf8e_iso_8859_4_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af,
+  0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b,
+  0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a,
+  0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df,
+  0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b,
+  0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9,
+};
+
+const uint16_t tutf8e_iso_8859_5_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f,
+  0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
+  0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
+  0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
+  0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f,
+  0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f,
+};
+
+const uint16_t tutf8e_iso_8859_6_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0xffff, 0xffff, 0xffff, 0x00a4, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x060c, 0x00ad, 0xffff, 0xffff,
+  0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x061b, 0xffff, 0xffff, 0xffff, 0x061f,
+  0xffff, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
+  0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063a, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+  0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f,
+  0x0650, 0x0651, 0x0652, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+};
+
+const uint16_t tutf8e_iso_8859_7_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0xffff, 0x2015,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f,
+  0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f,
+  0x03a0, 0x03a1, 0xffff, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af,
+  0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf,
+  0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xffff,
+};
+
+const uint16_t tutf8e_iso_8859_8_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0xffff, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0xffff,
+  0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+  0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x2017,
+  0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df,
+  0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0xffff, 0xffff, 0x200e, 0x200f, 0xffff,
+};
+
+const uint16_t tutf8e_iso_8859_9_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
+  0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff,
+};
+
+const uint16_t tutf8e_windows_1250_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0xffff, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a,
+  0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b,
+  0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c,
+  0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
+  0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
+  0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
+  0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9,
+};
+
+const uint16_t tutf8e_windows_1251_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f,
+  0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f,
+  0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407,
+  0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457,
+  0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
+  0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
+  0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
+  0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f,
+};
+
+const uint16_t tutf8e_windows_1252_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xffff, 0x017d, 0xffff,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xffff, 0x017e, 0x0178,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff,
+};
+
+const uint16_t tutf8e_windows_1253_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0xffff, 0x2039, 0xffff, 0xffff, 0xffff, 0xffff,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0xffff, 0x203a, 0xffff, 0xffff, 0xffff, 0xffff,
+  0x00a0, 0x0385, 0x0386, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0xffff, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x2015,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x00b5, 0x00b6, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f,
+  0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f,
+  0x03a0, 0x03a1, 0xffff, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af,
+  0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf,
+  0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xffff,
+};
+
+const uint16_t tutf8e_windows_1254_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xffff, 0xffff, 0xffff,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xffff, 0xffff, 0x0178,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+  0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+  0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff,
+};
+
+const uint16_t tutf8e_windows_1255_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0xffff, 0x2039, 0xffff, 0xffff, 0xffff, 0xffff,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0xffff, 0x203a, 0xffff, 0xffff, 0xffff, 0xffff,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20aa, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, 0x05b8, 0x05b9, 0xffff, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf,
+  0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f0, 0x05f1, 0x05f2, 0x05f3, 0x05f4, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+  0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df,
+  0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0xffff, 0xffff, 0x200e, 0x200f, 0xffff,
+};
+
+const uint16_t tutf8e_windows_1256_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
+  0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba,
+  0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f,
+  0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
+  0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643,
+  0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef,
+  0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2,
+};
+
+const uint16_t tutf8e_windows_1257_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0xffff, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0xffff, 0x2039, 0xffff, 0x00a8, 0x02c7, 0x00b8,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0xffff, 0x203a, 0xffff, 0x00af, 0x02db, 0xffff,
+  0x00a0, 0xffff, 0x00a2, 0x00a3, 0x00a4, 0xffff, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6,
+  0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b,
+  0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df,
+  0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c,
+  0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x02d9,
+};
+
+const uint16_t tutf8e_windows_1258_utf8[256] =
+{
+  0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
+  0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
+  0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
+  0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
+  0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
+  0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
+  0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
+  0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
+  0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0xffff, 0x2039, 0x0152, 0xffff, 0xffff, 0xffff,
+  0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0xffff, 0x203a, 0x0153, 0xffff, 0xffff, 0x0178,
+  0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+  0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+  0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x0300, 0x00cd, 0x00ce, 0x00cf,
+  0x0110, 0x00d1, 0x0309, 0x00d3, 0x00d4, 0x01a0, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x01af, 0x0303, 0x00df,
+  0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0301, 0x00ed, 0x00ee, 0x00ef,
+  0x0111, 0x00f1, 0x0323, 0x00f3, 0x00f4, 0x01a1, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x01b0, 0x20ab, 0x00ff,
+};
+
+const TUTF8encoder tutf8e_encoder_iso_8859_1 = (TUTF8encoder) tutf8e_iso_8859_1_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_10 = (TUTF8encoder) tutf8e_iso_8859_10_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_11 = (TUTF8encoder) tutf8e_iso_8859_11_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_13 = (TUTF8encoder) tutf8e_iso_8859_13_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_14 = (TUTF8encoder) tutf8e_iso_8859_14_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_15 = (TUTF8encoder) tutf8e_iso_8859_15_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_16 = (TUTF8encoder) tutf8e_iso_8859_16_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_2 = (TUTF8encoder) tutf8e_iso_8859_2_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_3 = (TUTF8encoder) tutf8e_iso_8859_3_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_4 = (TUTF8encoder) tutf8e_iso_8859_4_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_5 = (TUTF8encoder) tutf8e_iso_8859_5_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_6 = (TUTF8encoder) tutf8e_iso_8859_6_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_7 = (TUTF8encoder) tutf8e_iso_8859_7_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_8 = (TUTF8encoder) tutf8e_iso_8859_8_utf8;
+const TUTF8encoder tutf8e_encoder_iso_8859_9 = (TUTF8encoder) tutf8e_iso_8859_9_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1250 = (TUTF8encoder) tutf8e_windows_1250_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1251 = (TUTF8encoder) tutf8e_windows_1251_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1252 = (TUTF8encoder) tutf8e_windows_1252_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1253 = (TUTF8encoder) tutf8e_windows_1253_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1254 = (TUTF8encoder) tutf8e_windows_1254_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1255 = (TUTF8encoder) tutf8e_windows_1255_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1256 = (TUTF8encoder) tutf8e_windows_1256_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1257 = (TUTF8encoder) tutf8e_windows_1257_utf8;
+const TUTF8encoder tutf8e_encoder_windows_1258 = (TUTF8encoder) tutf8e_windows_1258_utf8;
+
+TUTF8encoder tutf8e_encoder(const char *encoding)
+{
+  if (!strcmp(encoding, "iso-8859-1")) return tutf8e_encoder_iso_8859_1;
+  if (!strcmp(encoding, "iso-8859-10")) return tutf8e_encoder_iso_8859_10;
+  if (!strcmp(encoding, "iso-8859-11")) return tutf8e_encoder_iso_8859_11;
+  if (!strcmp(encoding, "iso-8859-13")) return tutf8e_encoder_iso_8859_13;
+  if (!strcmp(encoding, "iso-8859-14")) return tutf8e_encoder_iso_8859_14;
+  if (!strcmp(encoding, "iso-8859-15")) return tutf8e_encoder_iso_8859_15;
+  if (!strcmp(encoding, "iso-8859-16")) return tutf8e_encoder_iso_8859_16;
+  if (!strcmp(encoding, "iso-8859-2")) return tutf8e_encoder_iso_8859_2;
+  if (!strcmp(encoding, "iso-8859-3")) return tutf8e_encoder_iso_8859_3;
+  if (!strcmp(encoding, "iso-8859-4")) return tutf8e_encoder_iso_8859_4;
+  if (!strcmp(encoding, "iso-8859-5")) return tutf8e_encoder_iso_8859_5;
+  if (!strcmp(encoding, "iso-8859-6")) return tutf8e_encoder_iso_8859_6;
+  if (!strcmp(encoding, "iso-8859-7")) return tutf8e_encoder_iso_8859_7;
+  if (!strcmp(encoding, "iso-8859-8")) return tutf8e_encoder_iso_8859_8;
+  if (!strcmp(encoding, "iso-8859-9")) return tutf8e_encoder_iso_8859_9;
+  if (!strcmp(encoding, "windows-1250")) return tutf8e_encoder_windows_1250;
+  if (!strcmp(encoding, "windows-1251")) return tutf8e_encoder_windows_1251;
+  if (!strcmp(encoding, "windows-1252")) return tutf8e_encoder_windows_1252;
+  if (!strcmp(encoding, "windows-1253")) return tutf8e_encoder_windows_1253;
+  if (!strcmp(encoding, "windows-1254")) return tutf8e_encoder_windows_1254;
+  if (!strcmp(encoding, "windows-1255")) return tutf8e_encoder_windows_1255;
+  if (!strcmp(encoding, "windows-1256")) return tutf8e_encoder_windows_1256;
+  if (!strcmp(encoding, "windows-1257")) return tutf8e_encoder_windows_1257;
+  if (!strcmp(encoding, "windows-1258")) return tutf8e_encoder_windows_1258;
+
+  return NULL;
+}