mirror of
https://github.com/Xahau/xahaud.git
synced 2025-11-29 23:15:49 +00:00
Compare commits
11 Commits
nd-add-pyt
...
ce7b1c4f1d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce7b1c4f1d | ||
|
|
e062dcae58 | ||
|
|
a9d284fec1 | ||
|
|
065d0c3e07 | ||
|
|
4fda40b709 | ||
|
|
6014356d91 | ||
|
|
d790f97430 | ||
|
|
9ed20a4f1c | ||
|
|
89ffc1969b | ||
|
|
79fdafe638 | ||
|
|
2a10013dfc |
282
.github/actions/xahau-actions-cache-restore/action.yml
vendored
Normal file
282
.github/actions/xahau-actions-cache-restore/action.yml
vendored
Normal file
@@ -0,0 +1,282 @@
|
||||
name: 'Xahau Cache Restore (S3 + OverlayFS)'
|
||||
description: 'Drop-in replacement for actions/cache/restore using S3 and OverlayFS for delta caching'
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: 'A list of files, directories, and wildcard patterns to cache (currently only single path supported)'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for restoring the cache'
|
||||
required: true
|
||||
restore-keys:
|
||||
description: 'An ordered list of prefix-matched keys to use for restoring stale cache if no cache hit occurred for key'
|
||||
required: false
|
||||
default: ''
|
||||
s3-bucket:
|
||||
description: 'S3 bucket name for cache storage'
|
||||
required: false
|
||||
default: 'xahaud-github-actions-cache-niq'
|
||||
s3-region:
|
||||
description: 'S3 region'
|
||||
required: false
|
||||
default: 'us-east-1'
|
||||
fail-on-cache-miss:
|
||||
description: 'Fail the workflow if cache entry is not found'
|
||||
required: false
|
||||
default: 'false'
|
||||
lookup-only:
|
||||
description: 'Check if a cache entry exists for the given input(s) without downloading it'
|
||||
required: false
|
||||
default: 'false'
|
||||
use-deltas:
|
||||
description: 'Enable delta caching (download/upload incremental changes). Set to false for base-only caching.'
|
||||
required: false
|
||||
default: 'true'
|
||||
# Note: Composite actions can't access secrets.* directly - must be passed from workflow
|
||||
aws-access-key-id:
|
||||
description: 'AWS Access Key ID for S3 access'
|
||||
required: true
|
||||
aws-secret-access-key:
|
||||
description: 'AWS Secret Access Key for S3 access'
|
||||
required: true
|
||||
|
||||
outputs:
|
||||
cache-hit:
|
||||
description: 'A boolean value to indicate an exact match was found for the primary key'
|
||||
value: ${{ steps.restore-cache.outputs.cache-hit }}
|
||||
cache-primary-key:
|
||||
description: 'The key that was used to restore the cache (may be from restore-keys)'
|
||||
value: ${{ steps.restore-cache.outputs.cache-primary-key }}
|
||||
cache-matched-key:
|
||||
description: 'The key that matched (same as cache-primary-key for compatibility)'
|
||||
value: ${{ steps.restore-cache.outputs.cache-primary-key }}
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Restore cache from S3 with OverlayFS
|
||||
id: restore-cache
|
||||
shell: bash
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
|
||||
S3_BUCKET: ${{ inputs.s3-bucket }}
|
||||
S3_REGION: ${{ inputs.s3-region }}
|
||||
CACHE_KEY: ${{ inputs.key }}
|
||||
RESTORE_KEYS: ${{ inputs.restore-keys }}
|
||||
TARGET_PATH: ${{ inputs.path }}
|
||||
FAIL_ON_MISS: ${{ inputs.fail-on-cache-miss }}
|
||||
LOOKUP_ONLY: ${{ inputs.lookup-only }}
|
||||
USE_DELTAS: ${{ inputs.use-deltas }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "=========================================="
|
||||
echo "Xahau Cache Restore (S3 + OverlayFS)"
|
||||
echo "=========================================="
|
||||
echo "Target path: ${TARGET_PATH}"
|
||||
echo "Primary key: ${CACHE_KEY}"
|
||||
echo "S3 bucket: s3://${S3_BUCKET}"
|
||||
echo "Use deltas: ${USE_DELTAS}"
|
||||
echo ""
|
||||
|
||||
# Generate unique cache workspace
|
||||
CACHE_HASH=$(echo "${CACHE_KEY}" | md5sum | cut -d' ' -f1)
|
||||
CACHE_WORKSPACE="/tmp/xahau-cache-${CACHE_HASH}"
|
||||
|
||||
echo "Cache workspace: ${CACHE_WORKSPACE}"
|
||||
|
||||
# Create OverlayFS directory structure
|
||||
mkdir -p "${CACHE_WORKSPACE}"/{base,upper,work,merged}
|
||||
|
||||
# Function to try downloading from S3
|
||||
try_restore_key() {
|
||||
local try_key="$1"
|
||||
local s3_base="s3://${S3_BUCKET}/${try_key}-base.tar.zst"
|
||||
|
||||
echo "Trying cache key: ${try_key}"
|
||||
|
||||
# Check if base exists (one base per key, immutable)
|
||||
echo "Checking for base layer..."
|
||||
if aws s3 ls "${s3_base}" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
echo "✓ Found base layer: ${s3_base}"
|
||||
|
||||
if [ "${LOOKUP_ONLY}" = "true" ]; then
|
||||
echo "Lookup-only mode: cache exists, skipping download"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Download base layer
|
||||
echo "Downloading base layer..."
|
||||
aws s3 cp "${s3_base}" /tmp/cache-base.tar.zst --region "${S3_REGION}" --quiet
|
||||
|
||||
# Extract base layer
|
||||
echo "Extracting base layer..."
|
||||
tar -xf /tmp/cache-base.tar.zst -C "${CACHE_WORKSPACE}/base"
|
||||
rm /tmp/cache-base.tar.zst
|
||||
|
||||
# Query for latest timestamped delta (only if use-deltas enabled)
|
||||
if [ "${USE_DELTAS}" = "true" ]; then
|
||||
echo "Querying for latest delta..."
|
||||
LATEST_DELTA=$(aws s3api list-objects-v2 \
|
||||
--bucket "${S3_BUCKET}" \
|
||||
--prefix "${try_key}-delta-" \
|
||||
--region "${S3_REGION}" \
|
||||
--query 'sort_by(Contents, &LastModified)[-1].Key' \
|
||||
--output text 2>/dev/null || echo "")
|
||||
|
||||
if [ -n "${LATEST_DELTA}" ] && [ "${LATEST_DELTA}" != "None" ]; then
|
||||
echo "✓ Found latest delta: ${LATEST_DELTA}"
|
||||
echo "Downloading delta layer..."
|
||||
aws s3 cp "s3://${S3_BUCKET}/${LATEST_DELTA}" /tmp/cache-delta.tar.zst --region "${S3_REGION}" --quiet
|
||||
|
||||
echo "Extracting delta layer..."
|
||||
tar -xf /tmp/cache-delta.tar.zst -C "${CACHE_WORKSPACE}/upper" 2>/dev/null || true
|
||||
rm /tmp/cache-delta.tar.zst
|
||||
else
|
||||
echo "ℹ No delta layer found (this is fine for first build)"
|
||||
fi
|
||||
else
|
||||
echo "ℹ Delta caching disabled (use-deltas: false)"
|
||||
fi
|
||||
|
||||
return 0
|
||||
else
|
||||
echo "✗ No base layer found for key: ${try_key}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Try primary key first
|
||||
MATCHED_KEY=""
|
||||
EXACT_MATCH="false"
|
||||
|
||||
if try_restore_key "${CACHE_KEY}"; then
|
||||
MATCHED_KEY="${CACHE_KEY}"
|
||||
EXACT_MATCH="true"
|
||||
echo ""
|
||||
echo "🎯 Exact cache hit for key: ${CACHE_KEY}"
|
||||
else
|
||||
# Try restore-keys (prefix matching)
|
||||
if [ -n "${RESTORE_KEYS}" ]; then
|
||||
echo ""
|
||||
echo "Primary key not found, trying restore-keys..."
|
||||
|
||||
# Split restore-keys by newline
|
||||
while IFS= read -r restore_key; do
|
||||
# Skip empty lines
|
||||
[ -z "${restore_key}" ] && continue
|
||||
|
||||
# Trim whitespace
|
||||
restore_key=$(echo "${restore_key}" | xargs)
|
||||
|
||||
if try_restore_key "${restore_key}"; then
|
||||
MATCHED_KEY="${restore_key}"
|
||||
EXACT_MATCH="false"
|
||||
echo ""
|
||||
echo "✓ Cache restored from fallback key: ${restore_key}"
|
||||
break
|
||||
fi
|
||||
done <<< "${RESTORE_KEYS}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if we found anything
|
||||
if [ -z "${MATCHED_KEY}" ]; then
|
||||
echo ""
|
||||
echo "❌ No cache found for key: ${CACHE_KEY}"
|
||||
echo "This is BOOTSTRAP mode - first build for this cache key"
|
||||
|
||||
if [ "${FAIL_ON_MISS}" = "true" ]; then
|
||||
echo "fail-on-cache-miss is enabled, failing workflow"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set outputs for cache miss
|
||||
echo "cache-hit=false" >> $GITHUB_OUTPUT
|
||||
echo "cache-primary-key=" >> $GITHUB_OUTPUT
|
||||
|
||||
# Create empty cache directory for bootstrap
|
||||
mkdir -p "${TARGET_PATH}"
|
||||
|
||||
# Record bootstrap mode for save action
|
||||
# Format: path:workspace:matched_key:primary_key:exact_match:use_deltas
|
||||
# For bootstrap: workspace="bootstrap", matched_key=primary_key, exact_match=false
|
||||
MOUNT_REGISTRY="/tmp/xahau-cache-mounts.txt"
|
||||
echo "${TARGET_PATH}:bootstrap:${CACHE_KEY}:${CACHE_KEY}:false:${USE_DELTAS}" >> "${MOUNT_REGISTRY}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache restore completed (bootstrap mode)"
|
||||
echo "Created empty cache directory: ${TARGET_PATH}"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If lookup-only, we're done
|
||||
if [ "${LOOKUP_ONLY}" = "true" ]; then
|
||||
echo "cache-hit=${EXACT_MATCH}" >> $GITHUB_OUTPUT
|
||||
echo "cache-primary-key=${MATCHED_KEY}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Clean up workspace
|
||||
rm -rf "${CACHE_WORKSPACE}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache lookup completed (lookup-only mode)"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Mount OverlayFS
|
||||
echo ""
|
||||
echo "Mounting OverlayFS..."
|
||||
sudo mount -t overlay overlay \
|
||||
-o lowerdir="${CACHE_WORKSPACE}/base",upperdir="${CACHE_WORKSPACE}/upper",workdir="${CACHE_WORKSPACE}/work" \
|
||||
"${CACHE_WORKSPACE}/merged"
|
||||
|
||||
# Verify mount
|
||||
if mount | grep -q "${CACHE_WORKSPACE}/merged"; then
|
||||
echo "✓ OverlayFS mounted successfully"
|
||||
else
|
||||
echo "❌ Failed to mount OverlayFS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create target directory parent if needed
|
||||
TARGET_PARENT=$(dirname "${TARGET_PATH}")
|
||||
mkdir -p "${TARGET_PARENT}"
|
||||
|
||||
# Remove existing target if it exists
|
||||
if [ -e "${TARGET_PATH}" ]; then
|
||||
echo "Removing existing target: ${TARGET_PATH}"
|
||||
rm -rf "${TARGET_PATH}"
|
||||
fi
|
||||
|
||||
# Symlink target path to merged view
|
||||
echo "Creating symlink: ${TARGET_PATH} -> ${CACHE_WORKSPACE}/merged"
|
||||
ln -s "${CACHE_WORKSPACE}/merged" "${TARGET_PATH}"
|
||||
|
||||
# Save mount info for cleanup/save later
|
||||
# Format: path:workspace:matched_key:primary_key:exact_match:use_deltas
|
||||
# This tells save action whether to create new base (partial match) or just delta (exact match)
|
||||
MOUNT_REGISTRY="/tmp/xahau-cache-mounts.txt"
|
||||
echo "${TARGET_PATH}:${CACHE_WORKSPACE}:${MATCHED_KEY}:${CACHE_KEY}:${EXACT_MATCH}:${USE_DELTAS}" >> "${MOUNT_REGISTRY}"
|
||||
|
||||
# Set outputs
|
||||
echo "cache-hit=${EXACT_MATCH}" >> $GITHUB_OUTPUT
|
||||
echo "cache-primary-key=${MATCHED_KEY}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Show statistics
|
||||
echo ""
|
||||
echo "Cache statistics:"
|
||||
echo " Base layer size: $(du -sh ${CACHE_WORKSPACE}/base 2>/dev/null | cut -f1 || echo '0')"
|
||||
echo " Delta layer size: $(du -sh ${CACHE_WORKSPACE}/upper 2>/dev/null | cut -f1 || echo '0')"
|
||||
echo " Merged view size: $(du -sh ${CACHE_WORKSPACE}/merged 2>/dev/null | cut -f1 || echo '0')"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache restore completed successfully"
|
||||
echo "Exact match: ${EXACT_MATCH}"
|
||||
echo "Matched key: ${MATCHED_KEY}"
|
||||
echo "=========================================="
|
||||
342
.github/actions/xahau-actions-cache-save/action.yml
vendored
Normal file
342
.github/actions/xahau-actions-cache-save/action.yml
vendored
Normal file
@@ -0,0 +1,342 @@
|
||||
name: 'Xahau Cache Save (S3 + OverlayFS)'
|
||||
description: 'Drop-in replacement for actions/cache/save using S3 and OverlayFS for delta caching'
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: 'A list of files, directories, and wildcard patterns to cache (currently only single path supported)'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for saving the cache'
|
||||
required: true
|
||||
s3-bucket:
|
||||
description: 'S3 bucket name for cache storage'
|
||||
required: false
|
||||
default: 'xahaud-github-actions-cache-niq'
|
||||
s3-region:
|
||||
description: 'S3 region'
|
||||
required: false
|
||||
default: 'us-east-1'
|
||||
use-deltas:
|
||||
description: 'Enable delta caching (download/upload incremental changes). Set to false for base-only caching.'
|
||||
required: false
|
||||
default: 'true'
|
||||
# Note: Composite actions can't access secrets.* directly - must be passed from workflow
|
||||
aws-access-key-id:
|
||||
description: 'AWS Access Key ID for S3 access'
|
||||
required: true
|
||||
aws-secret-access-key:
|
||||
description: 'AWS Secret Access Key for S3 access'
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Save cache to S3 with OverlayFS delta
|
||||
shell: bash
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
|
||||
S3_BUCKET: ${{ inputs.s3-bucket }}
|
||||
S3_REGION: ${{ inputs.s3-region }}
|
||||
CACHE_KEY: ${{ inputs.key }}
|
||||
TARGET_PATH: ${{ inputs.path }}
|
||||
USE_DELTAS: ${{ inputs.use-deltas }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "=========================================="
|
||||
echo "Xahau Cache Save (S3 + OverlayFS)"
|
||||
echo "=========================================="
|
||||
echo "Target path: ${TARGET_PATH}"
|
||||
echo "Cache key: ${CACHE_KEY}"
|
||||
echo "S3 bucket: s3://${S3_BUCKET}"
|
||||
echo ""
|
||||
|
||||
# Find the cache workspace from mount registry
|
||||
MOUNT_REGISTRY="/tmp/xahau-cache-mounts.txt"
|
||||
|
||||
if [ ! -f "${MOUNT_REGISTRY}" ]; then
|
||||
echo "⚠️ No cache mounts found (mount registry doesn't exist)"
|
||||
echo "This usually means cache restore was not called, or there was no cache to restore."
|
||||
echo "Skipping cache save."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Find entry for this path
|
||||
# Format: path:workspace:matched_key:primary_key:exact_match:use_deltas
|
||||
# Bootstrap mode: path:bootstrap:key:key:false:true/false (workspace="bootstrap")
|
||||
CACHE_WORKSPACE=""
|
||||
MATCHED_KEY=""
|
||||
PRIMARY_KEY=""
|
||||
EXACT_MATCH=""
|
||||
REGISTRY_USE_DELTAS=""
|
||||
|
||||
while IFS=: read -r mount_path mount_workspace mount_matched_key mount_primary_key mount_exact_match mount_use_deltas; do
|
||||
if [ "${mount_path}" = "${TARGET_PATH}" ]; then
|
||||
CACHE_WORKSPACE="${mount_workspace}"
|
||||
MATCHED_KEY="${mount_matched_key}"
|
||||
PRIMARY_KEY="${mount_primary_key}"
|
||||
EXACT_MATCH="${mount_exact_match}"
|
||||
REGISTRY_USE_DELTAS="${mount_use_deltas}"
|
||||
break
|
||||
fi
|
||||
done < "${MOUNT_REGISTRY}"
|
||||
|
||||
if [ -z "${CACHE_WORKSPACE}" ] && [ -z "${MATCHED_KEY}" ]; then
|
||||
echo "⚠️ No cache entry found for path: ${TARGET_PATH}"
|
||||
echo "This usually means cache restore was not called for this path."
|
||||
echo "Skipping cache save."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine cache mode
|
||||
if [ "${CACHE_WORKSPACE}" = "bootstrap" ]; then
|
||||
CACHE_MODE="bootstrap"
|
||||
PRIMARY_KEY="${MATCHED_KEY}" # In bootstrap, matched_key field contains primary key
|
||||
echo "Cache mode: BOOTSTRAP (first build for this key)"
|
||||
echo "Primary key: ${PRIMARY_KEY}"
|
||||
elif [ "${EXACT_MATCH}" = "false" ]; then
|
||||
CACHE_MODE="partial-match"
|
||||
echo "Cache mode: PARTIAL MATCH (restore-key used)"
|
||||
echo "Cache workspace: ${CACHE_WORKSPACE}"
|
||||
echo "Matched key from restore: ${MATCHED_KEY}"
|
||||
echo "Primary key (will save new base): ${PRIMARY_KEY}"
|
||||
else
|
||||
CACHE_MODE="exact-match"
|
||||
echo "Cache mode: EXACT MATCH (cache hit)"
|
||||
echo "Cache workspace: ${CACHE_WORKSPACE}"
|
||||
echo "Matched key: ${MATCHED_KEY}"
|
||||
fi
|
||||
echo "Use deltas: ${REGISTRY_USE_DELTAS}"
|
||||
echo ""
|
||||
|
||||
# Handle different cache modes
|
||||
if [ "${CACHE_MODE}" = "bootstrap" ]; then
|
||||
# Bootstrap: Save entire cache as base layer (no OverlayFS was used)
|
||||
echo "Bootstrap mode: Creating initial base layer from ${TARGET_PATH}"
|
||||
|
||||
BASE_TARBALL="/tmp/xahau-cache-base-$$.tar.zst"
|
||||
echo "Creating base tarball..."
|
||||
tar -cf - -C "${TARGET_PATH}" . | zstd -3 -T0 -q -o "${BASE_TARBALL}"
|
||||
|
||||
BASE_SIZE=$(du -h "${BASE_TARBALL}" | cut -f1)
|
||||
echo "✓ Base tarball created: ${BASE_SIZE}"
|
||||
echo ""
|
||||
|
||||
# Use static base name (one base per key, immutable)
|
||||
S3_BASE_KEY="s3://${S3_BUCKET}/${PRIMARY_KEY}-base.tar.zst"
|
||||
|
||||
# Check if base already exists (immutability - first write wins)
|
||||
if aws s3 ls "${S3_BASE_KEY}" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
echo "⚠️ Base layer already exists: ${S3_BASE_KEY}"
|
||||
echo "Skipping upload (immutability - first write wins, like GitHub Actions)"
|
||||
else
|
||||
echo "Uploading base layer to S3..."
|
||||
echo " Key: ${PRIMARY_KEY}-base.tar.zst"
|
||||
|
||||
aws s3 cp "${BASE_TARBALL}" "${S3_BASE_KEY}" \
|
||||
--region "${S3_REGION}" \
|
||||
--tagging "type=base" \
|
||||
--quiet
|
||||
|
||||
echo "✓ Uploaded: ${S3_BASE_KEY}"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm -f "${BASE_TARBALL}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Bootstrap cache save completed"
|
||||
echo "Base size: ${BASE_SIZE}"
|
||||
echo "Cache key: ${PRIMARY_KEY}"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
|
||||
elif [ "${CACHE_MODE}" = "partial-match" ]; then
|
||||
# Partial match: Save merged view as new base ONLY (no delta)
|
||||
# The delta is relative to the OLD base, not the NEW base we're creating
|
||||
echo "Partial match mode: Saving new base layer for primary key"
|
||||
echo "Note: Delta will NOT be saved (it's relative to old base)"
|
||||
|
||||
BASE_TARBALL="/tmp/xahau-cache-base-$$.tar.zst"
|
||||
echo "Creating base tarball from merged view..."
|
||||
tar -cf - -C "${CACHE_WORKSPACE}/merged" . | zstd -3 -T0 -q -o "${BASE_TARBALL}"
|
||||
|
||||
BASE_SIZE=$(du -h "${BASE_TARBALL}" | cut -f1)
|
||||
echo "✓ Base tarball created: ${BASE_SIZE}"
|
||||
echo ""
|
||||
|
||||
# Use static base name (one base per key, immutable)
|
||||
S3_BASE_KEY="s3://${S3_BUCKET}/${PRIMARY_KEY}-base.tar.zst"
|
||||
|
||||
# Check if base already exists (immutability - first write wins)
|
||||
if aws s3 ls "${S3_BASE_KEY}" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
echo "⚠️ Base layer already exists: ${S3_BASE_KEY}"
|
||||
echo "Skipping upload (immutability - first write wins, like GitHub Actions)"
|
||||
else
|
||||
echo "Uploading new base layer to S3..."
|
||||
echo " Key: ${PRIMARY_KEY}-base.tar.zst"
|
||||
|
||||
aws s3 cp "${BASE_TARBALL}" "${S3_BASE_KEY}" \
|
||||
--region "${S3_REGION}" \
|
||||
--tagging "type=base" \
|
||||
--quiet
|
||||
|
||||
echo "✓ Uploaded: ${S3_BASE_KEY}"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm -f "${BASE_TARBALL}"
|
||||
|
||||
# Unmount and cleanup
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
if mount | grep -q "${CACHE_WORKSPACE}/merged"; then
|
||||
sudo umount "${CACHE_WORKSPACE}/merged" || {
|
||||
echo "⚠️ Warning: Failed to unmount ${CACHE_WORKSPACE}/merged"
|
||||
echo "Attempting lazy unmount..."
|
||||
sudo umount -l "${CACHE_WORKSPACE}/merged" || true
|
||||
}
|
||||
fi
|
||||
rm -rf "${CACHE_WORKSPACE}"
|
||||
|
||||
# Remove from registry
|
||||
if [ -f "${MOUNT_REGISTRY}" ]; then
|
||||
grep -v "^${TARGET_PATH}:" "${MOUNT_REGISTRY}" > "${MOUNT_REGISTRY}.tmp" 2>/dev/null || true
|
||||
mv "${MOUNT_REGISTRY}.tmp" "${MOUNT_REGISTRY}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "✓ Cleanup completed"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Partial match cache save completed"
|
||||
echo "New base created for: ${PRIMARY_KEY}"
|
||||
echo "Base size: ${BASE_SIZE}"
|
||||
if [ "${REGISTRY_USE_DELTAS}" = "true" ]; then
|
||||
echo "Next exact-match build will create deltas from this base"
|
||||
else
|
||||
echo "Next exact-match build will reuse this base (base-only mode)"
|
||||
fi
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# For exact-match ONLY: Save delta (if use-deltas enabled)
|
||||
if [ "${CACHE_MODE}" = "exact-match" ]; then
|
||||
# If deltas are disabled, just cleanup and exit
|
||||
if [ "${REGISTRY_USE_DELTAS}" != "true" ]; then
|
||||
echo "ℹ️ Delta caching disabled (use-deltas: false)"
|
||||
echo "Base already exists for this key, nothing to save."
|
||||
|
||||
# Unmount and cleanup
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
if mount | grep -q "${CACHE_WORKSPACE}/merged"; then
|
||||
sudo umount "${CACHE_WORKSPACE}/merged" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "${CACHE_WORKSPACE}"
|
||||
|
||||
# Remove from registry
|
||||
if [ -f "${MOUNT_REGISTRY}" ]; then
|
||||
grep -v "^${TARGET_PATH}:" "${MOUNT_REGISTRY}" > "${MOUNT_REGISTRY}.tmp" 2>/dev/null || true
|
||||
mv "${MOUNT_REGISTRY}.tmp" "${MOUNT_REGISTRY}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache save completed (base-only mode)"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check if upper layer has any changes
|
||||
if [ -z "$(ls -A ${CACHE_WORKSPACE}/upper 2>/dev/null)" ]; then
|
||||
echo "ℹ️ No changes detected in upper layer (cache is unchanged)"
|
||||
echo "Skipping delta upload to save bandwidth."
|
||||
|
||||
# Still unmount and cleanup
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
sudo umount "${CACHE_WORKSPACE}/merged" 2>/dev/null || true
|
||||
rm -rf "${CACHE_WORKSPACE}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache save completed (no changes)"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Show delta statistics
|
||||
echo "Delta layer statistics:"
|
||||
echo " Files changed: $(find ${CACHE_WORKSPACE}/upper -type f 2>/dev/null | wc -l)"
|
||||
echo " Delta size: $(du -sh ${CACHE_WORKSPACE}/upper 2>/dev/null | cut -f1)"
|
||||
echo ""
|
||||
|
||||
# Create delta tarball from upper layer
|
||||
echo "Creating delta tarball..."
|
||||
DELTA_TARBALL="/tmp/xahau-cache-delta-$$.tar.zst"
|
||||
|
||||
tar -cf - -C "${CACHE_WORKSPACE}/upper" . | zstd -3 -T0 -q -o "${DELTA_TARBALL}"
|
||||
|
||||
DELTA_SIZE=$(du -h "${DELTA_TARBALL}" | cut -f1)
|
||||
echo "✓ Delta tarball created: ${DELTA_SIZE}"
|
||||
echo ""
|
||||
|
||||
# Upload timestamped delta (no overwrites = zero concurrency issues)
|
||||
TIMESTAMP=$(date +%Y%m%d%H%M%S)
|
||||
COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Use PRIMARY_KEY for delta (ensures deltas match their base)
|
||||
S3_DELTA_TIMESTAMPED="s3://${S3_BUCKET}/${PRIMARY_KEY}-delta-${TIMESTAMP}-${COMMIT_SHA}.tar.zst"
|
||||
|
||||
echo "Uploading timestamped delta to S3..."
|
||||
echo " Key: ${PRIMARY_KEY}-delta-${TIMESTAMP}-${COMMIT_SHA}.tar.zst"
|
||||
|
||||
# Upload with tag for auto-deletion after 7 days
|
||||
aws s3 cp "${DELTA_TARBALL}" "${S3_DELTA_TIMESTAMPED}" \
|
||||
--region "${S3_REGION}" \
|
||||
--tagging "type=delta-archive" \
|
||||
--quiet
|
||||
|
||||
echo "✓ Uploaded: ${S3_DELTA_TIMESTAMPED}"
|
||||
echo " (tagged for auto-deletion after 7 days)"
|
||||
|
||||
# Cleanup delta tarball
|
||||
rm -f "${DELTA_TARBALL}"
|
||||
|
||||
# Cleanup: Unmount OverlayFS and remove workspace
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
|
||||
if mount | grep -q "${CACHE_WORKSPACE}/merged"; then
|
||||
sudo umount "${CACHE_WORKSPACE}/merged" || {
|
||||
echo "⚠️ Warning: Failed to unmount ${CACHE_WORKSPACE}/merged"
|
||||
echo "Attempting lazy unmount..."
|
||||
sudo umount -l "${CACHE_WORKSPACE}/merged" || true
|
||||
}
|
||||
fi
|
||||
|
||||
# Remove workspace
|
||||
rm -rf "${CACHE_WORKSPACE}"
|
||||
fi
|
||||
|
||||
# Remove from registry
|
||||
if [ -f "${MOUNT_REGISTRY}" ]; then
|
||||
grep -v "^${TARGET_PATH}:" "${MOUNT_REGISTRY}" > "${MOUNT_REGISTRY}.tmp" 2>/dev/null || true
|
||||
mv "${MOUNT_REGISTRY}.tmp" "${MOUNT_REGISTRY}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "✓ Cleanup completed"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Cache save completed successfully"
|
||||
echo "Mode: ${CACHE_MODE}"
|
||||
echo "Cache key: ${PRIMARY_KEY}"
|
||||
if [ -n "${DELTA_SIZE:-}" ]; then
|
||||
echo "Delta size: ${DELTA_SIZE}"
|
||||
fi
|
||||
echo "=========================================="
|
||||
182
.github/workflows/test-overlayfs-delta.yml
vendored
Normal file
182
.github/workflows/test-overlayfs-delta.yml
vendored
Normal file
@@ -0,0 +1,182 @@
|
||||
name: Test OverlayFS Delta Extraction
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["*"]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test-overlayfs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# - name: Test encrypted secrets (decrypt test message)
|
||||
# run: |
|
||||
# echo "========================================"
|
||||
# echo "TESTING ENCRYPTED SECRETS"
|
||||
# echo "========================================"
|
||||
# echo ""
|
||||
# echo "Decrypting test message from .github/secrets/test-message.gpg"
|
||||
# echo "Using encryption key from GitHub Secrets..."
|
||||
# echo ""
|
||||
#
|
||||
# # Decrypt using key from GitHub Secrets
|
||||
# echo "${{ secrets.TEST_ENCRYPTION_KEY }}" | \
|
||||
# gpg --batch --yes --passphrase-fd 0 \
|
||||
# --decrypt .github/secrets/test-message.gpg
|
||||
#
|
||||
# echo ""
|
||||
# echo "========================================"
|
||||
# echo "If you see the success message above,"
|
||||
# echo "then encrypted secrets work! 🎉"
|
||||
# echo "========================================"
|
||||
# echo ""
|
||||
|
||||
- name: Setup OverlayFS layers
|
||||
run: |
|
||||
echo "=== Creating directory structure ==="
|
||||
mkdir -p /tmp/test/{base,delta,upper,work,merged}
|
||||
|
||||
echo "=== Creating base layer files ==="
|
||||
echo "base file 1" > /tmp/test/base/file1.txt
|
||||
echo "base file 2" > /tmp/test/base/file2.txt
|
||||
echo "base file 3" > /tmp/test/base/file3.txt
|
||||
mkdir -p /tmp/test/base/subdir
|
||||
echo "base subdir file" > /tmp/test/base/subdir/file.txt
|
||||
|
||||
echo "=== Base layer contents ==="
|
||||
find /tmp/test/base -type f -exec sh -c 'echo "{}:"; cat "{}"' \;
|
||||
|
||||
echo "=== Mounting OverlayFS ==="
|
||||
sudo mount -t overlay overlay \
|
||||
-o lowerdir=/tmp/test/base,upperdir=/tmp/test/upper,workdir=/tmp/test/work \
|
||||
/tmp/test/merged
|
||||
|
||||
echo "=== Mounted successfully ==="
|
||||
mount | grep overlay
|
||||
|
||||
- name: Verify merged view shows base files
|
||||
run: |
|
||||
echo "=== Contents of /merged (should show base files) ==="
|
||||
ls -R /tmp/test/merged
|
||||
find /tmp/test/merged -type f -exec sh -c 'echo "{}:"; cat "{}"' \;
|
||||
|
||||
- name: Make changes via merged layer
|
||||
run: |
|
||||
echo "=== Making changes via /merged ==="
|
||||
|
||||
# Overwrite existing file
|
||||
echo "MODIFIED file 2" > /tmp/test/merged/file2.txt
|
||||
echo "Modified file2.txt"
|
||||
|
||||
# Create new file
|
||||
echo "NEW file 4" > /tmp/test/merged/file4.txt
|
||||
echo "Created new file4.txt"
|
||||
|
||||
# Create new directory with file
|
||||
mkdir -p /tmp/test/merged/newdir
|
||||
echo "NEW file in new dir" > /tmp/test/merged/newdir/newfile.txt
|
||||
echo "Created newdir/newfile.txt"
|
||||
|
||||
# Add file to existing directory
|
||||
echo "NEW file in existing subdir" > /tmp/test/merged/subdir/newfile.txt
|
||||
echo "Created subdir/newfile.txt"
|
||||
|
||||
echo "=== Changes complete ==="
|
||||
|
||||
- name: Show the delta (upperdir)
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "THE DELTA (only changes in /upper):"
|
||||
echo "========================================"
|
||||
|
||||
if [ -z "$(ls -A /tmp/test/upper)" ]; then
|
||||
echo "Upper directory is empty - no changes detected"
|
||||
else
|
||||
echo "Upper directory structure:"
|
||||
ls -R /tmp/test/upper
|
||||
echo ""
|
||||
echo "Upper directory files with content:"
|
||||
find /tmp/test/upper -type f -exec sh -c 'echo "---"; echo "FILE: {}"; cat "{}"; echo ""' \;
|
||||
|
||||
echo "========================================"
|
||||
echo "SIZE OF DELTA:"
|
||||
du -sh /tmp/test/upper
|
||||
echo "========================================"
|
||||
fi
|
||||
|
||||
- name: Compare base vs upper vs merged
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "COMPARISON:"
|
||||
echo "========================================"
|
||||
|
||||
echo "BASE layer (original, untouched):"
|
||||
ls -la /tmp/test/base/
|
||||
echo ""
|
||||
|
||||
echo "UPPER layer (DELTA - only changes):"
|
||||
ls -la /tmp/test/upper/
|
||||
echo ""
|
||||
|
||||
echo "MERGED layer (unified view = base + upper):"
|
||||
ls -la /tmp/test/merged/
|
||||
echo ""
|
||||
|
||||
echo "========================================"
|
||||
echo "PROOF: Upper dir contains ONLY the delta!"
|
||||
echo "========================================"
|
||||
|
||||
- name: Simulate tarball creation (what we'd upload)
|
||||
run: |
|
||||
echo "=== Creating tarball of delta ==="
|
||||
tar -czf /tmp/delta.tar.gz -C /tmp/test/upper .
|
||||
|
||||
echo "Delta tarball size:"
|
||||
ls -lh /tmp/delta.tar.gz
|
||||
|
||||
echo ""
|
||||
echo "Delta tarball contents:"
|
||||
tar -tzf /tmp/delta.tar.gz
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "This is what we'd upload to S3/rsync!"
|
||||
echo "Only ~few KB instead of entire cache!"
|
||||
echo "========================================"
|
||||
|
||||
- name: Upload delta to S3 (actual test!)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "UPLOADING TO S3"
|
||||
echo "========================================"
|
||||
|
||||
# Upload the delta tarball
|
||||
aws s3 cp /tmp/delta.tar.gz \
|
||||
s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz \
|
||||
--region us-east-1
|
||||
|
||||
echo ""
|
||||
echo "✅ Successfully uploaded to S3!"
|
||||
echo "File: s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz"
|
||||
echo ""
|
||||
|
||||
# Verify it exists
|
||||
echo "Verifying upload..."
|
||||
aws s3 ls s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz --region us-east-1
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "S3 upload test complete! 🚀"
|
||||
echo "========================================"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== Unmounting OverlayFS ==="
|
||||
sudo umount /tmp/test/merged || true
|
||||
@@ -458,6 +458,7 @@ target_sources (rippled PRIVATE
|
||||
src/ripple/app/tx/impl/CreateOffer.cpp
|
||||
src/ripple/app/tx/impl/CreateTicket.cpp
|
||||
src/ripple/app/tx/impl/Cron.cpp
|
||||
src/ripple/app/tx/impl/CronSet.cpp
|
||||
src/ripple/app/tx/impl/DeleteAccount.cpp
|
||||
src/ripple/app/tx/impl/DepositPreauth.cpp
|
||||
src/ripple/app/tx/impl/Escrow.cpp
|
||||
@@ -475,7 +476,6 @@ target_sources (rippled PRIVATE
|
||||
src/ripple/app/tx/impl/Payment.cpp
|
||||
src/ripple/app/tx/impl/Remit.cpp
|
||||
src/ripple/app/tx/impl/SetAccount.cpp
|
||||
src/ripple/app/tx/impl/SetCron.cpp
|
||||
src/ripple/app/tx/impl/SetHook.cpp
|
||||
src/ripple/app/tx/impl/SetRemarks.cpp
|
||||
src/ripple/app/tx/impl/SetRegularKey.cpp
|
||||
|
||||
@@ -72,15 +72,15 @@ It generates many files of [results](results):
|
||||
desired as described above. In a perfect repo, this file will be
|
||||
empty.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../.github/workflows/levelization.yml) to validate
|
||||
Github workflow](../../.github/workflows/levelization.yml.disabled) to validate
|
||||
that nothing changed.
|
||||
* [`ordering.txt`](results/ordering.txt): A list showing relationships
|
||||
between modules where there are no loops as they actually exist, as
|
||||
opposed to how they are desired as described above.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../.github/workflows/levelization.yml) to validate
|
||||
Github workflow](../../.github/workflows/levelization.yml.disabled) to validate
|
||||
that nothing changed.
|
||||
* [`levelization.yml`](../../.github/workflows/levelization.yml)
|
||||
* [`levelization.yml`](../../.github/workflows/levelization.yml.disabled)
|
||||
Github Actions workflow to test that levelization loops haven't
|
||||
changed. Unfortunately, if changes are detected, it can't tell if
|
||||
they are improvements or not, so if you have resolved any issues or
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
#define KEYLET_NFT_OFFER 23
|
||||
#define KEYLET_HOOK_DEFINITION 24
|
||||
#define KEYLET_HOOK_STATE_DIR 25
|
||||
#define KEYLET_CRON 26
|
||||
|
||||
#define COMPARE_EQUAL 1U
|
||||
#define COMPARE_LESS 2U
|
||||
|
||||
@@ -278,8 +278,7 @@ enum keylet_code : uint32_t {
|
||||
NFT_OFFER = 23,
|
||||
HOOK_DEFINITION = 24,
|
||||
HOOK_STATE_DIR = 25,
|
||||
LAST_KLTYPE_V0 = HOOK_DEFINITION,
|
||||
LAST_KLTYPE_V1 = HOOK_STATE_DIR,
|
||||
CRON = 26
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -2903,17 +2903,6 @@ DEFINE_HOOK_FUNCTION(
|
||||
if (write_len < 34)
|
||||
return TOO_SMALL;
|
||||
|
||||
bool const v1 = applyCtx.view().rules().enabled(featureHooksUpdate1);
|
||||
|
||||
if (keylet_type == 0)
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
auto const last =
|
||||
v1 ? keylet_code::LAST_KLTYPE_V1 : keylet_code::LAST_KLTYPE_V0;
|
||||
|
||||
if (keylet_type > last)
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
try
|
||||
{
|
||||
switch (keylet_type)
|
||||
@@ -3015,7 +3004,8 @@ DEFINE_HOOK_FUNCTION(
|
||||
return serialize_keylet(kl, memory, write_ptr, write_len);
|
||||
}
|
||||
|
||||
// keylets that take 20 byte account id, and 4 byte uint
|
||||
// keylets that take 20 byte account id, and (4 byte uint for 32
|
||||
// byte hash)
|
||||
case keylet_code::OFFER:
|
||||
case keylet_code::CHECK:
|
||||
case keylet_code::ESCROW:
|
||||
@@ -3058,6 +3048,33 @@ DEFINE_HOOK_FUNCTION(
|
||||
return serialize_keylet(kl, memory, write_ptr, write_len);
|
||||
}
|
||||
|
||||
// keylets that take 20 byte account id, and 4 byte uint
|
||||
case keylet_code::CRON: {
|
||||
if (!applyCtx.view().rules().enabled(featureCron))
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
if (a == 0 || b == 0)
|
||||
return INVALID_ARGUMENT;
|
||||
if (e != 0 || f != 0 || d != 0)
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
uint32_t read_ptr = a, read_len = b;
|
||||
|
||||
if (NOT_IN_BOUNDS(read_ptr, read_len, memory_length))
|
||||
return OUT_OF_BOUNDS;
|
||||
|
||||
if (read_len != 20)
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
ripple::AccountID id = AccountID::fromVoid(memory + read_ptr);
|
||||
|
||||
uint32_t seq = c;
|
||||
|
||||
ripple::Keylet kl = ripple::keylet::cron(seq, id);
|
||||
|
||||
return serialize_keylet(kl, memory, write_ptr, write_len);
|
||||
}
|
||||
|
||||
// keylets that take a 32 byte uint and an 8byte uint64
|
||||
case keylet_code::PAGE: {
|
||||
if (a == 0 || b == 0)
|
||||
@@ -3105,6 +3122,9 @@ DEFINE_HOOK_FUNCTION(
|
||||
}
|
||||
|
||||
case keylet_code::HOOK_STATE_DIR: {
|
||||
if (!applyCtx.view().rules().enabled(featureHooksUpdate1))
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
if (a == 0 || b == 0 || c == 0 || d == 0)
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
@@ -3279,7 +3299,7 @@ DEFINE_HOOK_FUNCTION(
|
||||
return INTERNAL_ERROR;
|
||||
}
|
||||
|
||||
return NO_SUCH_KEYLET;
|
||||
return INVALID_ARGUMENT;
|
||||
|
||||
HOOK_TEARDOWN();
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include <ripple/app/tx/impl/SetCron.h>
|
||||
#include <ripple/app/tx/impl/CronSet.h>
|
||||
#include <ripple/basics/Log.h>
|
||||
#include <ripple/ledger/View.h>
|
||||
#include <ripple/protocol/Feature.h>
|
||||
@@ -28,13 +28,13 @@
|
||||
namespace ripple {
|
||||
|
||||
TxConsequences
|
||||
SetCron::makeTxConsequences(PreflightContext const& ctx)
|
||||
CronSet::makeTxConsequences(PreflightContext const& ctx)
|
||||
{
|
||||
return TxConsequences{ctx.tx, TxConsequences::normal};
|
||||
}
|
||||
|
||||
NotTEC
|
||||
SetCron::preflight(PreflightContext const& ctx)
|
||||
CronSet::preflight(PreflightContext const& ctx)
|
||||
{
|
||||
if (!ctx.rules.enabled(featureCron))
|
||||
return temDISABLED;
|
||||
@@ -47,7 +47,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
|
||||
if (tx.getFlags() & tfCronSetMask)
|
||||
{
|
||||
JLOG(j.warn()) << "SetCron: Invalid flags set.";
|
||||
JLOG(j.warn()) << "CronSet: Invalid flags set.";
|
||||
return temINVALID_FLAG;
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
// delete operation
|
||||
if (hasDelay || hasRepeat || hasStartTime)
|
||||
{
|
||||
JLOG(j.debug()) << "SetCron: tfCronUnset flag cannot be used with "
|
||||
JLOG(j.debug()) << "CronSet: tfCronUnset flag cannot be used with "
|
||||
"DelaySeconds, RepeatCount or StartTime.";
|
||||
return temMALFORMED;
|
||||
}
|
||||
@@ -81,7 +81,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
if (!hasStartTime)
|
||||
{
|
||||
JLOG(j.debug())
|
||||
<< "SetCron: StartTime is required. Use StartTime=0 for "
|
||||
<< "CronSet: StartTime is required. Use StartTime=0 for "
|
||||
"immediate execution, or specify a future timestamp.";
|
||||
return temMALFORMED;
|
||||
}
|
||||
@@ -89,7 +89,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
if ((!hasDelay && hasRepeat) || (hasDelay && !hasRepeat))
|
||||
{
|
||||
JLOG(j.debug())
|
||||
<< "SetCron: DelaySeconds and RepeatCount must both be present "
|
||||
<< "CronSet: DelaySeconds and RepeatCount must both be present "
|
||||
"for recurring crons, or both absent for one-off crons.";
|
||||
return temMALFORMED;
|
||||
}
|
||||
@@ -101,7 +101,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
if (delay > 31536000UL /* 365 days in seconds */)
|
||||
{
|
||||
JLOG(j.debug())
|
||||
<< "SetCron: DelaySeconds was too high. (max 365 "
|
||||
<< "CronSet: DelaySeconds was too high. (max 365 "
|
||||
"days in seconds).";
|
||||
return temMALFORMED;
|
||||
}
|
||||
@@ -114,7 +114,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
if (recur == 0)
|
||||
{
|
||||
JLOG(j.debug())
|
||||
<< "SetCron: RepeatCount must be greater than 0."
|
||||
<< "CronSet: RepeatCount must be greater than 0."
|
||||
"For one-time execution, omit DelaySeconds and "
|
||||
"RepeatCount.";
|
||||
return temMALFORMED;
|
||||
@@ -122,8 +122,8 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
if (recur > 256)
|
||||
{
|
||||
JLOG(j.debug())
|
||||
<< "SetCron: RepeatCount too high. Limit is 256. Issue "
|
||||
"new SetCron to increase.";
|
||||
<< "CronSet: RepeatCount too high. Limit is 256. Issue "
|
||||
"new CronSet to increase.";
|
||||
return temMALFORMED;
|
||||
}
|
||||
}
|
||||
@@ -133,7 +133,7 @@ SetCron::preflight(PreflightContext const& ctx)
|
||||
}
|
||||
|
||||
TER
|
||||
SetCron::preclaim(PreclaimContext const& ctx)
|
||||
CronSet::preclaim(PreclaimContext const& ctx)
|
||||
{
|
||||
if (ctx.tx.isFieldPresent(sfStartTime) &&
|
||||
ctx.tx.getFieldU32(sfStartTime) != 0)
|
||||
@@ -146,7 +146,7 @@ SetCron::preclaim(PreclaimContext const& ctx)
|
||||
|
||||
if (startTime < parentCloseTime)
|
||||
{
|
||||
JLOG(ctx.j.debug()) << "SetCron: StartTime must be in the future "
|
||||
JLOG(ctx.j.debug()) << "CronSet: StartTime must be in the future "
|
||||
"(or 0 for immediate execution)";
|
||||
return tecEXPIRED;
|
||||
}
|
||||
@@ -154,7 +154,7 @@ SetCron::preclaim(PreclaimContext const& ctx)
|
||||
if (startTime > ctx.view.parentCloseTime().time_since_epoch().count() +
|
||||
365 * 24 * 60 * 60)
|
||||
{
|
||||
JLOG(ctx.j.debug()) << "SetCron: StartTime is too far in the "
|
||||
JLOG(ctx.j.debug()) << "CronSet: StartTime is too far in the "
|
||||
"future (max 365 days).";
|
||||
return tecEXPIRED;
|
||||
}
|
||||
@@ -163,7 +163,7 @@ SetCron::preclaim(PreclaimContext const& ctx)
|
||||
}
|
||||
|
||||
TER
|
||||
SetCron::doApply()
|
||||
CronSet::doApply()
|
||||
{
|
||||
auto& view = ctx_.view();
|
||||
auto const& tx = ctx_.tx;
|
||||
@@ -205,21 +205,21 @@ SetCron::doApply()
|
||||
auto sleCron = view.peek(klOld);
|
||||
if (!sleCron)
|
||||
{
|
||||
JLOG(j_.warn()) << "SetCron: Cron object didn't exist.";
|
||||
JLOG(j_.warn()) << "CronSet: Cron object didn't exist.";
|
||||
return tefBAD_LEDGER;
|
||||
}
|
||||
|
||||
if (safe_cast<LedgerEntryType>(
|
||||
sleCron->getFieldU16(sfLedgerEntryType)) != ltCRON)
|
||||
{
|
||||
JLOG(j_.warn()) << "SetCron: sfCron pointed to non-cron object!!";
|
||||
JLOG(j_.warn()) << "CronSet: sfCron pointed to non-cron object!!";
|
||||
return tefBAD_LEDGER;
|
||||
}
|
||||
|
||||
if (!view.dirRemove(
|
||||
keylet::ownerDir(id), (*sleCron)[sfOwnerNode], klOld, false))
|
||||
{
|
||||
JLOG(j_.warn()) << "SetCron: Ownerdir bad. " << id;
|
||||
JLOG(j_.warn()) << "CronSet: Ownerdir bad. " << id;
|
||||
return tefBAD_LEDGER;
|
||||
}
|
||||
|
||||
@@ -278,7 +278,7 @@ SetCron::doApply()
|
||||
}
|
||||
|
||||
XRPAmount
|
||||
SetCron::calculateBaseFee(ReadView const& view, STTx const& tx)
|
||||
CronSet::calculateBaseFee(ReadView const& view, STTx const& tx)
|
||||
{
|
||||
auto const baseFee = Transactor::calculateBaseFee(view, tx);
|
||||
|
||||
@@ -290,7 +290,7 @@ SetCron::calculateBaseFee(ReadView const& view, STTx const& tx)
|
||||
tx.isFieldPresent(sfRepeatCount) ? tx.getFieldU32(sfRepeatCount) : 0;
|
||||
|
||||
// factor a cost based on the total number of txns expected
|
||||
// for RepeatCount of 0 we have this txn (SetCron) and the
|
||||
// for RepeatCount of 0 we have this txn (CronSet) and the
|
||||
// single Cron txn (2). For a RepeatCount of 1 we have this txn,
|
||||
// the first time the cron executes, and the second time (3).
|
||||
uint32_t const additionalExpectedExecutions = 1 + repeatCount;
|
||||
@@ -17,8 +17,8 @@
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#ifndef RIPPLE_TX_SETCRON_H_INCLUDED
|
||||
#define RIPPLE_TX_SETCRON_H_INCLUDED
|
||||
#ifndef RIPPLE_TX_CRONSET_H_INCLUDED
|
||||
#define RIPPLE_TX_CRONSET_H_INCLUDED
|
||||
|
||||
#include <ripple/app/tx/impl/Transactor.h>
|
||||
#include <ripple/basics/Log.h>
|
||||
@@ -26,12 +26,12 @@
|
||||
|
||||
namespace ripple {
|
||||
|
||||
class SetCron : public Transactor
|
||||
class CronSet : public Transactor
|
||||
{
|
||||
public:
|
||||
static constexpr ConsequencesFactoryType ConsequencesFactory{Custom};
|
||||
|
||||
explicit SetCron(ApplyContext& ctx) : Transactor(ctx)
|
||||
explicit CronSet(ApplyContext& ctx) : Transactor(ctx)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
#include <ripple/app/tx/impl/CreateOffer.h>
|
||||
#include <ripple/app/tx/impl/CreateTicket.h>
|
||||
#include <ripple/app/tx/impl/Cron.h>
|
||||
#include <ripple/app/tx/impl/CronSet.h>
|
||||
#include <ripple/app/tx/impl/DeleteAccount.h>
|
||||
#include <ripple/app/tx/impl/DepositPreauth.h>
|
||||
#include <ripple/app/tx/impl/Escrow.h>
|
||||
@@ -44,7 +45,6 @@
|
||||
#include <ripple/app/tx/impl/Payment.h>
|
||||
#include <ripple/app/tx/impl/Remit.h>
|
||||
#include <ripple/app/tx/impl/SetAccount.h>
|
||||
#include <ripple/app/tx/impl/SetCron.h>
|
||||
#include <ripple/app/tx/impl/SetHook.h>
|
||||
#include <ripple/app/tx/impl/SetRegularKey.h>
|
||||
#include <ripple/app/tx/impl/SetRemarks.h>
|
||||
@@ -184,7 +184,7 @@ invoke_preflight(PreflightContext const& ctx)
|
||||
case ttURITOKEN_CANCEL_SELL_OFFER:
|
||||
return invoke_preflight_helper<URIToken>(ctx);
|
||||
case ttCRON_SET:
|
||||
return invoke_preflight_helper<SetCron>(ctx);
|
||||
return invoke_preflight_helper<CronSet>(ctx);
|
||||
case ttCRON:
|
||||
return invoke_preflight_helper<Cron>(ctx);
|
||||
default:
|
||||
@@ -313,7 +313,7 @@ invoke_preclaim(PreclaimContext const& ctx)
|
||||
case ttURITOKEN_CANCEL_SELL_OFFER:
|
||||
return invoke_preclaim<URIToken>(ctx);
|
||||
case ttCRON_SET:
|
||||
return invoke_preclaim<SetCron>(ctx);
|
||||
return invoke_preclaim<CronSet>(ctx);
|
||||
case ttCRON:
|
||||
return invoke_preclaim<Cron>(ctx);
|
||||
default:
|
||||
@@ -404,7 +404,7 @@ invoke_calculateBaseFee(ReadView const& view, STTx const& tx)
|
||||
case ttURITOKEN_CANCEL_SELL_OFFER:
|
||||
return URIToken::calculateBaseFee(view, tx);
|
||||
case ttCRON_SET:
|
||||
return SetCron::calculateBaseFee(view, tx);
|
||||
return CronSet::calculateBaseFee(view, tx);
|
||||
case ttCRON:
|
||||
return Cron::calculateBaseFee(view, tx);
|
||||
default:
|
||||
@@ -601,7 +601,7 @@ invoke_apply(ApplyContext& ctx)
|
||||
return p();
|
||||
}
|
||||
case ttCRON_SET: {
|
||||
SetCron p(ctx);
|
||||
CronSet p(ctx);
|
||||
return p();
|
||||
}
|
||||
case ttCRON: {
|
||||
|
||||
@@ -72,7 +72,7 @@ enum class LedgerNameSpace : std::uint16_t {
|
||||
URI_TOKEN = 'U',
|
||||
IMPORT_VLSEQ = 'I',
|
||||
UNL_REPORT = 'R',
|
||||
CRON = 'A',
|
||||
CRON = 'L',
|
||||
|
||||
// No longer used or supported. Left here to reserve the space
|
||||
// to avoid accidental reuse.
|
||||
|
||||
@@ -376,6 +376,8 @@ LedgerFormats::LedgerFormats()
|
||||
{sfDelaySeconds, soeREQUIRED},
|
||||
{sfRepeatCount, soeREQUIRED},
|
||||
{sfOwnerNode, soeREQUIRED},
|
||||
{sfPreviousTxnID, soeREQUIRED},
|
||||
{sfPreviousTxnLgrSeq, soeREQUIRED}
|
||||
},
|
||||
commonFields);
|
||||
|
||||
|
||||
@@ -256,6 +256,7 @@ JSS(coins);
|
||||
JSS(children);
|
||||
JSS(ctid); // in/out: Tx RPC
|
||||
JSS(cres);
|
||||
JSS(cron);
|
||||
JSS(currency_a); // out: BookChanges
|
||||
JSS(currency_b); // out: BookChanges
|
||||
JSS(currentShard); // out: NodeToShardStatus
|
||||
|
||||
@@ -506,6 +506,36 @@ doLedgerEntry(RPC::JsonContext& context)
|
||||
jvResult[jss::error] = "malformedRequest";
|
||||
}
|
||||
}
|
||||
else if (context.params.isMember(jss::cron))
|
||||
{
|
||||
expectedType = ltCRON;
|
||||
if (!context.params[jss::cron].isObject())
|
||||
{
|
||||
if (!uNodeIndex.parseHex(context.params[jss::cron].asString()))
|
||||
{
|
||||
uNodeIndex = beast::zero;
|
||||
jvResult[jss::error] = "malformedRequest";
|
||||
}
|
||||
}
|
||||
else if (
|
||||
!context.params[jss::cron].isMember(jss::owner) ||
|
||||
!context.params[jss::cron].isMember(jss::time))
|
||||
{
|
||||
jvResult[jss::error] = "malformedRequest";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto const id = parseBase58<AccountID>(
|
||||
context.params[jss::cron][jss::owner].asString());
|
||||
if (!id)
|
||||
jvResult[jss::error] = "malformedAddress";
|
||||
else
|
||||
uNodeIndex =
|
||||
keylet::cron(
|
||||
context.params[jss::cron][jss::time].asUInt(), *id)
|
||||
.key;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (context.params.isMember("params") &&
|
||||
|
||||
@@ -11147,6 +11147,7 @@ public:
|
||||
#define KEYLET_PAYCHAN 21
|
||||
#define KEYLET_EMITTED_TXN 22
|
||||
#define KEYLET_NFT_OFFER 23
|
||||
#define KEYLET_CRON 26
|
||||
#define ASSERT(x)\
|
||||
if (!(x))\
|
||||
rollback((uint32_t)#x, sizeof(#x), __LINE__);
|
||||
@@ -11209,6 +11210,9 @@ public:
|
||||
// Test min size
|
||||
ASSERT(util_keylet((uint32_t)buf, 33, KEYLET_SKIP, 0,0,0,0,0,0) == TOO_SMALL);
|
||||
|
||||
// Invalid keylet type
|
||||
ASSERT(util_keylet((uint32_t)buf, 34, 0, 0,0,0,0,0,0) == INVALID_ARGUMENT);
|
||||
ASSERT(util_keylet((uint32_t)buf, 34, 0x99999999, 0,0,0,0,0,0) == INVALID_ARGUMENT);
|
||||
|
||||
// Test one of each type
|
||||
ASSERT(34 == (e=util_keylet(buf, 34, KEYLET_HOOK,
|
||||
@@ -11651,6 +11655,17 @@ public:
|
||||
0,0
|
||||
)));
|
||||
|
||||
ASSERT(34 == (e=util_keylet(buf, 34, KEYLET_CRON, SBUF(a), 1, 0, 0, 0)));
|
||||
{
|
||||
uint8_t ans[] =
|
||||
{
|
||||
0x00U,0x41U,0xF7U,0xB6U,0x45U,0x43U,0x61U,0x87U,0xCCU,0x61U,
|
||||
0x00U,0x00U,0x00U,0x01U,0x0AU,0x45U,0x80U,0x75U,0x7CU,0xDAU,
|
||||
0xD9U,0x16U,0x7EU,0xEEU,0xC1U,0x3CU,0x6CU,0x15U,0xD5U,0x17U,
|
||||
0xE2U,0x72U,0x9EU,0xC8
|
||||
};
|
||||
ASSERT_KL_EQ(ans);
|
||||
}
|
||||
accept(0,0,0);
|
||||
}
|
||||
)[test.hook]"];
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1839,6 +1839,88 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testLedgerEntryCron()
|
||||
{
|
||||
testcase("ledger_entry Request Cron");
|
||||
using namespace test::jtx;
|
||||
|
||||
Env env{*this};
|
||||
|
||||
Account const alice{"alice"};
|
||||
env.fund(XRP(10000), alice);
|
||||
env.close();
|
||||
|
||||
auto const startTime =
|
||||
env.current()->parentCloseTime().time_since_epoch().count() + 100;
|
||||
env(cron::set(alice),
|
||||
cron::startTime(startTime),
|
||||
cron::delay(100),
|
||||
cron::repeat(200),
|
||||
fee(XRP(1)),
|
||||
ter(tesSUCCESS));
|
||||
env.close();
|
||||
|
||||
std::string const ledgerHash{to_string(env.closed()->info().hash)};
|
||||
|
||||
uint256 const cronIndex{keylet::cron(startTime, alice).key};
|
||||
{
|
||||
// Request the cron using its index.
|
||||
Json::Value jvParams;
|
||||
jvParams[jss::cron] = to_string(cronIndex);
|
||||
jvParams[jss::ledger_hash] = ledgerHash;
|
||||
Json::Value const jrr = env.rpc(
|
||||
"json", "ledger_entry", to_string(jvParams))[jss::result];
|
||||
BEAST_EXPECT(jrr[jss::node][sfOwner.jsonName] == alice.human());
|
||||
BEAST_EXPECT(jrr[jss::node][sfStartTime.jsonName] == startTime);
|
||||
BEAST_EXPECT(jrr[jss::node][sfDelaySeconds.jsonName] == 100);
|
||||
BEAST_EXPECT(jrr[jss::node][sfRepeatCount.jsonName] == 200);
|
||||
}
|
||||
{
|
||||
// Request the cron using its owner and time.
|
||||
Json::Value jvParams;
|
||||
jvParams[jss::cron] = Json::objectValue;
|
||||
jvParams[jss::cron][jss::owner] = alice.human();
|
||||
jvParams[jss::cron][jss::time] = startTime;
|
||||
jvParams[jss::ledger_hash] = ledgerHash;
|
||||
Json::Value const jrr = env.rpc(
|
||||
"json", "ledger_entry", to_string(jvParams))[jss::result];
|
||||
BEAST_EXPECT(jrr[jss::node][sfOwner.jsonName] == alice.human());
|
||||
BEAST_EXPECT(jrr[jss::node][sfStartTime.jsonName] == startTime);
|
||||
BEAST_EXPECT(jrr[jss::node][sfDelaySeconds.jsonName] == 100);
|
||||
BEAST_EXPECT(jrr[jss::node][sfRepeatCount.jsonName] == 200);
|
||||
}
|
||||
{
|
||||
// Malformed uritoken object. Missing owner member.
|
||||
Json::Value jvParams;
|
||||
jvParams[jss::cron] = Json::objectValue;
|
||||
jvParams[jss::cron][jss::time] = startTime;
|
||||
jvParams[jss::ledger_hash] = ledgerHash;
|
||||
Json::Value const jrr = env.rpc(
|
||||
"json", "ledger_entry", to_string(jvParams))[jss::result];
|
||||
checkErrorValue(jrr, "malformedRequest", "");
|
||||
}
|
||||
{
|
||||
// Malformed uritoken object. Missing time member.
|
||||
Json::Value jvParams;
|
||||
jvParams[jss::cron] = Json::objectValue;
|
||||
jvParams[jss::cron][jss::owner] = alice.human();
|
||||
jvParams[jss::ledger_hash] = ledgerHash;
|
||||
Json::Value const jrr = env.rpc(
|
||||
"json", "ledger_entry", to_string(jvParams))[jss::result];
|
||||
checkErrorValue(jrr, "malformedRequest", "");
|
||||
}
|
||||
{
|
||||
// Request an index that is not a uritoken.
|
||||
Json::Value jvParams;
|
||||
jvParams[jss::cron] = ledgerHash;
|
||||
jvParams[jss::ledger_hash] = ledgerHash;
|
||||
Json::Value const jrr = env.rpc(
|
||||
"json", "ledger_entry", to_string(jvParams))[jss::result];
|
||||
checkErrorValue(jrr, "entryNotFound", "");
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testLedgerEntryUnknownOption()
|
||||
{
|
||||
@@ -2365,6 +2447,7 @@ public:
|
||||
testLedgerEntryTicket();
|
||||
testLedgerEntryURIToken();
|
||||
testLedgerEntryImportVLSeq();
|
||||
testLedgerEntryCron();
|
||||
testLedgerEntryUnknownOption();
|
||||
testLookupLedger();
|
||||
testNoQueue();
|
||||
|
||||
Reference in New Issue
Block a user