feat: add get range feat to export tool (#2131)

tools/snapshot/go.mod

@@ -1,8 +1,8 @@
 module xrplf/clio/clio_snapshot
 
-go 1.22
+go 1.23.0
 
-toolchain go1.22.11
+toolchain go1.23.5
 
 require (
 	github.com/golang/mock v1.6.0
@@ -13,12 +13,19 @@ require (
 	google.golang.org/protobuf v1.36.3
 )
 
+require (
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+)
+
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gocql/gocql v1.7.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

tools/snapshot/go.sum

@@ -1,23 +1,41 @@
+github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
+github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=
+github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -37,8 +55,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -46,13 +64,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -67,5 +85,7 @@ google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1B
 google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/snapshot/internal/parse_args/parse_args.go

@@ -16,6 +16,11 @@ type Args struct {
 	WsServer   string
 	ServerMode bool
 	ShowRange  bool
+	GetRange   bool
+	// for authenticating DB
+	Host     string
+	Username string
+	Password string
 }
 
 func Parse() (*Args, error) {
@@ -31,33 +36,56 @@ func Parse() (*Args, error) {
 	wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address")
 	serverMode := fs.BoolP("server", "s", false, "Start server mode")
 	showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot")
+	getRange := fs.Bool("getRange", false, "Get the available DB range of the host")
+
+	host := fs.String("host", "", "Database host (e.g., 127.0.0.1)")
+	username := fs.String("username", "", "Database username")
+	password := fs.String("password", "", "Database password")
+
 	fs.Parse(os.Args[1:])
 
 	if *serverMode && *exportMode != "" {
-		return nil, fmt.Errorf("Invalid usage: --server and --export cannot be used at the same time.")
+		return nil, fmt.Errorf("invalid usage: --server and --export cannot be used at the same time")
 	}
 
 	if *serverMode {
 		if *grpcServer == "" || *wsServer == "" || *path == "" {
-			return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.")
+			return nil, fmt.Errorf("invalid usage: --grpc_server and --ws_server and --path are required for server mode")
 		}
 	} else if *exportMode != "" {
 		if *exportMode == "full" || *exportMode == "delta" {
 			if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
-				return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.")
+				return nil, fmt.Errorf("invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
 			}
 		} else {
-			return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
+			return nil, fmt.Errorf("invalid usage: Invalid export mode. Use 'full' or 'delta'")
 		}
 	} else if *showRange {
 		if *path == "" {
-			return nil, fmt.Errorf("Invalid usage: --path is required for show range.")
+			return nil, fmt.Errorf("invalid usage: --path is required for show range")
+		}
+	} else if *getRange {
+		if *host == "" || *username == "" || *password == "" {
+			return nil, fmt.Errorf("invalid usage: --host, --username, and --password are required for getRange")
 		}
 	} else {
-		return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.")
+		return nil, fmt.Errorf("invalid usage: --export or --server or --range flag is required")
 	}
 
-	return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil
+	return &Args{
+		ExportMode: *exportMode,
+		StartSeq:   *seq,
+		EndSeq:     *endSeq,
+		Path:       *path,
+		GrpcServer: *grpcServer,
+		WsServer:   *wsServer,
+		ServerMode: *serverMode,
+		ShowRange:  *showRange,
+		GetRange:   *getRange,
+		Host:       *host,
+		Username:   *username,
+		Password:   *password,
+	}, nil
 }
 
 func PrintUsage() {

tools/snapshot/internal/parse_args/parse_args_test.go

@@ -35,14 +35,14 @@ func TestParse(t *testing.T) {
 			args:       []string{"cmd", "--export=delta", "--start_seq=1"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.",
+			errMessage: "invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
 		},
 		{
 			name:       "Invalid export mode",
 			args:       []string{"cmd", "--export=invalid"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: Invalid export mode. Use 'full' or 'delta'.",
+			errMessage: "invalid usage: Invalid export mode. Use 'full' or 'delta'",
 		},
 		{
 			name: "Server mode with default grpc server flags",
@@ -62,21 +62,21 @@ func TestParse(t *testing.T) {
 			args:       []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.",
+			errMessage: "invalid usage: --grpc_server and --ws_server and --path are required for server mode",
 		},
 		{
 			name:       "Server and export mode together (error)",
 			args:       []string{"cmd", "--server", "--export=full"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
+			errMessage: "invalid usage: --server and --export cannot be used at the same time",
 		},
 		{
 			name:       "Show range without path",
 			args:       []string{"cmd", "--range"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: --path is required for show range.",
+			errMessage: "invalid usage: --path is required for show range",
 		},
 		{
 			name: "Show range",

tools/snapshot/internal/util/db.go

@@ -0,0 +1,36 @@
+package util
+
+import (
+	"github.com/gocql/gocql"
+)
+
+type Range struct {
+	FirstLedgerIdx  uint64
+	LatestLedgerIdx uint64
+}
+
+func GetLedgerRange(cluster *gocql.ClusterConfig) (*Range, error) {
+	session, err := cluster.CreateSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	var (
+		firstLedgerIdx  uint64
+		latestLedgerIdx uint64
+	)
+
+	if err := session.Query("SELECT sequence FROM ledger_range WHERE is_latest = ?", false).Scan(&firstLedgerIdx); err != nil {
+		return nil, err
+	}
+
+	if err := session.Query("SELECT sequence FROM ledger_range WHERE is_latest = ?", true).Scan(&latestLedgerIdx); err != nil {
+		return nil, err
+	}
+
+	return &Range{
+		FirstLedgerIdx:  firstLedgerIdx,
+		LatestLedgerIdx: latestLedgerIdx,
+	}, nil
+}

tools/snapshot/snapshot.go

@@ -1,12 +1,17 @@
 package main
 
 import (
+	"fmt"
 	"log"
+	"strings"
 
 	"xrplf/clio/clio_snapshot/internal/export"
 	"xrplf/clio/clio_snapshot/internal/ledgers"
 	"xrplf/clio/clio_snapshot/internal/parse_args"
 	"xrplf/clio/clio_snapshot/internal/server"
+	"xrplf/clio/clio_snapshot/internal/util"
+
+	"github.com/gocql/gocql"
 )
 
 func main() {
@@ -33,5 +38,20 @@ func main() {
 		} else {
 			log.Fatalf("Failed to get snapshot range: %v", err)
 		}
+	} else if args.GetRange {
+		hosts := strings.Split(args.Host, ",")
+		cluster := gocql.NewCluster(hosts...)
+
+		cluster.Authenticator = gocql.PasswordAuthenticator{
+			Username: args.Username,
+			Password: args.Password,
+		}
+
+		dbRange, err := util.GetLedgerRange(cluster)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("Range: %d -> %d\n", dbRange.FirstLedgerIdx, dbRange.LatestLedgerIdx)
 	}
 }