diff --git a/tools/snapshot/go.mod b/tools/snapshot/go.mod index cbc1f207..acbbc56f 100644 --- a/tools/snapshot/go.mod +++ b/tools/snapshot/go.mod @@ -1,8 +1,8 @@ module xrplf/clio/clio_snapshot -go 1.22 +go 1.23.0 -toolchain go1.22.11 +toolchain go1.23.5 require ( github.com/golang/mock v1.6.0 @@ -13,12 +13,19 @@ require ( google.golang.org/protobuf v1.36.3 ) +require ( + github.com/golang/snappy v0.0.3 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + gopkg.in/inf.v0 v0.9.1 // indirect +) + require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gocql/gocql v1.7.0 github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/snapshot/go.sum b/tools/snapshot/go.sum index 4907d7e3..f889c7a2 100644 --- a/tools/snapshot/go.sum +++ b/tools/snapshot/go.sum @@ -1,23 +1,41 @@ +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -37,8 +55,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -46,13 +64,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -67,5 +85,7 @@ google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1B google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/snapshot/internal/parse_args/parse_args.go b/tools/snapshot/internal/parse_args/parse_args.go index a474aee2..d41579dc 100644 --- a/tools/snapshot/internal/parse_args/parse_args.go +++ b/tools/snapshot/internal/parse_args/parse_args.go @@ -16,6 +16,11 @@ type Args struct { WsServer string ServerMode bool ShowRange bool + GetRange bool + // for authenticating DB + Host string + Username string + Password string } func Parse() (*Args, error) { @@ -31,33 +36,56 @@ func Parse() (*Args, error) { wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address") serverMode := fs.BoolP("server", "s", false, "Start server mode") showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot") + getRange := fs.Bool("getRange", false, "Get the available DB range of the host") + + host := fs.String("host", "", "Database host (e.g., 127.0.0.1)") + username := fs.String("username", "", "Database username") + password := fs.String("password", "", "Database password") + fs.Parse(os.Args[1:]) if *serverMode && *exportMode != "" { - return nil, fmt.Errorf("Invalid usage: --server and --export cannot be used at the same time.") + return nil, fmt.Errorf("invalid usage: --server and --export cannot be used at the same time") } if *serverMode { if *grpcServer == "" || *wsServer == "" || *path == "" { - return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.") + return nil, fmt.Errorf("invalid usage: --grpc_server and --ws_server and --path are required for server mode") } } else if *exportMode != "" { if *exportMode == "full" || *exportMode == "delta" { if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" { - return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.") + return nil, fmt.Errorf("invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export") } } else { - return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.") + return nil, fmt.Errorf("invalid usage: Invalid export mode. Use 'full' or 'delta'") } } else if *showRange { if *path == "" { - return nil, fmt.Errorf("Invalid usage: --path is required for show range.") + return nil, fmt.Errorf("invalid usage: --path is required for show range") + } + } else if *getRange { + if *host == "" || *username == "" || *password == "" { + return nil, fmt.Errorf("invalid usage: --host, --username, and --password are required for getRange") } } else { - return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.") + return nil, fmt.Errorf("invalid usage: --export or --server or --range flag is required") } - return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil + return &Args{ + ExportMode: *exportMode, + StartSeq: *seq, + EndSeq: *endSeq, + Path: *path, + GrpcServer: *grpcServer, + WsServer: *wsServer, + ServerMode: *serverMode, + ShowRange: *showRange, + GetRange: *getRange, + Host: *host, + Username: *username, + Password: *password, + }, nil } func PrintUsage() { diff --git a/tools/snapshot/internal/parse_args/parse_args_test.go b/tools/snapshot/internal/parse_args/parse_args_test.go index c047e998..c6dcd609 100644 --- a/tools/snapshot/internal/parse_args/parse_args_test.go +++ b/tools/snapshot/internal/parse_args/parse_args_test.go @@ -35,14 +35,14 @@ func TestParse(t *testing.T) { args: []string{"cmd", "--export=delta", "--start_seq=1"}, want: nil, expectErr: true, - errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.", + errMessage: "invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export", }, { name: "Invalid export mode", args: []string{"cmd", "--export=invalid"}, want: nil, expectErr: true, - errMessage: "Invalid usage: Invalid export mode. Use 'full' or 'delta'.", + errMessage: "invalid usage: Invalid export mode. Use 'full' or 'delta'", }, { name: "Server mode with default grpc server flags", @@ -62,21 +62,21 @@ func TestParse(t *testing.T) { args: []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"}, want: nil, expectErr: true, - errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.", + errMessage: "invalid usage: --grpc_server and --ws_server and --path are required for server mode", }, { name: "Server and export mode together (error)", args: []string{"cmd", "--server", "--export=full"}, want: nil, expectErr: true, - errMessage: "Invalid usage: --server and --export cannot be used at the same time.", + errMessage: "invalid usage: --server and --export cannot be used at the same time", }, { name: "Show range without path", args: []string{"cmd", "--range"}, want: nil, expectErr: true, - errMessage: "Invalid usage: --path is required for show range.", + errMessage: "invalid usage: --path is required for show range", }, { name: "Show range", diff --git a/tools/snapshot/internal/util/db.go b/tools/snapshot/internal/util/db.go new file mode 100644 index 00000000..902b87b0 --- /dev/null +++ b/tools/snapshot/internal/util/db.go @@ -0,0 +1,36 @@ +package util + +import ( + "github.com/gocql/gocql" +) + +type Range struct { + FirstLedgerIdx uint64 + LatestLedgerIdx uint64 +} + +func GetLedgerRange(cluster *gocql.ClusterConfig) (*Range, error) { + session, err := cluster.CreateSession() + if err != nil { + return nil, err + } + defer session.Close() + + var ( + firstLedgerIdx uint64 + latestLedgerIdx uint64 + ) + + if err := session.Query("SELECT sequence FROM ledger_range WHERE is_latest = ?", false).Scan(&firstLedgerIdx); err != nil { + return nil, err + } + + if err := session.Query("SELECT sequence FROM ledger_range WHERE is_latest = ?", true).Scan(&latestLedgerIdx); err != nil { + return nil, err + } + + return &Range{ + FirstLedgerIdx: firstLedgerIdx, + LatestLedgerIdx: latestLedgerIdx, + }, nil +} diff --git a/tools/snapshot/snapshot.go b/tools/snapshot/snapshot.go index 7a159a7b..98e93403 100644 --- a/tools/snapshot/snapshot.go +++ b/tools/snapshot/snapshot.go @@ -1,12 +1,17 @@ package main import ( + "fmt" "log" + "strings" "xrplf/clio/clio_snapshot/internal/export" "xrplf/clio/clio_snapshot/internal/ledgers" "xrplf/clio/clio_snapshot/internal/parse_args" "xrplf/clio/clio_snapshot/internal/server" + "xrplf/clio/clio_snapshot/internal/util" + + "github.com/gocql/gocql" ) func main() { @@ -33,5 +38,20 @@ func main() { } else { log.Fatalf("Failed to get snapshot range: %v", err) } + } else if args.GetRange { + hosts := strings.Split(args.Host, ",") + cluster := gocql.NewCluster(hosts...) + + cluster.Authenticator = gocql.PasswordAuthenticator{ + Username: args.Username, + Password: args.Password, + } + + dbRange, err := util.GetLedgerRange(cluster) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Range: %d -> %d\n", dbRange.FirstLedgerIdx, dbRange.LatestLedgerIdx) } }