From 5b5f4e7c62129df428bf9c11cc9eb504d56aab9e Mon Sep 17 00:00:00 2001 From: mDuo13 Date: Tue, 21 Aug 2018 16:40:58 -0700 Subject: [PATCH] Start troubleshooting category --- .../server-wont-start.md | 147 ++++++++++++++++++ dactyl-config.yml | 25 ++- 2 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 content/tutorials/troubleshoot-the-rippled-server/server-wont-start.md diff --git a/content/tutorials/troubleshoot-the-rippled-server/server-wont-start.md b/content/tutorials/troubleshoot-the-rippled-server/server-wont-start.md new file mode 100644 index 0000000000..1ae4c576d9 --- /dev/null +++ b/content/tutorials/troubleshoot-the-rippled-server/server-wont-start.md @@ -0,0 +1,147 @@ +# rippled Server Won't Start (Troubleshooting) + +This page explains possible reasons the `rippled` server does not start successfully, and how to fix them. + +These instructions assume you have [installed `rippled`](install-rippled.html) on a supported platform. + + +## File Descriptors Limit + +On some Linux variants, you may get an error message such as the following when trying to run `rippled`: + +```text +WARNING: There are only 1024 file descriptors (soft limit) available, which +limit the number of simultaneous connections. +``` + +This occurs because the system has a security limit on the number of files a single process may open, but the limit is set too low for `rippled`. To fix the problem, **root access is required**. Increase the number of files `rippled` is allowed to open with the following steps: + +1. Add the following lines to the end of your `/etc/security/limits.conf` file: + + * soft nofile 5200 + * hard nofile 10240 + +2. Check that the number of files that can be opened is now `10240`: + + ulimit -Hn + + The command prints the hard limit on the number of open files, which should be 10240. + +3. Try starting `rippled` again. + + systemctl start rippled + +4. If `rippled` still does not start, open `/etc/sysctl.conf` and add the following line to the bottom of the file: + + fs.file-max = 65536 + + +## Failed to open /etc/opt/ripple/rippled.cfg + +If `rippled` crashes on startup with an error such as the following, it means that `rippled` cannot read its config file: + +```text +Loading: "/etc/opt/ripple/rippled.cfg" +Failed to open '"/etc/opt/ripple/rippled.cfg"'. +Terminating thread rippled: main: unhandled St13runtime_error 'Can not create "/var/opt/ripple"' +Aborted (core dumped) +``` + +Possible solutions: + +- Check that `/etc/opt/ripple/rippled.cfg` exists and the `rippled` user has read permissions to the file. (Assuming you use the `rippled` user to run the `rippled` process, and you want to use the default location for the config file.) + +- Create a config file that can be read by the `rippled` user to `$HOME/.config/ripple/rippled.cfg` (where `$HOME` points to the `rippled` user's home directory). + + **Tip:** The `rippled` repository contains [an example `rippled.cfg` file](https://github.com/ripple/rippled/blob/master/cfg/rippled-example.cfg) which is provided as the default config when you do an RPM installation. If you do not have the file, you can copy it from there. + +- Specify the path to your preferred config file using the `--conf` [commandline option](commandline-usage.html). + +## Failed to open validators file + +If `rippled` crashes on startup with an error such as the following, it means it can read its primary config file, but that config file specifies a separate validators config file (typically named `validators.txt`), which `rippled` cannot read. + +```text +Loading: "/home/rippled/.config/ripple/rippled.cfg" +Terminating thread rippled: main: unhandled St13runtime_error 'The file specified in [validators_file] does not exist: /home/rippled/.config/ripple/validators.txt' +Aborted (core dumped) +``` + +Possible solutions: + +- Check that the `[validators.txt]` file exists and the `rippled` user has permissions to read it. + + **Tip:** The `rippled` repository contains [an example `validators.txt` file](https://github.com/ripple/rippled/blob/master/cfg/validators-example.txt) which is provided as the default config when you do an RPM installation. If you do not have the file, you can copy it from there. + +- Edit your `rippled.cfg` file and modify the `[validators_file]` setting to have the correct path to your `validators.txt` (or equivalent) file. Check for extra whitespace before or after the filename. + +- Edit your `rippled.cfg` file and remove the `[validators_file]` setting. Add validator settings directly to your `rippled.cfg` file. For example: + + [validator_list_sites] + https://vl.ripple.com + + [validator_list_keys] + ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734 + + +## Cannot create database path + +If `rippled` crashes on startup with an error such as the following, it means the server does not have write permissions to the `[database_path]` from its config file. + +```text +Loading: "/home/rippled/.config/ripple/rippled.cfg" +Terminating thread rippled: main: unhandled St13runtime_error 'Can not create "/var/lib/rippled/db"' +Aborted (core dumped) +``` + +The paths to the configuration file (`/home/rippled/.config/ripple/rippled.cfg`) and the database path (`/var/lib/rippled/db`) may vary depending on your system. + +Possible solutions: + +- Run `rippled` as a different user that has write permissions to the database path printed in the error message. + +- Edit your `rippled.cfg` file and change the `[database_path]` setting to use a path that the `rippled` user has write permissions to. + +- Grant the `rippled` user write permissions to the configured database path. + + +## State DB Error + +The following error can occur if the `rippled` server's state database is corrupted (possibly as the result of being shutdown unexpectedly): + +```text +2018-Aug-21 23:06:38.675117810 SHAMapStore:ERR state db error: + writableDbExists false archiveDbExists false + writableDb '/var/lib/rippled/db/rocksdb/rippledb.11a9' archiveDb '/var/lib/rippled/db/rocksdb/rippledb.2d73' + +To resume operation, make backups of and remove the files matching /var/lib/rippled/db/state* and contents of the directory /var/lib/rippled/db/rocksdb + +Terminating thread rippled: main: unhandled St13runtime_error 'state db error' +``` + +The easiest way to fix this problem is to delete the databases entirely. You may want to back them up elsewhere instead. For example: + +```sh +mv /var/lib/rippled/db /var/lib/rippled/db-bak +``` + +Or, if you are sure you don't need the databases: + +```sh +rm -r /var/lib/rippled/db +``` + +**Tip:** It is generally safe to delete the `rippled` databases, because any individual server can re-download ledger history from other servers in the XRP Ledger network. If you are using [clustering](clustering.html), be sure your servers each have a unique `[node_seed]` configured first; if not, servers may not be recognized as part of the cluster after you restart them. + + +## Online Delete is Less Than Ledger History + +An error message such as the following indicates that the `rippled.cfg` file has contradictory values for `[ledger_history]` and `online_delete`. + +```text +Terminating thread rippled: main: unhandled St13runtime_error 'online_delete must not be less than ledger_history (currently 3000) +``` + +The `[ledger_history]` setting represents how many ledgers of history the server should seek to back-fill. The `online_delete` field (in the `[node_db]` stanza) indicates how many ledgers of history to keep when dropping older history. The `online_delete` value must be equal or larger than `[ledger_history]` to prevent the server from deleting historical ledgers that it is also trying to download. + +To fix the problem, edit the `rippled.cfg` file and change or remove either the `[ledger_history]` or `online_delete` options. diff --git a/dactyl-config.yml b/dactyl-config.yml index 4aba0408e9..ce988d7f96 100644 --- a/dactyl-config.yml +++ b/dactyl-config.yml @@ -866,7 +866,7 @@ pages: funnel: Docs doc_type: Tutorials category: Manage the rippled Server - blurb: Build, install, and run the rippled server. Configure it to your use case and troubleshoot problems. + blurb: Build, install, configure, and run the rippled server. template: template-landing-children.html targets: - local @@ -943,8 +943,27 @@ pages: targets: - local - # TODO: troubleshoot rippled, - # run a validator, run in stand-alone mode, set up private peers + # TODO: run a validator, run in stand-alone mode, set up private peers + + - name: Troubleshoot the rippled Server + html: troubleshoot-the-rippled-server.html + funnel: Docs + doc_type: Tutorials + category: Troubleshoot the rippled Server + blurb: Troubleshoot all kinds of problems with the rippled server. + template: template-landing-children.html + targets: + - local + + - md: tutorials/troubleshoot-the-rippled-server/server-wont-start.md + html: server-wont-start.html + funnel: Docs + doc_type: Tutorials + category: Troubleshoot the rippled Server + targets: + - local + + # TODO: more troubleshooting rippled articles # References -------------------------------------------------------------------