Dactyl improvements - default filters, toc by default, exit on preprocessor error by default, skip preprocessor option

2025-11-20 19:55:54 +00:00 · 2016-10-11 22:46:00 -07:00
parent 596d6ea5e3
commit bb97c7eef2
2 changed files with 54 additions and 56 deletions
--- a/tool/dactyl-config.yml
+++ b/tool/dactyl-config.yml
@@ -22,25 +22,20 @@ temporary_files_path: /tmp/

 prince_executable: prince

+default_filters:
+    - multicode_tabs
+    - standardize_header_ids
+    - buttonize
+    - callouts
+    - badges
+
 targets:
    # First member is the default that gets built when target not specified
    -   name: local
        display_name: Ripple Developer Portal
-        filters:
-            - multicode_tabs
-            - standardize_header_ids
-            - buttonize
-            - callouts
-            - badges

    -   name: ripple.com
        display_name: Ripple Developer Center
-        filters:
-            - multicode_tabs
-            - standardize_header_ids
-            - buttonize
-            - callouts
-            - badges
        template: template-contentwithtoc.html
        image_subs:
            "img/funds_flow_diagram.png": https://ripple.com/wp-content/uploads/2016/03/funds_flow_diagram.png
@@ -62,8 +57,7 @@ targets:
            "img/noripple-06.png": https://ripple.com/wp-content/uploads/2015/01/noripple-06.png

    -   name: rippled-setup
-        filters:
-            - standardize_header_ids
+        display_name: rippled Setup Guide

 pages:
 # Intro pages is not directly replicated on ripple.com at this time
@@ -85,7 +79,6 @@ pages:
        filters:
            - remove_doctoc
            - add_version
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -95,7 +88,6 @@ pages:
        html: reference-rippled.html
        md: reference-rippled.md
        ripple.com: https://ripple.com/build/rippled-apis/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -105,7 +97,6 @@ pages:
        html: reference-transaction-format.html
        md: reference-transaction-format.md
        ripple.com: https://ripple.com/build/transactions/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -115,7 +106,6 @@ pages:
        html: reference-ledger-format.html
        md: reference-ledger-format.md
        ripple.com: https://ripple.com/build/ledger-format/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -125,7 +115,6 @@ pages:
        html: reference-data-api.html
        md: reference-data-api.md
        ripple.com: https://ripple.com/build/data-api-v2/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -136,7 +125,6 @@ pages:
        html: tutorial-multisign.html
        md: tutorial-multisign.md
        ripple.com: https://ripple.com/build/how-to-multi-sign/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -147,7 +135,6 @@ pages:
        html: concept-issuing-and-operational-addresses.html
        md: concept-issuing-and-operational-addresses.md
        ripple.com: https://ripple.com/build/issuing-operational-addresses/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -157,7 +144,6 @@ pages:
        html: tutorial-reliable-transaction-submission.html
        md: tutorial-reliable-transaction-submission.md
        ripple.com: https://ripple.com/build/reliable-transaction-submission/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -167,7 +153,6 @@ pages:
        html: tutorial-rippleapi-beginners-guide.html
        md: tutorial-rippleapi-beginners-guide.md
        ripple.com: https://ripple.com/build/rippleapi-beginners-guide/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -177,7 +162,6 @@ pages:
        html: tutorial-rippled-setup.html
        md: tutorial-rippled-setup.md
        ripple.com: https://ripple.com/build/rippled-setup/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -189,7 +173,6 @@ pages:
        html: tutorial-gateway-guide.html
        md: tutorial-gateway-guide.md
        ripple.com: https://ripple.com/build/gateway-guide/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -200,7 +183,6 @@ pages:
        html: concept-amendments.html
        md: concept-amendments.md
        ripple.com: https://ripple.com/build/amendments/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -210,7 +192,6 @@ pages:
        html: concept-fee-voting.html
        md: concept-fee-voting.md
        ripple.com: https://ripple.com/build/fee-voting/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -220,7 +201,6 @@ pages:
        html: concept-fees.html
        md: concept-fees.md
        ripple.com: https://ripple.com/build/fees-disambiguation/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -230,7 +210,6 @@ pages:
        html: concept-freeze.html
        md: concept-freeze.md
        ripple.com: https://ripple.com/build/freeze/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -240,7 +219,6 @@ pages:
        html: concept-paths.html
        md: concept-paths.md
        ripple.com: https://ripple.com/build/paths/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -250,7 +228,6 @@ pages:
        html: concept-reserves.html
        md: concept-reserves.md
        ripple.com: https://ripple.com/build/reserves/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -260,7 +237,6 @@ pages:
        html: concept-stand-alone-mode.html
        md: concept-stand-alone-mode.md
        ripple.com: https://ripple.com/build/stand-alone-mode/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -270,7 +246,6 @@ pages:
        html: concept-transaction-cost.html
        md: concept-transaction-cost.md
        ripple.com: https://ripple.com/build/transaction-cost/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -280,7 +255,6 @@ pages:
        html: concept-transfer-fees.html
        md: concept-transfer-fees.md
        ripple.com: https://ripple.com/build/transfer-fees/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -290,7 +264,6 @@ pages:
        html: concept-noripple.html
        md: concept-noripple.md
        ripple.com: https://ripple.com/build/understanding-the-noripple-flag/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -300,7 +273,6 @@ pages:
        html: gb-2015-06.html
        md: gb-2015-06.md
        ripple.com: https://ripple.com/build/gateway-guide/gb-2015-06-corrections-autobridging/
-        sidebar: toc
        targets:
            - local
            - ripple.com
@@ -310,7 +282,6 @@ pages:
        html: gb-2015-05.html
        md: gb-2015-05.md
        ripple.com: https://ripple.com/build/gateway-guide/gb-2015-05-historical-ledger-query-migration/
-        sidebar: toc
        targets:
            - local
            - ripple.com
--- a/tool/dactyl_build.py
+++ b/tool/dactyl_build.py
@@ -14,6 +14,7 @@ import re
 import yaml
 import argparse
 import logging
+import traceback

 # Necessary to copy static files to the output dir
 from distutils.dir_util import copy_tree
@@ -67,6 +68,8 @@ def load_config(config_file=DEFAULT_CONFIG_FILE):
        assert(config["content_static_path"])
        if "prince_executable" not in config or not config["prince_executable"]:
            config["prince_executable"] = "prince" # A reasonable default
+        if "default_filters" not in config:
+            config["default_filters"] = []

        # Warn if any pages aren't part of a target
        for page in config["pages"]:
@@ -77,12 +80,12 @@ def load_config(config_file=DEFAULT_CONFIG_FILE):
                else:
                    logging.warn("Page %s is not part of any targets." % page)
            if "md" in page and "name" not in page:
-                logging.info("Guessing page name for page %s" % page)
+                logging.debug("Guessing page name for page %s" % page)
                page_path = os.path.join(config["content_path"], page["md"])
                page["name"] = guess_title_from_md_file(page_path)

        # Figure out which filters we need and import them
-        filternames = set()
+        filternames = set(config["default_filters"])
        for target in config["targets"]:
            if "filters" in target:
                filternames.update(target["filters"])
@@ -175,7 +178,6 @@ def make_adhoc_target(inpages, no_cover):
    t = {
        "name": ADHOC_TARGET,
        "display_name": "(Untitled)",
-        "sidebar": "toc" # should probably make this default anyway?
    }

    if not no_cover:
@@ -241,7 +243,7 @@ def guess_title_from_md_file(filepath):
    return os.path.basename(filepath)

 def get_filters_for_page(page, target=None):
-    ffp = set()
+    ffp = set(config["default_filters"])
    target = get_target(target)
    if "filters" in target:
        ffp.update(target["filters"])
@@ -249,7 +251,7 @@ def get_filters_for_page(page, target=None):
        ffp.update(page["filters"])
    return ffp

-def parse_markdown(page, target=None, pages=None):
+def parse_markdown(page, target=None, pages=None, bypass_errors=False):
    """Take a markdown string and output HTML for that content"""
    target = get_target(target)
    logging.info("Preparing page %s" % page["name"])
@@ -260,7 +262,8 @@ def parse_markdown(page, target=None, pages=None):
    # We'll apply these filters to the page
    page_filters = get_filters_for_page(page, target)

-    md = get_markdown_for_page(page["md"], pp_env=pp_env, target=target)
+    md = get_markdown_for_page(page["md"], pp_env=pp_env, target=target,
+                            bypass_errors=bypass_errors, currentpage=page)

    # Apply markdown-based filters here
    for filter_name in page_filters:
@@ -385,21 +388,31 @@ def get_categories(pages):
    return categories


-def read_markdown_local(filename, pp_env, target=None):
+def read_markdown_local(filename, pp_env, target=None, bypass_errors=False, currentpage={}):
    """Read in a markdown file and pre-process any templating lang in it,
       returning the parsed contents."""
    target = get_target(target)
    pages = get_pages(target)
    logging.info("reading markdown from file: %s" % filename)
-    try:
-        md_raw = pp_env.get_template(filename)
-        md_out = md_raw.render(target=target, pages=pages)
-    except TemplateError:
-        logging.warn("Error pre-processing page %s; trying to load it raw"
-                     % filename)
+
+    if config["skip_preprocessor"]:
        fpath = pp_env.loader.searchpath[0]
        with open(os.path.join(fpath,filename), "r") as f:
            md_out = f.read()
+    else:
+        try:
+            md_raw = pp_env.get_template(filename)
+            md_out = md_raw.render(target=target, pages=pages, currentpage=currentpage)
+        except TemplateError as e:
+            traceback.print_tb(e.__traceback__)
+            if bypass_errors:
+                logging.warn("Error pre-processing page %s; trying to load it raw"
+                             % filename)
+                fpath = pp_env.loader.searchpath[0]
+                with open(os.path.join(fpath,filename), "r") as f:
+                    md_out = f.read()
+            else:
+                exit("Error pre-processing page %s: %s" % (filename, e))
    return md_out


@@ -412,13 +425,21 @@ def read_markdown_remote(url):
        raise requests.RequestException("Status code for page was not 200")


-def get_markdown_for_page(md_where, pp_env=None, target=None):
+def get_markdown_for_page(md_where, pp_env=None, target=None, bypass_errors=False, currentpage={}):
    """Read/Fetch and pre-process markdown file"""
    target = get_target(target)
    if "http:" in md_where or "https:" in md_where:
-        return read_markdown_remote(md_where)
+        try:
+            mdr = read_markdown_remote(md_where)
+        except requests.RequestException as e:
+            if bypass_errors:
+                mdr = ""
+            else:
+                traceback.print_tb(e.__traceback__)
+                exit("Error fetching page %s: %s" % (md_where, e))
+        return mdr
    else:
-        return read_markdown_local(md_where, pp_env, target)
+        return read_markdown_local(md_where, pp_env, target, bypass_errors, currentpage=currentpage)


 def copy_static_files(template_static=True, content_static=True, out_path=None):
@@ -510,10 +531,9 @@ def render_pages(target=None, for_pdf=False, bypass_errors=False):

            try:
                html_content = parse_markdown(currentpage, target=target,
-                                              pages=pages)
+                                      pages=pages, bypass_errors=bypass_errors)

            except Exception as e:
-                import traceback
                if bypass_errors:
                    traceback.print_tb(e.__traceback__)
                    logging.warning( ("Skipping page %s " +
@@ -527,7 +547,10 @@ def render_pages(target=None, for_pdf=False, bypass_errors=False):
        else:
            html_content = ""

-        if "sidebar" in currentpage and currentpage["sidebar"] == "toc":
+        # default to a table-of-contents sidebar...
+        if "sidebar" not in currentpage:
+            currentpage["sidebar"] = "toc"
+        if currentpage["sidebar"] == "toc":
            sidebar_content = toc_from_headers(html_content)
        else:
            sidebar_content = None
@@ -683,6 +706,8 @@ if __name__ == "__main__":
    parser.add_argument("--no_cover", "-n", action="store_true",
                        help="(with --pages only) Don't automatically add a "+\
                        "cover page / index.html file.")
+    parser.add_argument("--skip_preprocessor", action="store_true", default=False,
+                        help="Don't pre-process Jinja syntax in markdown files")
    parser.add_argument("--title", type=str, help="Override target display "+\
                        "name. Useful when passing multiple args to --pages.")
    parser.add_argument("--list_targets_only", "-l", action="store_true",
@@ -712,6 +737,8 @@ if __name__ == "__main__":
    if cli_args.out_dir:
        config["out_path"] = cli_args.out_dir

+    config["skip_preprocessor"] = cli_args.skip_preprocessor
+
    if cli_args.pages:
        make_adhoc_target(cli_args.pages, cli_args.no_cover)
        cli_args.target = ADHOC_TARGET