From be1d1a13bec44d01797c6e09a0f243c7cd9d46a1 Mon Sep 17 00:00:00 2001 From: CJ Cobb Date: Thu, 10 Feb 2022 15:08:56 +0000 Subject: [PATCH] more aggressive postgres partitioning --- src/backend/Pg.cpp | 66 +++++++++++---------------------- src/backend/PostgresBackend.cpp | 3 +- 2 files changed, 22 insertions(+), 47 deletions(-) diff --git a/src/backend/Pg.cpp b/src/backend/Pg.cpp index 187999de..337bc2e8 100644 --- a/src/backend/Pg.cpp +++ b/src/backend/Pg.cpp @@ -1049,16 +1049,6 @@ CREATE TABLE IF NOT EXISTS objects ( CREATE INDEX objects_idx ON objects USING btree(key,ledger_seq); -create table if not exists objects1 partition of objects for values from (0) to (10000000); -create table if not exists objects2 partition of objects for values from (10000000) to (20000000); -create table if not exists objects3 partition of objects for values from (20000000) to (30000000); -create table if not exists objects4 partition of objects for values from (30000000) to (40000000); -create table if not exists objects5 partition of objects for values from (40000000) to (50000000); -create table if not exists objects6 partition of objects for values from (50000000) to (60000000); -create table if not exists objects7 partition of objects for values from (60000000) to (70000000); -create table if not exists objects8 partition of objects for values from (70000000) to (80000000); -create table if not exists objects9 partition of objects for values from (80000000) to (90000000); -create table if not exists objects10 partition of objects for values from (90000000) to (100000000); -- Index for lookups by ledger hash. @@ -1074,16 +1064,6 @@ CREATE TABLE IF NOT EXISTS transactions ( transaction bytea NOT NULL, metadata bytea NOT NULL ) PARTITION BY RANGE(ledger_seq); -create table if not exists transactions1 partition of transactions for values from (0) to (10000000); -create table if not exists transactions2 partition of transactions for values from (10000000) to (20000000); -create table if not exists transactions3 partition of transactions for values from (20000000) to (30000000); -create table if not exists transactions4 partition of transactions for values from (30000000) to (40000000); -create table if not exists transactions5 partition of transactions for values from (40000000) to (50000000); -create table if not exists transactions6 partition of transactions for values from (50000000) to (60000000); -create table if not exists transactions7 partition of transactions for values from (60000000) to (70000000); -create table if not exists transactions8 partition of transactions for values from (70000000) to (80000000); -create table if not exists transactions9 partition of transactions for values from (80000000) to (90000000); -create table if not exists transactions10 partition of transactions for values from (90000000) to (100000000); create index if not exists tx_by_hash on transactions using hash (hash); create index if not exists tx_by_lgr_seq on transactions using hash (ledger_seq); @@ -1097,17 +1077,6 @@ CREATE TABLE IF NOT EXISTS account_transactions ( hash bytea NOT NULL, PRIMARY KEY (account, ledger_seq, transaction_index, hash) ) PARTITION BY RANGE (ledger_seq); -create table if not exists account_transactions1 partition of account_transactions for values from (0) to (10000000); -create table if not exists account_transactions2 partition of account_transactions for values from (10000000) to (20000000); -create table if not exists account_transactions3 partition of account_transactions for values from (20000000) to (30000000); -create table if not exists account_transactions4 partition of account_transactions for values from (30000000) to (40000000); -create table if not exists account_transactions5 partition of account_transactions for values from (40000000) to (50000000); -create table if not exists account_transactions6 partition of account_transactions for values from (50000000) to (60000000); -create table if not exists account_transactions7 partition of account_transactions for values from (60000000) to (70000000); -create table if not exists account_transactions8 partition of account_transactions for values from (70000000) to (80000000); -create table if not exists account_transactions9 partition of account_transactions for values from (80000000) to (90000000); -create table if not exists account_transactions10 partition of account_transactions for values from (90000000) to (100000000); - CREATE TABLE IF NOT EXISTS successor ( key bytea NOT NULL, @@ -1116,16 +1085,6 @@ CREATE TABLE IF NOT EXISTS successor ( PRIMARY KEY(key, ledger_seq) ) PARTITION BY RANGE(ledger_seq); -create table if not exists successor1 partition of successor for values from (0) to (10000000); -create table if not exists successor2 partition of successor for values from (10000000) to (20000000); -create table if not exists successor3 partition of successor for values from (20000000) to (30000000); -create table if not exists successor4 partition of successor for values from (30000000) to (40000000); -create table if not exists successor5 partition of successor for values from (40000000) to (50000000); -create table if not exists successor6 partition of successor for values from (50000000) to (60000000); -create table if not exists successor7 partition of successor for values from (60000000) to (70000000); -create table if not exists successor8 partition of successor for values from (70000000) to (80000000); -create table if not exists successor9 partition of successor for values from (80000000) to (90000000); -create table if not exists successor10 partition of successor for values from (90000000) to (100000000); -- Avoid inadvertent administrative tampering with committed data. @@ -1173,23 +1132,39 @@ CREATE OR REPLACE FUNCTION insert_ancestry() RETURNS TRIGGER AS $$ DECLARE _parent bytea; _child bytea; + _partition text; + _lowerBound bigint; + _upperBound bigint; + _interval bigint; BEGIN IF length(NEW.ledger_hash) != 32 OR length(NEW.prev_hash) != 32 THEN RAISE 'ledger_hash and prev_hash must each be 32 bytes: %', NEW; END IF; + + _interval = 1000000; + _lowerBound = (NEW.ledger_seq / _interval); + _partition= _lowerBound; + _lowerBound = _lowerBound * _interval; + _upperBound = _lowerBound + _interval; + + + EXECUTE format('create table if not exists public.objects_part_%s partition of public.objects for values from (%s) to (%s)',_partition,_lowerBound,_upperBound); + EXECUTE format('create table if not exists public.successor_part_%s partition of public.successor for values from (%s) to (%s)',_partition,_lowerBound,_upperBound); + EXECUTE format('create table if not exists public.transactions_part_%s partition of public.transactions for values from (%s) to (%s)',_partition,_lowerBound,_upperBound); + EXECUTE format('create table if not exists public.account_transactions_part_%s partition of public.account_transactions for values from (%s) to (%s)',_partition,_lowerBound,_upperBound); IF (SELECT ledger_hash - FROM ledgers + FROM public.ledgers ORDER BY ledger_seq DESC LIMIT 1) = NEW.prev_hash THEN RETURN NEW; END IF; - IF NOT EXISTS (SELECT 1 FROM LEDGERS) THEN RETURN NEW; END IF; + IF NOT EXISTS (SELECT 1 FROM public.ledgers) THEN RETURN NEW; END IF; _parent := (SELECT ledger_hash - FROM ledgers + FROM public.ledgers WHERE ledger_seq = NEW.ledger_seq - 1); _child := (SELECT prev_hash - FROM ledgers + FROM public.ledgers WHERE ledger_seq = NEW.ledger_seq + 1); IF _parent IS NULL AND _child IS NULL THEN RAISE 'Ledger Ancestry error: orphan.'; @@ -1201,6 +1176,7 @@ BEGIN RAISE 'Ledger Ancestry error: bad child.'; END IF; + RETURN NEW; END; $$ LANGUAGE plpgsql; diff --git a/src/backend/PostgresBackend.cpp b/src/backend/PostgresBackend.cpp index d101d218..1b82b998 100644 --- a/src/backend/PostgresBackend.cpp +++ b/src/backend/PostgresBackend.cpp @@ -691,8 +691,7 @@ PostgresBackend::fetchAccountTransactions( void PostgresBackend::open(bool readOnly) { - if (!readOnly) - initSchema(pgPool_); + initSchema(pgPool_); initAccountTx(pgPool_); }