From ca5ca11d4dba6c29d34c46861f206b943983b18b Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Mon, 3 Apr 2017 20:30:00 +0900 Subject: [PATCH 1/5] Specify the storage option for TOAST table when create table. The storage option such as AUTOVACUUM_VACUUM_SCALE_FACTOR can be set to both heap table and TOAST table. But the storage parameter for TOAST table had gone after repacked. This change create new function get_storage_param which returns all storage paramters including for TOAST table and OID setting. Issue #10. --- lib/pg_repack.sql.in | 37 ++++++++++++++++++++++++++++++++++++- regress/expected/repack.out | 21 +++++++++++++++++++++ regress/sql/repack.sql | 14 ++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index d294548..ef7d160 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -152,6 +152,41 @@ WHERE $$ LANGUAGE sql STABLE STRICT; +-- Get a comma-separated storage paramter for the table including +-- paramters for the corresponding TOAST table. +-- Note that since oid setting is always not NULL, this function +-- never returns NULL +CREATE FUNCTION repack.get_storage_param(oid) + RETURNS TEXT AS +$$ +SELECT array_to_string(array_agg(param), ', ') +FROM ( + -- table storage parameter + SELECT unnest(reloptions) as param + FROM pg_class + WHERE oid = $1 + UNION ALL + -- TOAST table storage parameter + SELECT ('toast.' || unnest(reloptions)) as param + FROM ( + SELECT reltoastrelid from pg_class where oid = $1 + ) as t, + pg_class as c + WHERE c.oid = t.reltoastrelid + UNION ALL + -- table oid + SELECT 'oids = ' || + CASE WHEN relhasoids + THEN 'true' + ELSE 'false' + END + FROM pg_class + WHERE oid = $1 + + ) as t +$$ +LANGUAGE sql STABLE STRICT; + -- includes not only PRIMARY KEYS but also UNIQUE NOT NULL keys CREATE VIEW repack.primary_keys AS SELECT indrelid, (repack.array_accum(indexrelid))[1] AS indexrelid @@ -183,7 +218,7 @@ CREATE VIEW repack.tables AS 'CREATE TABLE repack.log_' || R.oid || ' (id bigserial PRIMARY KEY, pk repack.pk_' || R.oid || ', row ' || repack.oid2text(R.oid) || ')' AS create_log, repack.get_create_trigger(R.oid, PK.indexrelid) AS create_trigger, repack.get_enable_trigger(R.oid) as enable_trigger, - 'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE ' AS create_table_1, + 'CREATE TABLE repack.table_' || R.oid || ' WITH (' || repack.get_storage_param(R.oid) || ') TABLESPACE ' AS create_table_1, coalesce(quote_ident(S.spcname), 'pg_default') as tablespace_orig, ' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table_2, repack.get_drop_columns(R.oid, 'repack.table_' || R.oid) AS drop_columns, diff --git a/regress/expected/repack.out b/regress/expected/repack.out index ef0ed4a..f5d4ac2 100644 --- a/regress/expected/repack.out +++ b/regress/expected/repack.out @@ -58,6 +58,12 @@ CREATE INDEX idxopts_t ON tbl_idxopts (t DESC NULLS LAST) WHERE (t != 'aaa'); -- Use this table to play with attribute options too ALTER TABLE tbl_idxopts ALTER i SET STATISTICS 1; ALTER TABLE tbl_idxopts ALTER t SET (n_distinct = -0.5); +CREATE TABLE tbl_with_toast ( + i integer PRIMARY KEY, + c text +); +ALTER TABLE tbl_with_toast SET (AUTOVACUUM_VACUUM_SCALE_FACTOR = 30, AUTOVACUUM_VACUUM_THRESHOLD = 300); +ALTER TABLE tbl_with_toast SET (TOAST.AUTOVACUUM_VACUUM_SCALE_FACTOR = 40, TOAST.AUTOVACUUM_VACUUM_THRESHOLD = 400); -- -- insert data -- @@ -131,6 +137,7 @@ INFO: repacking table "tbl_with_dropped_toast" INFO: repacking table "tbl_badindex" WARNING: skipping invalid index: CREATE UNIQUE INDEX idx_badindex_n ON tbl_badindex USING btree (n) INFO: repacking table "tbl_idxopts" +INFO: repacking table "tbl_with_toast" -- -- after -- @@ -285,6 +292,20 @@ SELECT * FROM tbl_with_dropped_toast; RESET enable_seqscan; RESET enable_indexscan; +-- check if storage option for both table and TOAST table didn't go away. +SELECT CASE relkind + WHEN 'r' THEN relname + WHEN 't' THEN 'toast_table' + END as table, + reloptions +FROM pg_class +WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid; + table | reloptions +----------------+--------------------------------------------------------------------- + toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} + tbl_with_toast | {autovacuum_vacuum_scale_factor=30,autovacuum_vacuum_threshold=300} +(2 rows) + -- -- check broken links or orphan toast relations -- diff --git a/regress/sql/repack.sql b/regress/sql/repack.sql index 58b358f..fa2ef8a 100644 --- a/regress/sql/repack.sql +++ b/regress/sql/repack.sql @@ -69,6 +69,12 @@ CREATE INDEX idxopts_t ON tbl_idxopts (t DESC NULLS LAST) WHERE (t != 'aaa'); -- Use this table to play with attribute options too ALTER TABLE tbl_idxopts ALTER i SET STATISTICS 1; ALTER TABLE tbl_idxopts ALTER t SET (n_distinct = -0.5); +CREATE TABLE tbl_with_toast ( + i integer PRIMARY KEY, + c text +); +ALTER TABLE tbl_with_toast SET (AUTOVACUUM_VACUUM_SCALE_FACTOR = 30, AUTOVACUUM_VACUUM_THRESHOLD = 300); +ALTER TABLE tbl_with_toast SET (TOAST.AUTOVACUUM_VACUUM_SCALE_FACTOR = 40, TOAST.AUTOVACUUM_VACUUM_THRESHOLD = 400); -- -- insert data @@ -157,6 +163,14 @@ SELECT * FROM view_for_dropped_column; SELECT * FROM tbl_with_dropped_toast; RESET enable_seqscan; RESET enable_indexscan; +-- check if storage option for both table and TOAST table didn't go away. +SELECT CASE relkind + WHEN 'r' THEN relname + WHEN 't' THEN 'toast_table' + END as table, + reloptions +FROM pg_class +WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid; -- -- check broken links or orphan toast relations From c44306ba742ca6a148a293668b618f9c200ffa52 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Mon, 3 Apr 2017 21:10:28 +0900 Subject: [PATCH 2/5] Fix regression test failure. --- regress/expected/repack.out | 5 +++-- regress/expected/repack_1.out | 22 ++++++++++++++++++++++ regress/sql/repack.sql | 3 ++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/regress/expected/repack.out b/regress/expected/repack.out index f5d4ac2..4840b64 100644 --- a/regress/expected/repack.out +++ b/regress/expected/repack.out @@ -299,11 +299,12 @@ SELECT CASE relkind END as table, reloptions FROM pg_class -WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid; +WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid +ORDER BY 1; table | reloptions ----------------+--------------------------------------------------------------------- - toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} tbl_with_toast | {autovacuum_vacuum_scale_factor=30,autovacuum_vacuum_threshold=300} + toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} (2 rows) -- diff --git a/regress/expected/repack_1.out b/regress/expected/repack_1.out index bc0333c..38a91ac 100644 --- a/regress/expected/repack_1.out +++ b/regress/expected/repack_1.out @@ -58,6 +58,12 @@ CREATE INDEX idxopts_t ON tbl_idxopts (t DESC NULLS LAST) WHERE (t != 'aaa'); -- Use this table to play with attribute options too ALTER TABLE tbl_idxopts ALTER i SET STATISTICS 1; ALTER TABLE tbl_idxopts ALTER t SET (n_distinct = -0.5); +CREATE TABLE tbl_with_toast ( + i integer PRIMARY KEY, + c text +); +ALTER TABLE tbl_with_toast SET (AUTOVACUUM_VACUUM_SCALE_FACTOR = 30, AUTOVACUUM_VACUUM_THRESHOLD = 300); +ALTER TABLE tbl_with_toast SET (TOAST.AUTOVACUUM_VACUUM_SCALE_FACTOR = 40, TOAST.AUTOVACUUM_VACUUM_THRESHOLD = 400); -- -- insert data -- @@ -131,6 +137,7 @@ INFO: repacking table "tbl_with_dropped_toast" INFO: repacking table "tbl_badindex" WARNING: skipping invalid index: CREATE UNIQUE INDEX idx_badindex_n ON tbl_badindex USING btree (n) INFO: repacking table "tbl_idxopts" +INFO: repacking table "tbl_with_toast" -- -- after -- @@ -285,6 +292,21 @@ SELECT * FROM tbl_with_dropped_toast; RESET enable_seqscan; RESET enable_indexscan; +-- check if storage option for both table and TOAST table didn't go away. +SELECT CASE relkind + WHEN 'r' THEN relname + WHEN 't' THEN 'toast_table' + END as table, + reloptions +FROM pg_class +WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid +ORDER BY 1; + table | reloptions +----------------+--------------------------------------------------------------------- + tbl_with_toast | {autovacuum_vacuum_scale_factor=30,autovacuum_vacuum_threshold=300} + toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} +(2 rows) + -- -- check broken links or orphan toast relations -- diff --git a/regress/sql/repack.sql b/regress/sql/repack.sql index fa2ef8a..b04e86f 100644 --- a/regress/sql/repack.sql +++ b/regress/sql/repack.sql @@ -170,7 +170,8 @@ SELECT CASE relkind END as table, reloptions FROM pg_class -WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid; +WHERE relname = 'tbl_with_toast' OR relname = 'pg_toast_' || 'tbl_with_toast'::regclass::oid +ORDER BY 1; -- -- check broken links or orphan toast relations From 3afe19e7c4ea92c39f5fbef4f6a1919fe40ec6ee Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 10 Apr 2017 23:16:19 +0100 Subject: [PATCH 3/5] Report issue fixed in docs Close #10 --- doc/pg_repack.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index c35c6fa..89baf8d 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -469,6 +469,7 @@ Releases * added ``--no-kill-backend`` option (issue #108) * added ``--no-superuser-check`` option (issue #114) * added ``--exclude-extension`` option (#97) + * restore TOAST storage parameters on repacked tables (issue #10) * pg_repack 1.3.4 From a6b2b0e82ee05c0d055ff58e633f93b822e0f87a Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Tue, 11 Apr 2017 23:43:11 +0900 Subject: [PATCH 4/5] Fix a mistake of merging. Also fix regression test failure. --- lib/pg_repack.sql.in | 2 ++ regress/expected/repack.out | 1 + regress/expected/repack_1.out | 1 + 3 files changed, 4 insertions(+) diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index 33c9868..862ed84 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -184,6 +184,8 @@ FROM ( WHERE oid = $1 ) as t +$$ +LANGUAGE sql STABLE STRICT; -- GET a SQL text to set column storage option for the table. CREATE FUNCTION repack.get_alter_col_storage(oid) diff --git a/regress/expected/repack.out b/regress/expected/repack.out index ebecc9a..dd4541d 100644 --- a/regress/expected/repack.out +++ b/regress/expected/repack.out @@ -314,6 +314,7 @@ ORDER BY 1; tbl_with_toast | {autovacuum_vacuum_scale_factor=30,autovacuum_vacuum_threshold=300} toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} (2 rows) + SELECT pg_relation_size(reltoastrelid) = 0 as check_toast_rel_size FROM pg_class WHERE relname = 'tbl_with_mod_column_storage'; check_toast_rel_size ---------------------- diff --git a/regress/expected/repack_1.out b/regress/expected/repack_1.out index 3c2f62e..a7bbe2f 100644 --- a/regress/expected/repack_1.out +++ b/regress/expected/repack_1.out @@ -314,6 +314,7 @@ ORDER BY 1; tbl_with_toast | {autovacuum_vacuum_scale_factor=30,autovacuum_vacuum_threshold=300} toast_table | {autovacuum_vacuum_scale_factor=40,autovacuum_vacuum_threshold=400} (2 rows) + SELECT pg_relation_size(reltoastrelid) = 0 as check_toast_rel_size FROM pg_class WHERE relname = 'tbl_with_mod_column_storage'; check_toast_rel_size ---------------------- From 6de31a1249f95abc4662865a6c1d19aa123bdab9 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Tue, 11 Apr 2017 23:49:10 +0900 Subject: [PATCH 5/5] Add new item in Japanese doc. --- doc/pg_repack_jp.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/pg_repack_jp.rst b/doc/pg_repack_jp.rst index 168ba72..fecee17 100644 --- a/doc/pg_repack_jp.rst +++ b/doc/pg_repack_jp.rst @@ -862,6 +862,7 @@ ACCESS EXCLUSIVEロックを取得します。その他のステップでは、A .. * added ``--no-kill-backend`` option (issue #108) .. * added ``--no-superuser-check`` option (issue #114) .. * added ``--exclude-extension`` option (#97) +.. * restore TOAST storage parameters on repacked tables (issue #10) .. * restore columns storage types in repacked tables (issue #94) * pg_repack 1.4 @@ -872,7 +873,8 @@ ACCESS EXCLUSIVEロックを取得します。その他のステップでは、A * ``--no-kill-backend`` オプションを追加しました (issue #108) * ``--no-superuser-check`` オプションを追加しました (issue #114) * ``--exclude-extension`` オプションを追加しました (#97) - * 列の格納タイプを再編成後のテーブルに再設定する様にしました (issue #94) + * TOASTテーブルの格納オプションを再編成後のテーブルに再設定するようにしました (issue #10) + * 列の格納タイプを再編成後のテーブルに再設定するようにしました (issue #94) .. * pg_repack 1.3.4 .. * grab exclusive lock before dropping original table (#81)