Set column storage type before first copying table.
Previously, even if the table whose column storage type has been changed the pg_repack did first copy the data to table without changing column storage paramater. This cause of that the existing data is pushed out to its toast table even if actual column storage type is "main". Issue #94.
This commit is contained in:
Masahiko Sawada
Daniele Varrazzo
parent
6d7b1dbca4
commit
5adff6ff0b
@ -193,7 +193,9 @@ typedef struct repack_table
|
||||
const char *create_log; /* CREATE TABLE log */
|
||||
const char *create_trigger; /* CREATE TRIGGER repack_trigger */
|
||||
const char *enable_trigger; /* ALTER TABLE ENABLE ALWAYS TRIGGER repack_trigger */
|
||||
const char *create_table; /* CREATE TABLE table AS SELECT */
|
||||
const char *create_table; /* CREATE TABLE table AS SELECT WITH NO DATA*/
|
||||
const char *copy_data; /* INSERT INTO */
|
||||
const char *alter_col_storage; /* ALTER TABLE ALTER COLUMN SET STORAGE */
|
||||
const char *drop_columns; /* ALTER TABLE DROP COLUMNs */
|
||||
const char *delete_log; /* DELETE FROM log */
|
||||
const char *lock_table; /* LOCK TABLE table */
|
||||
@ -759,6 +761,8 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
|
||||
create_table_1 = getstr(res, i, c++);
|
||||
tablespace = getstr(res, i, c++); /* to be clobbered */
|
||||
create_table_2 = getstr(res, i, c++);
|
||||
table.copy_data = getstr(res, i , c++);
|
||||
table.alter_col_storage = getstr(res, i, c++);
|
||||
table.drop_columns = getstr(res, i, c++);
|
||||
table.delete_log = getstr(res, i, c++);
|
||||
table.lock_table = getstr(res, i, c++);
|
||||
@ -781,27 +785,25 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
|
||||
/* CLUSTER mode */
|
||||
appendStringInfoString(&sql, " ORDER BY ");
|
||||
appendStringInfoString(&sql, ckey);
|
||||
table.create_table = sql.data;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* VACUUM FULL mode (non-clustered tables) */
|
||||
table.create_table = sql.data;
|
||||
}
|
||||
|
||||
/* else, VACUUM FULL mode (non-clustered tables) */
|
||||
}
|
||||
else if (!orderby[0])
|
||||
{
|
||||
/* VACUUM FULL mode (for clustered tables too) */
|
||||
table.create_table = sql.data;
|
||||
/* VACUUM FULL mode (for clustered tables too), do nothing */
|
||||
}
|
||||
else
|
||||
{
|
||||
/* User specified ORDER BY */
|
||||
appendStringInfoString(&sql, " ORDER BY ");
|
||||
appendStringInfoString(&sql, orderby);
|
||||
table.create_table = sql.data;
|
||||
}
|
||||
|
||||
/* Always append WITH NOT DATA */
|
||||
appendStringInfoString(&sql, " WITH NO DATA");
|
||||
table.create_table = sql.data;
|
||||
|
||||
repack_one_table(&table, orderby);
|
||||
}
|
||||
ret = true;
|
||||
@ -1086,25 +1088,28 @@ repack_one_table(repack_table *table, const char *orderby)
|
||||
elog(INFO, "repacking table \"%s\"", table->target_name);
|
||||
|
||||
elog(DEBUG2, "---- repack_one_table ----");
|
||||
elog(DEBUG2, "target_name : %s", table->target_name);
|
||||
elog(DEBUG2, "target_oid : %u", table->target_oid);
|
||||
elog(DEBUG2, "target_toast : %u", table->target_toast);
|
||||
elog(DEBUG2, "target_tidx : %u", table->target_tidx);
|
||||
elog(DEBUG2, "pkid : %u", table->pkid);
|
||||
elog(DEBUG2, "ckid : %u", table->ckid);
|
||||
elog(DEBUG2, "create_pktype : %s", table->create_pktype);
|
||||
elog(DEBUG2, "create_log : %s", table->create_log);
|
||||
elog(DEBUG2, "create_trigger : %s", table->create_trigger);
|
||||
elog(DEBUG2, "enable_trigger : %s", table->enable_trigger);
|
||||
elog(DEBUG2, "create_table : %s", table->create_table);
|
||||
elog(DEBUG2, "drop_columns : %s", table->drop_columns ? table->drop_columns : "(skipped)");
|
||||
elog(DEBUG2, "delete_log : %s", table->delete_log);
|
||||
elog(DEBUG2, "lock_table : %s", table->lock_table);
|
||||
elog(DEBUG2, "sql_peek : %s", table->sql_peek);
|
||||
elog(DEBUG2, "sql_insert : %s", table->sql_insert);
|
||||
elog(DEBUG2, "sql_delete : %s", table->sql_delete);
|
||||
elog(DEBUG2, "sql_update : %s", table->sql_update);
|
||||
elog(DEBUG2, "sql_pop : %s", table->sql_pop);
|
||||
elog(DEBUG2, "target_name : %s", table->target_name);
|
||||
elog(DEBUG2, "target_oid : %u", table->target_oid);
|
||||
elog(DEBUG2, "target_toast : %u", table->target_toast);
|
||||
elog(DEBUG2, "target_tidx : %u", table->target_tidx);
|
||||
elog(DEBUG2, "pkid : %u", table->pkid);
|
||||
elog(DEBUG2, "ckid : %u", table->ckid);
|
||||
elog(DEBUG2, "create_pktype : %s", table->create_pktype);
|
||||
elog(DEBUG2, "create_log : %s", table->create_log);
|
||||
elog(DEBUG2, "create_trigger : %s", table->create_trigger);
|
||||
elog(DEBUG2, "enable_trigger : %s", table->enable_trigger);
|
||||
elog(DEBUG2, "create_table : %s", table->create_table);
|
||||
elog(DEBUG2, "copy_data : %s", table->copy_data);
|
||||
elog(DEBUG2, "alter_col_storage : %s", table->alter_col_storage ?
|
||||
table->alter_col_storage : "(skipped)");
|
||||
elog(DEBUG2, "drop_columns : %s", table->drop_columns ? table->drop_columns : "(skipped)");
|
||||
elog(DEBUG2, "delete_log : %s", table->delete_log);
|
||||
elog(DEBUG2, "lock_table : %s", table->lock_table);
|
||||
elog(DEBUG2, "sql_peek : %s", table->sql_peek);
|
||||
elog(DEBUG2, "sql_insert : %s", table->sql_insert);
|
||||
elog(DEBUG2, "sql_delete : %s", table->sql_delete);
|
||||
elog(DEBUG2, "sql_update : %s", table->sql_update);
|
||||
elog(DEBUG2, "sql_pop : %s", table->sql_pop);
|
||||
|
||||
if (dryrun)
|
||||
return;
|
||||
@ -1346,7 +1351,14 @@ repack_one_table(repack_table *table, const char *orderby)
|
||||
if (!(lock_access_share(connection, table->target_oid, table->target_name)))
|
||||
goto cleanup;
|
||||
|
||||
/*
|
||||
* Before copying data to the target table, we need to set the column storage
|
||||
* type if its storage type has been changed from the type default.
|
||||
*/
|
||||
command(table->create_table, 0, NULL);
|
||||
if (table->alter_col_storage)
|
||||
command(table->alter_col_storage, 0, NULL);
|
||||
command(table->copy_data, 0, NULL);
|
||||
temp_obj_num++;
|
||||
printfStringInfo(&sql, "SELECT repack.disable_autovacuum('repack.table_%u')", table->target_oid);
|
||||
if (table->drop_columns)
|
||||
|
||||
@ -152,6 +152,37 @@ WHERE
|
||||
$$
|
||||
LANGUAGE sql STABLE STRICT;
|
||||
|
||||
-- GET a SQL text to set column storage option for the table.
|
||||
CREATE FUNCTION repack.get_alter_col_storage(oid)
|
||||
RETURNS text AS
|
||||
$$
|
||||
SELECT 'ALTER TABLE repack.table_' || $1 || array_to_string(column_storage, ',')
|
||||
FROM (
|
||||
SELECT
|
||||
repack.array_accum(' ALTER ' || quote_ident(attname) ||
|
||||
CASE attstorage
|
||||
WHEN 'p' THEN ' SET STORAGE PLAIN'
|
||||
WHEN 'm' THEN ' SET STORAGE MAIN'
|
||||
WHEN 'e' THEN ' SET STORAGE EXTERNAL'
|
||||
WHEN 'x' THEN ' SET STORAGE EXTENDED'
|
||||
END) AS column_storage
|
||||
FROM (
|
||||
SELECT *
|
||||
FROM pg_attribute a
|
||||
JOIN pg_type t on t.oid = atttypid
|
||||
JOIN pg_class r on r.oid = a.attrelid
|
||||
JOIN pg_namespace s on s.oid = r.relnamespace
|
||||
WHERE typstorage <> attstorage
|
||||
AND attrelid = $1
|
||||
AND attnum > 0
|
||||
AND NOT attisdropped
|
||||
ORDER BY attnum
|
||||
) T
|
||||
) T
|
||||
WHERE array_upper(column_storage , 1) > 0
|
||||
$$
|
||||
LANGUAGE sql STABLE STRICT;
|
||||
|
||||
-- includes not only PRIMARY KEYS but also UNIQUE NOT NULL keys
|
||||
CREATE VIEW repack.primary_keys AS
|
||||
SELECT indrelid, (repack.array_accum(indexrelid))[1] AS indexrelid
|
||||
@ -186,6 +217,8 @@ CREATE VIEW repack.tables AS
|
||||
'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE ' AS create_table_1,
|
||||
coalesce(quote_ident(S.spcname), 'pg_default') as tablespace_orig,
|
||||
' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table_2,
|
||||
'INSERT INTO repack.table_' || R.oid || ' SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS copy_data,
|
||||
repack.get_alter_col_storage(R.oid) AS alter_col_storage,
|
||||
repack.get_drop_columns(R.oid, 'repack.table_' || R.oid) AS drop_columns,
|
||||
'DELETE FROM repack.log_' || R.oid AS delete_log,
|
||||
'LOCK TABLE ' || repack.oid2text(R.oid) || ' IN ACCESS EXCLUSIVE MODE' AS lock_table,
|
||||
|
||||
@ -58,6 +58,11 @@ CREATE INDEX idxopts_t ON tbl_idxopts (t DESC NULLS LAST) WHERE (t != 'aaa');
|
||||
-- Use this table to play with attribute options too
|
||||
ALTER TABLE tbl_idxopts ALTER i SET STATISTICS 1;
|
||||
ALTER TABLE tbl_idxopts ALTER t SET (n_distinct = -0.5);
|
||||
CREATE TABLE tbl_with_mod_column_storage (
|
||||
id integer PRIMARY KEY,
|
||||
c text
|
||||
);
|
||||
ALTER TABLE tbl_with_mod_column_storage ALTER c SET STORAGE MAIN;
|
||||
--
|
||||
-- insert data
|
||||
--
|
||||
@ -85,6 +90,8 @@ INSERT INTO tbl_with_dropped_toast VALUES(2, 20, sqrt(2::numeric(1000,999))::tex
|
||||
ALTER TABLE tbl_with_dropped_toast DROP COLUMN t;
|
||||
INSERT INTO tbl_badindex VALUES(1, 10);
|
||||
INSERT INTO tbl_badindex VALUES(2, 10);
|
||||
-- insert data that is always stored into the toast table if column type is extended.
|
||||
INSERT INTO tbl_with_mod_column_storage SELECT 1, array_to_string(ARRAY(SELECT chr(code) FROM generate_series(33,3000) code), '');
|
||||
-- This will fail. Silence the message as it's different across PG versions.
|
||||
SET client_min_messages = fatal;
|
||||
CREATE UNIQUE INDEX CONCURRENTLY idx_badindex_n ON tbl_badindex (n);
|
||||
@ -131,6 +138,7 @@ INFO: repacking table "tbl_with_dropped_toast"
|
||||
INFO: repacking table "tbl_badindex"
|
||||
WARNING: skipping invalid index: CREATE UNIQUE INDEX idx_badindex_n ON tbl_badindex USING btree (n)
|
||||
INFO: repacking table "tbl_idxopts"
|
||||
INFO: repacking table "tbl_with_mod_column_storage"
|
||||
--
|
||||
-- after
|
||||
--
|
||||
@ -206,14 +214,14 @@ Indexes:
|
||||
"tbl_idxopts_pkey" PRIMARY KEY, btree (i)
|
||||
"idxopts_t" btree (t DESC NULLS LAST) WHERE t <> 'aaa'::text
|
||||
|
||||
SELECT col1, to_char("time", 'YYYY-MM-DD HH24:MI:SS'), ","")" FROM tbl_cluster;
|
||||
SELECT col1, to_char("time", 'YYYY-MM-DD HH24:MI:SS'), ","")" FROM tbl_cluster ORDER BY 1, 2;
|
||||
col1 | to_char | ,")
|
||||
------+---------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
1 | 2008-12-31 10:00:00 | admin
|
||||
2 | 2008-01-01 00:00:00 | king
|
||||
5 | 2008-01-01 00:30:00 | 1.4142135623730950488016887242096980785696718753769480731766797379907324784621070388503875343276415727350138462309122970249248360558507372126441214970999358314132226659275055927557999505011527820605714701095599716059702745345968620147285174186408891986095523292304843087143214508397626036279952514079896872533965463318088296406206152583523950547457502877599617298355752203375318570113543746034084988471603868999706990048150305440277903164542478230684929369186215805784631115966687130130156185689872372352885092648612494977154218334204285686060146824720771435854874155657069677653720226485447015858801620758474922657226002085584466521458398893944370926591800311388246468157082630100594858704003186480342194897278290641045072636881313739855256117322040245091227700226941127573627280495738108967504018369868368450725799364729060762996941380475654823728997180326802474420629269124859052181004459842150591120249441341728531478105803603371077309182869314710171111683916581726889419758716582152128229518488471.732050807568877293527446341505872366942805253810380628055806979451933016908800037081146186757248575675626141415406703029969945094998952478811655512094373648528093231902305582067974820101084674923265015312343266903322886650672254668921837971227047131660367861588019049986537379859389467650347506576050756618348129606100947602187190325083145829523959832997789824508288714463832917347224163984587855397667958063818353666110843173780894378316102088305524901670023520711144288695990956365797087168498072899493296484283020786408603988738697537582317317831395992983007838702877053913369563312103707264019249106768231199288375641141422016742752102372994270831059898459475987664288897796147837958390228854852903576033852808064381972344661059689722872865264153822664698420021195484155278441181286534507035191650016689294415480846071277143999762926834629577438361895110127148638746976545982451788550975379013880664961911962222957110555242923723192197738262561631468842032853716682938649611917049738836395495938
|
||||
3 | 2008-03-04 12:00:00 | joker
|
||||
4 | 2008-03-05 15:00:00 | queen
|
||||
1 | 2008-12-31 10:00:00 | admin
|
||||
5 | 2008-01-01 00:30:00 | 1.4142135623730950488016887242096980785696718753769480731766797379907324784621070388503875343276415727350138462309122970249248360558507372126441214970999358314132226659275055927557999505011527820605714701095599716059702745345968620147285174186408891986095523292304843087143214508397626036279952514079896872533965463318088296406206152583523950547457502877599617298355752203375318570113543746034084988471603868999706990048150305440277903164542478230684929369186215805784631115966687130130156185689872372352885092648612494977154218334204285686060146824720771435854874155657069677653720226485447015858801620758474922657226002085584466521458398893944370926591800311388246468157082630100594858704003186480342194897278290641045072636881313739855256117322040245091227700226941127573627280495738108967504018369868368450725799364729060762996941380475654823728997180326802474420629269124859052181004459842150591120249441341728531478105803603371077309182869314710171111683916581726889419758716582152128229518488471.732050807568877293527446341505872366942805253810380628055806979451933016908800037081146186757248575675626141415406703029969945094998952478811655512094373648528093231902305582067974820101084674923265015312343266903322886650672254668921837971227047131660367861588019049986537379859389467650347506576050756618348129606100947602187190325083145829523959832997789824508288714463832917347224163984587855397667958063818353666110843173780894378316102088305524901670023520711144288695990956365797087168498072899493296484283020786408603988738697537582317317831395992983007838702877053913369563312103707264019249106768231199288375641141422016742752102372994270831059898459475987664288897796147837958390228854852903576033852808064381972344661059689722872865264153822664698420021195484155278441181286534507035191650016689294415480846071277143999762926834629577438361895110127148638746976545982451788550975379013880664961911962222957110555242923723192197738262561631468842032853716682938649611917049738836395495938
|
||||
(5 rows)
|
||||
|
||||
SELECT * FROM tbl_only_ckey ORDER BY 1;
|
||||
@ -239,14 +247,14 @@ SELECT * FROM tbl_gistkey ORDER BY 1;
|
||||
|
||||
SET enable_seqscan = on;
|
||||
SET enable_indexscan = off;
|
||||
SELECT * FROM tbl_with_dropped_column;
|
||||
SELECT * FROM tbl_with_dropped_column ;
|
||||
c1 | id | c2 | c3
|
||||
----+----+----+----
|
||||
c1 | 1 | c2 |
|
||||
c1 | 2 | c2 |
|
||||
c1 | 1 | c2 |
|
||||
(2 rows)
|
||||
|
||||
SELECT * FROM view_for_dropped_column;
|
||||
SELECT * FROM view_for_dropped_column ORDER BY 1, 2;
|
||||
c1 | id | c2 | c3
|
||||
----+----+----+----
|
||||
c1 | 1 | c2 |
|
||||
@ -262,7 +270,7 @@ SELECT * FROM tbl_with_dropped_toast;
|
||||
|
||||
SET enable_seqscan = off;
|
||||
SET enable_indexscan = on;
|
||||
SELECT * FROM tbl_with_dropped_column;
|
||||
SELECT * FROM tbl_with_dropped_column ORDER BY 1, 2;
|
||||
c1 | id | c2 | c3
|
||||
----+----+----+----
|
||||
c1 | 1 | c2 |
|
||||
@ -272,8 +280,8 @@ SELECT * FROM tbl_with_dropped_column;
|
||||
SELECT * FROM view_for_dropped_column;
|
||||
c1 | id | c2 | c3
|
||||
----+----+----+----
|
||||
c1 | 1 | c2 |
|
||||
c1 | 2 | c2 |
|
||||
c1 | 1 | c2 |
|
||||
(2 rows)
|
||||
|
||||
SELECT * FROM tbl_with_dropped_toast;
|
||||
@ -285,6 +293,12 @@ SELECT * FROM tbl_with_dropped_toast;
|
||||
|
||||
RESET enable_seqscan;
|
||||
RESET enable_indexscan;
|
||||
SELECT pg_relation_size(reltoastrelid) = 0 as check_toast_rel_size FROM pg_class WHERE relname = 'tbl_with_mod_column_storage';
|
||||
check_toast_rel_size
|
||||
----------------------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
--
|
||||
-- check broken links or orphan toast relations
|
||||
--
|
||||
|
||||
@ -69,6 +69,11 @@ CREATE INDEX idxopts_t ON tbl_idxopts (t DESC NULLS LAST) WHERE (t != 'aaa');
|
||||
-- Use this table to play with attribute options too
|
||||
ALTER TABLE tbl_idxopts ALTER i SET STATISTICS 1;
|
||||
ALTER TABLE tbl_idxopts ALTER t SET (n_distinct = -0.5);
|
||||
CREATE TABLE tbl_with_mod_column_storage (
|
||||
id integer PRIMARY KEY,
|
||||
c text
|
||||
);
|
||||
ALTER TABLE tbl_with_mod_column_storage ALTER c SET STORAGE MAIN;
|
||||
|
||||
--
|
||||
-- insert data
|
||||
@ -105,6 +110,9 @@ ALTER TABLE tbl_with_dropped_toast DROP COLUMN t;
|
||||
INSERT INTO tbl_badindex VALUES(1, 10);
|
||||
INSERT INTO tbl_badindex VALUES(2, 10);
|
||||
|
||||
-- insert data that is always stored into the toast table if column type is extended.
|
||||
INSERT INTO tbl_with_mod_column_storage SELECT 1, array_to_string(ARRAY(SELECT chr(code) FROM generate_series(33,3000) code), '');
|
||||
|
||||
-- This will fail. Silence the message as it's different across PG versions.
|
||||
SET client_min_messages = fatal;
|
||||
CREATE UNIQUE INDEX CONCURRENTLY idx_badindex_n ON tbl_badindex (n);
|
||||
@ -140,23 +148,24 @@ SELECT * FROM tbl_with_dropped_toast;
|
||||
\d tbl_with_dropped_toast
|
||||
\d tbl_idxopts
|
||||
|
||||
SELECT col1, to_char("time", 'YYYY-MM-DD HH24:MI:SS'), ","")" FROM tbl_cluster;
|
||||
SELECT col1, to_char("time", 'YYYY-MM-DD HH24:MI:SS'), ","")" FROM tbl_cluster ORDER BY 1, 2;
|
||||
SELECT * FROM tbl_only_ckey ORDER BY 1;
|
||||
SELECT * FROM tbl_only_pkey ORDER BY 1;
|
||||
SELECT * FROM tbl_gistkey ORDER BY 1;
|
||||
|
||||
SET enable_seqscan = on;
|
||||
SET enable_indexscan = off;
|
||||
SELECT * FROM tbl_with_dropped_column;
|
||||
SELECT * FROM view_for_dropped_column;
|
||||
SELECT * FROM tbl_with_dropped_column ;
|
||||
SELECT * FROM view_for_dropped_column ORDER BY 1, 2;
|
||||
SELECT * FROM tbl_with_dropped_toast;
|
||||
SET enable_seqscan = off;
|
||||
SET enable_indexscan = on;
|
||||
SELECT * FROM tbl_with_dropped_column;
|
||||
SELECT * FROM tbl_with_dropped_column ORDER BY 1, 2;
|
||||
SELECT * FROM view_for_dropped_column;
|
||||
SELECT * FROM tbl_with_dropped_toast;
|
||||
RESET enable_seqscan;
|
||||
RESET enable_indexscan;
|
||||
SELECT pg_relation_size(reltoastrelid) = 0 as check_toast_rel_size FROM pg_class WHERE relname = 'tbl_with_mod_column_storage';
|
||||
|
||||
--
|
||||
-- check broken links or orphan toast relations
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user