From 960930b645df8eeeda15f176c95d3e450786f78a Mon Sep 17 00:00:00 2001 From: Takahiro Itagaki Date: Fri, 29 Apr 2011 05:06:48 +0000 Subject: [PATCH] Fixed database corruption when target tables have dropped columns, and there are views or functions depending on columns after dropped ones. The issue was reported by depesz, and original patch by Denish Patel. Improved documentation how to build binaries from source. COPYRIGHT updated. --- COPYRIGHT | 10 +-- Makefile | 3 +- bin/Makefile | 3 +- bin/expected/reorg.out | 43 ++++++++++++ bin/pg_reorg.c | 18 +++-- bin/pgut/pgut-fe.c | 5 +- bin/pgut/pgut-fe.h | 5 +- bin/pgut/pgut.c | 5 +- bin/pgut/pgut.h | 5 +- bin/sql/reorg.sql | 14 ++++ doc/index-ja.html | 5 +- doc/index.html | 5 +- doc/pg_reorg-ja.html | 26 +++++-- doc/pg_reorg.html | 28 ++++++-- lib/Makefile | 3 +- lib/pg_reorg.sql.in | 43 +++++++++++- lib/pgut/pgut-be.c | 5 +- lib/pgut/pgut-be.h | 5 +- lib/pgut/pgut-spi.c | 5 +- lib/pgut/pgut-spi.h | 5 +- lib/reorg.c | 125 +--------------------------------- lib/uninstall_pg_reorg.sql | 3 +- msvc/bin.2010.vcxproj | 2 + msvc/bin.2010.vcxproj.filters | 4 ++ 24 files changed, 198 insertions(+), 177 deletions(-) diff --git a/COPYRIGHT b/COPYRIGHT index e806722..9c05a6d 100755 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,4 +1,5 @@ -Copyright (c) 2008-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +Portions Copyright (c) 2011, Itagaki Takahiro All rights reserved. Redistribution and use in source and binary forms, with or without @@ -9,10 +10,9 @@ modification, are permitted provided that the following conditions are met: * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION - (NTT) nor the names of its contributors may be used to endorse or - promote products derived from this software without specific prior - written permission. + * Neither the name of the authors nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE diff --git a/Makefile b/Makefile index db2b3da..01e02e3 100755 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ # # pg_reorg: Makefile # -# Copyright (c) 2008-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +# Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +# Portions Copyright (c) 2011, Itagaki Takahiro # ifndef USE_PGXS top_builddir = ../.. diff --git a/bin/Makefile b/bin/Makefile index 1749ab8..8d22c9d 100755 --- a/bin/Makefile +++ b/bin/Makefile @@ -1,7 +1,8 @@ # # pg_reorg: bin/Makefile # -# Copyright (c) 2008-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +# Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +# Portions Copyright (c) 2011, Itagaki Takahiro # SRCS = pg_reorg.c pgut/pgut.c pgut/pgut-fe.c OBJS = $(SRCS:.c=.o) diff --git a/bin/expected/reorg.out b/bin/expected/reorg.out index a0aff7d..8793c48 100755 --- a/bin/expected/reorg.out +++ b/bin/expected/reorg.out @@ -66,6 +66,8 @@ ALTER TABLE tbl_with_dropped_column DROP COLUMN d1; ALTER TABLE tbl_with_dropped_column DROP COLUMN d2; ALTER TABLE tbl_with_dropped_column DROP COLUMN d3; ALTER TABLE tbl_with_dropped_column ADD COLUMN c3 text; +CREATE VIEW view_for_dropped_column AS + SELECT * FROM tbl_with_dropped_column; INSERT INTO tbl_with_dropped_toast VALUES(1, 10, 'abc'); INSERT INTO tbl_with_dropped_toast VALUES(2, 20, sqrt(2::numeric(1000,999))::text || sqrt(3::numeric(1000,999))::text); ALTER TABLE tbl_with_dropped_toast DROP COLUMN t; @@ -79,6 +81,13 @@ SELECT * FROM tbl_with_dropped_column; c1 | 1 | c2 | (2 rows) +SELECT * FROM view_for_dropped_column; + c1 | id | c2 | c3 +----+----+----+---- + c1 | 2 | c2 | + c1 | 1 | c2 | +(2 rows) + SELECT * FROM tbl_with_dropped_toast; i | j ---+---- @@ -188,6 +197,8 @@ SELECT * FROM tbl_gistkey ORDER BY 1; 2 | <(4,5),6> (2 rows) +SET enable_seqscan = on; +SET enable_indexscan = off; SELECT * FROM tbl_with_dropped_column; c1 | id | c2 | c3 ----+----+----+---- @@ -195,6 +206,13 @@ SELECT * FROM tbl_with_dropped_column; c1 | 2 | c2 | (2 rows) +SELECT * FROM view_for_dropped_column; + c1 | id | c2 | c3 +----+----+----+---- + c1 | 1 | c2 | + c1 | 2 | c2 | +(2 rows) + SELECT * FROM tbl_with_dropped_toast; i | j ---+---- @@ -202,6 +220,31 @@ SELECT * FROM tbl_with_dropped_toast; 2 | 20 (2 rows) +SET enable_seqscan = off; +SET enable_indexscan = on; +SELECT * FROM tbl_with_dropped_column; + c1 | id | c2 | c3 +----+----+----+---- + c1 | 1 | c2 | + c1 | 2 | c2 | +(2 rows) + +SELECT * FROM view_for_dropped_column; + c1 | id | c2 | c3 +----+----+----+---- + c1 | 1 | c2 | + c1 | 2 | c2 | +(2 rows) + +SELECT * FROM tbl_with_dropped_toast; + i | j +---+---- + 1 | 10 + 2 | 20 +(2 rows) + +RESET enable_seqscan; +RESET enable_indexscan; -- -- check broken links or orphan toast relations -- diff --git a/bin/pg_reorg.c b/bin/pg_reorg.c index 4d3c2bc..55c42fc 100755 --- a/bin/pg_reorg.c +++ b/bin/pg_reorg.c @@ -1,14 +1,15 @@ /* * pg_reorg.c: bin/pg_reorg.c * - * Copyright (c) 2008-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2011, Itagaki Takahiro */ /** * @brief Client Modules */ -const char *PROGRAM_VERSION = "1.1.5"; +const char *PROGRAM_VERSION = "1.1.6"; const char *PROGRAM_URL = "http://reorg.projects.postgresql.org/"; const char *PROGRAM_EMAIL = "reorg-general@lists.pgfoundry.org"; @@ -19,6 +20,10 @@ const char *PROGRAM_EMAIL = "reorg-general@lists.pgfoundry.org"; #include #include +/* + * APPLY_COUNT: Number of applied logs per transaction. Larger values + * could be faster, but will be long transactions in the REDO phase. + */ #define APPLY_COUNT 1000 #define SQL_XID_SNAPSHOT_80300 \ @@ -60,6 +65,7 @@ typedef struct reorg_table const char *create_log; /* CREATE TABLE log */ const char *create_trigger; /* CREATE TRIGGER z_reorg_trigger */ const char *create_table; /* CREATE TABLE table AS SELECT */ + const char *drop_columns; /* ALTER TABLE DROP COLUMNs */ const char *delete_log; /* DELETE FROM log */ const char *lock_table; /* LOCK TABLE table */ const char *sql_peek; /* SQL used in flush */ @@ -300,6 +306,7 @@ reorg_one_database(const char *orderby, const char *table) table.create_trigger = getstr(res, i, c++); create_table = getstr(res, i, c++); + table.drop_columns = getstr(res, i, c++); table.delete_log = getstr(res, i, c++); table.lock_table = getstr(res, i, c++); ckey = getstr(res, i, c++); @@ -393,6 +400,7 @@ reorg_one_table(const reorg_table *table, const char *orderby) elog(DEBUG2, "create_log : %s", table->create_log); elog(DEBUG2, "create_trigger : %s", table->create_trigger); elog(DEBUG2, "create_table : %s", table->create_table); + elog(DEBUG2, "drop_columns : %s", table->drop_columns ? table->drop_columns : "(skipped)"); elog(DEBUG2, "delete_log : %s", table->delete_log); elog(DEBUG2, "lock_table : %s", table->lock_table); elog(DEBUG2, "sql_peek : %s", table->sql_peek); @@ -450,6 +458,8 @@ reorg_one_table(const reorg_table *table, const char *orderby) command(table->delete_log, 0, NULL); command(table->create_table, 0, NULL); printfStringInfo(&sql, "SELECT reorg.disable_autovacuum('reorg.table_%u')", table->target_oid); + if (table->drop_columns) + command(table->drop_columns, 0, NULL); command(sql.data, 0, NULL); command("COMMIT", 0, NULL); @@ -635,7 +645,7 @@ reorg_cleanup(bool fatal, void *userdata) if (fatal) { - fprintf(stderr, "!!!FATAL ERROR!!! Please refer to a manual.\n\n"); + fprintf(stderr, "!!!FATAL ERROR!!! Please refer to the manual.\n\n"); } else { @@ -671,6 +681,6 @@ pgut_help(bool details) printf(" -t, --table=TABLE reorg specific table only\n"); printf(" -n, --no-order do vacuum full instead of cluster\n"); printf(" -o, --order-by=columns order by columns instead of cluster keys\n"); - printf(" -T, --wait-timeout=secs timeout to cancel other backends on conflict.\n"); + printf(" -T, --wait-timeout=secs timeout to cancel other backends on conflict\n"); printf(" -Z, --no-analyze don't analyze at end\n"); } diff --git a/bin/pgut/pgut-fe.c b/bin/pgut/pgut-fe.c index c2a5604..7136a2a 100755 --- a/bin/pgut/pgut-fe.c +++ b/bin/pgut/pgut-fe.c @@ -1,9 +1,8 @@ /*------------------------------------------------------------------------- - * * pgut-fe.c * - * Copyright (c) 2009-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * + * Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2011, Itagaki Takahiro *------------------------------------------------------------------------- */ diff --git a/bin/pgut/pgut-fe.h b/bin/pgut/pgut-fe.h index 48d48ef..89807d7 100755 --- a/bin/pgut/pgut-fe.h +++ b/bin/pgut/pgut-fe.h @@ -1,9 +1,8 @@ /*------------------------------------------------------------------------- - * * pgut-fe.h * - * Copyright (c) 2009-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * + * Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2011, Itagaki Takahiro *------------------------------------------------------------------------- */ diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 5e00c7b..64ed480 100755 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -1,9 +1,8 @@ /*------------------------------------------------------------------------- - * * pgut.c * - * Copyright (c) 2009-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * + * Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2011, Itagaki Takahiro *------------------------------------------------------------------------- */ diff --git a/bin/pgut/pgut.h b/bin/pgut/pgut.h index f34548f..60a2a21 100755 --- a/bin/pgut/pgut.h +++ b/bin/pgut/pgut.h @@ -1,9 +1,8 @@ /*------------------------------------------------------------------------- - * * pgut.h * - * Copyright (c) 2009-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * + * Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2011, Itagaki Takahiro *------------------------------------------------------------------------- */ diff --git a/bin/sql/reorg.sql b/bin/sql/reorg.sql index ea05ada..1760c98 100755 --- a/bin/sql/reorg.sql +++ b/bin/sql/reorg.sql @@ -80,6 +80,8 @@ ALTER TABLE tbl_with_dropped_column DROP COLUMN d1; ALTER TABLE tbl_with_dropped_column DROP COLUMN d2; ALTER TABLE tbl_with_dropped_column DROP COLUMN d3; ALTER TABLE tbl_with_dropped_column ADD COLUMN c3 text; +CREATE VIEW view_for_dropped_column AS + SELECT * FROM tbl_with_dropped_column; INSERT INTO tbl_with_dropped_toast VALUES(1, 10, 'abc'); INSERT INTO tbl_with_dropped_toast VALUES(2, 20, sqrt(2::numeric(1000,999))::text || sqrt(3::numeric(1000,999))::text); @@ -89,6 +91,7 @@ ALTER TABLE tbl_with_dropped_toast DROP COLUMN t; -- SELECT * FROM tbl_with_dropped_column; +SELECT * FROM view_for_dropped_column; SELECT * FROM tbl_with_dropped_toast; -- @@ -114,8 +117,19 @@ SELECT col1, to_char("time", 'YYYY-MM-DD HH24:MI:SS'), ","")" FROM tbl_cluster; SELECT * FROM tbl_only_ckey ORDER BY 1; SELECT * FROM tbl_only_pkey ORDER BY 1; SELECT * FROM tbl_gistkey ORDER BY 1; + +SET enable_seqscan = on; +SET enable_indexscan = off; SELECT * FROM tbl_with_dropped_column; +SELECT * FROM view_for_dropped_column; SELECT * FROM tbl_with_dropped_toast; +SET enable_seqscan = off; +SET enable_indexscan = on; +SELECT * FROM tbl_with_dropped_column; +SELECT * FROM view_for_dropped_column; +SELECT * FROM tbl_with_dropped_toast; +RESET enable_seqscan; +RESET enable_indexscan; -- -- check broken links or orphan toast relations diff --git a/doc/index-ja.html b/doc/index-ja.html index 8d4471b..2ec9b72 100755 --- a/doc/index-ja.html +++ b/doc/index-ja.html @@ -121,7 +121,10 @@ CREATE INDEX idx_rnd ON tbl (rndkey);
-

Copyright (c) 2008-2010, NIPPON TELEGRAPH AND TELEPHONE CORPORATION

+

+Portions Copyright (c) 2008-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
+Portions Copyright (c) 2011, Itagaki Takahiro +