farrokhi/pg_repack
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix data corruption bug reported by robjderr (#1010664).

Browse Source 
pg_reorg broke catalog definition if the target table had any dropped columns.
Now pg_reorg removes dropped columns and renumbers valid columns.
You can use pg_reorg to shrink column definitions if you have many dropped
columns. (without pg_reorg, dropped columns are filled with zero forever)
This commit is contained in:
Takahiro Itagaki
2009-07-02 09:50:58 +00:00
parent 5fe3f037be
commit 6155932b60
10 changed files with 251 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
bin/expected/reorg.out
View File

@ -6,7 +6,7 @@ CREATE TABLE tbl_cluster (
col1 int,
col2 timestamp,
":-)" text,
primary key(":-)", col1)
PRIMARY KEY (":-)", col1)
) WITH (fillfactor = 70);
CREATE INDEX cidx_cluster ON tbl_cluster (col2, length(":-)"));
ALTER TABLE tbl_cluster CLUSTER ON cidx_cluster;
@ -27,6 +27,17 @@ CREATE TABLE tbl_gistkey (
);
CREATE INDEX cidx_circle ON tbl_gistkey USING gist (c);
ALTER TABLE tbl_gistkey CLUSTER ON cidx_circle;
CREATE TABLE tbl_with_dropped_column (
d1 text,
c1 text,
id integer PRIMARY KEY,
d2 text,
c2 text,
d3 text
);
ALTER TABLE tbl_with_dropped_column CLUSTER ON tbl_with_dropped_column_pkey;
CREATE INDEX idx_c1c2 ON tbl_with_dropped_column (c1, c2);
CREATE INDEX idx_c2c1 ON tbl_with_dropped_column (c2, c1);
--
-- insert data
--
@ -41,6 +52,22 @@ INSERT INTO tbl_only_ckey VALUES(1, '2008-01-01 00:00:00', 'abc');
INSERT INTO tbl_only_ckey VALUES(2, '2008-02-01 00:00:00', 'def');
INSERT INTO tbl_gistkey VALUES(1, '<(1,2),3>');
INSERT INTO tbl_gistkey VALUES(2, '<(4,5),6>');
INSERT INTO tbl_with_dropped_column VALUES('d1', 'c1', 2, 'd2', 'c2', 'd3');
INSERT INTO tbl_with_dropped_column VALUES('d1', 'c1', 1, 'd2', 'c2', 'd3');
ALTER TABLE tbl_with_dropped_column DROP COLUMN d1;
ALTER TABLE tbl_with_dropped_column DROP COLUMN d2;
ALTER TABLE tbl_with_dropped_column DROP COLUMN d3;
ALTER TABLE tbl_with_dropped_column ADD COLUMN c3 text;
--
-- before
--
SELECT * FROM tbl_with_dropped_column;
c1 | id | c2 | c3
----+----+----+----
c1 | 2 | c2 |
c1 | 1 | c2 |
(2 rows)
--
-- do reorg
--
@ -48,7 +75,7 @@ INSERT INTO tbl_gistkey VALUES(2, '<(4,5),6>');
\! pg_reorg --dbname=contrib_regression
\! pg_reorg --dbname=contrib_regression --table=tbl_cluster
--
-- results
-- after
--
\d tbl_cluster
Table "public.tbl_cluster"
@ -90,6 +117,19 @@ Indexes:
Indexes:
"tbl_only_pkey_pkey" PRIMARY KEY, btree (col1)
\d tbl_with_dropped_column
Table "public.tbl_with_dropped_column"
Column | Type | Modifiers
--------+---------+-----------
c1 | text |
id | integer | not null
c2 | text |
c3 | text |
Indexes:
"tbl_with_dropped_column_pkey" PRIMARY KEY, btree (id) CLUSTER
"idx_c1c2" btree (c1, c2)
"idx_c2c1" btree (c2, c1)
SELECT col1, to_char(col2, 'YYYY-MM-DD HH24:MI:SS'), ":-)" FROM tbl_cluster;
col1 | to_char | :-)
------+---------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@ -121,6 +161,13 @@ SELECT * FROM tbl_gistkey ORDER BY 1;
2 | <(4,5),6>
(2 rows)
SELECT * FROM tbl_with_dropped_column;
c1 | id | c2 | c3
----+----+----+----
c1 | 1 | c2 |
c1 | 2 | c2 |
(2 rows)
--
-- clean up
--
@ -128,4 +175,5 @@ DROP TABLE tbl_cluster;
DROP TABLE tbl_only_pkey;
DROP TABLE tbl_only_ckey;
DROP TABLE tbl_gistkey;
DROP TABLE tbl_with_dropped_column;
RESET client_min_messages;

25
bin/pg_reorg.c
View File

@ -8,7 +8,7 @@
* @brief Client Modules
*/
const char *PROGRAM_VERSION = "1.0.5";
const char *PROGRAM_VERSION = "1.0.6";
const char *PROGRAM_URL = "http://reorg.projects.postgresql.org/";
const char *PROGRAM_EMAIL = "reorg-general@lists.pgfoundry.org";
@ -93,7 +93,6 @@ static bool sqlstate_equals(PGresult *res, const char *state)
}
static bool verbose = false;
static bool quiet = false;
static bool analyze = true;
/*
@ -111,7 +110,6 @@ utoa(unsigned int value, char *buffer)
}
const struct option pgut_options[] = {
{"quiet", no_argument, NULL, 'q'},
{"verbose", no_argument, NULL, 'v'},
{"all", no_argument, NULL, 'a'},
{"table", required_argument, NULL, 't'},
@ -130,9 +128,6 @@ pgut_argument(int c, const char *arg)
{
switch (c)
{
case 'q':
quiet = true;
break;
case 'v':
verbose = true;
break;
@ -587,14 +582,17 @@ reorg_one_table(const reorg_table *table, const char *orderby)
* Note that current_table is already set to NULL here because analyze
* is an unimportant operation; No clean up even if failed.
*/
if (verbose)
fprintf(stderr, "---- analyze ----\n");
if (analyze)
{
if (verbose)
fprintf(stderr, "---- analyze ----\n");
command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL);
printfStringInfo(&sql, "ANALYZE %s%s",
(verbose ? "VERBOSE " : ""), table->target_name);
command(sql.data, 0, NULL);
command("COMMIT", 0, NULL);
command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL);
printfStringInfo(&sql, "ANALYZE %s%s",
(verbose ? "VERBOSE " : ""), table->target_name);
command(sql.data, 0, NULL);
command("COMMIT", 0, NULL);
}
termStringInfo(&sql);
}
@ -643,7 +641,6 @@ pgut_help(void)
" -n, --no-order do vacuum full instead of cluster\n"
" -o, --order-by=columns order by columns instead of cluster keys\n"
" -Z, --no-analyze don't analyze at end\n"
" -q, --quiet don't write any messages\n"
" -v, --verbose display detailed information during processing\n",
PROGRAM_NAME, PROGRAM_NAME);
}

37
bin/pgut/pgut.c
View File

@ -22,6 +22,7 @@ const char *port = NULL;
const char *username = NULL;
bool password = false;
bool debug = false;
bool quiet = false;
/* Database connections */
PGconn *connection = NULL;
@ -45,6 +46,7 @@ const struct option default_options[] =
{"dbname", required_argument, NULL, 'd'},
{"host", required_argument, NULL, 'h'},
{"port", required_argument, NULL, 'p'},
{"quiet", no_argument, NULL, 'q'},
{"username", required_argument, NULL, 'U'},
{"password", no_argument, NULL, 'W'},
{"debug", no_argument, NULL, '!'},
@ -136,6 +138,9 @@ parse_options(int argc, char **argv)
case 'p':
assign_option(&port, c, optarg);
break;
case 'q':
quiet = true;
break;
case 'U':
assign_option(&username, c, optarg);
break;
@ -196,13 +201,22 @@ assign_option(const char **value, int c, const char *arg)
return true;
}
void
reconnect(void)
/*
* the result is also available with the global variable 'connection'.
*/
PGconn *
reconnect_elevel(int elevel)
{
PGconn *conn;
char *pwd = NULL;
bool new_pass;
if (interrupted)
{
interrupted = false;
elog(ERROR, "%s: interrupted", PROGRAM_NAME);
}
disconnect();
if (password)
@ -218,7 +232,10 @@ reconnect(void)
conn = PQsetdbLogin(host, port, NULL, NULL, dbname, username, pwd);
if (!conn)
elog(ERROR, "could not connect to database %s", dbname);
{
elog(elevel, "could not connect to database %s", dbname);
return NULL;
}
if (PQstatus(conn) == CONNECTION_BAD &&
#if PG_VERSION_NUM >= 80300
@ -239,10 +256,17 @@ reconnect(void)
/* check to see that the backend connection was successfully made */
if (PQstatus(conn) == CONNECTION_BAD)
elog(ERROR, "could not connect to database %s: %s",
elog(elevel, "could not connect to database %s: %s",
dbname, PQerrorMessage(conn));
connection = conn;
return conn;
}
void
reconnect(void)
{
reconnect_elevel(ERROR);
}
void
@ -290,6 +314,7 @@ execute_elevel(const char *query, int nParams, const char **params, int elevel)
{
case PGRES_TUPLES_OK:
case PGRES_COMMAND_OK:
case PGRES_COPY_IN:
break;
default:
elog(elevel, "query failed: %squery was: %s",
@ -329,6 +354,8 @@ elog(int elevel, const char *fmt, ...)
if (!debug && elevel <= LOG)
return;
if (quiet && elevel <= WARNING)
return;
switch (elevel)
{
@ -481,6 +508,7 @@ static void help(void)
fprintf(stderr, " -U, --username=USERNAME user name to connect as\n");
fprintf(stderr, " -W, --password force password prompt\n");
fprintf(stderr, "\nGeneric options:\n");
fprintf(stderr, " -q, --quiet don't write any messages\n");
fprintf(stderr, " --debug debug mode\n");
fprintf(stderr, " --help show this help, then exit\n");
fprintf(stderr, " --version output version information, then exit\n\n");
@ -566,3 +594,4 @@ sleep(unsigned int seconds)
}
#endif /* WIN32 */

28
bin/pgut/pgut.h
View File

@ -13,6 +13,7 @@
#include "libpq-fe.h"
#include "pqexpbuffer.h"
#include <assert.h>
#include <getopt.h>
#if !defined(C_H) && !defined(__cplusplus)
@ -51,6 +52,7 @@ extern const char *port;
extern const char *username;
extern bool password;
extern bool debug;
extern bool quiet;
extern PGconn *connection;
extern bool interrupted;
@ -58,6 +60,8 @@ extern bool interrupted;
extern void parse_options(int argc, char **argv);
extern bool assign_option(const char **value, int c, const char *arg);
extern PGconn *reconnect_elevel(int elevel);
extern void reconnect(void);
extern void disconnect(void);
extern PGresult *execute_elevel(const char *query, int nParams, const char **params, int elevel);
@ -68,6 +72,15 @@ extern void command(const char *query, int nParams, const char **params);
extern unsigned int sleep(unsigned int seconds);
#endif
/*
* IsXXX
*/
#define IsSpace(c) (isspace((unsigned char)(c)))
#define IsAlpha(c) (isalpha((unsigned char)(c)))
#define IsAlnum(c) (isalnum((unsigned char)(c)))
#define IsIdentHead(c) (IsAlpha(c) || (c) == '_')
#define IsIdentBody(c) (IsAlnum(c) || (c) == '_')
/*
* elog
*/
@ -104,6 +117,20 @@ __attribute__((format(printf, 2, 3)));
#define appendStringInfoChar appendPQExpBufferChar
#define appendBinaryStringInfo appendBinaryPQExpBuffer
/*
* Assert
*/
#undef Assert
#undef AssertMacro
#ifdef USE_ASSERT_CHECKING
#define Assert(x) assert(x)
#define AssertMacro(x) assert(x)
#else
#define Assert(x) ((void) 0)
#define AssertMacro(x) ((void) 0)
#endif
/*
* import from postgres.h and catalog/genbki.h in 8.4
*/
@ -124,3 +151,4 @@ typedef int aclitem;
#endif
#endif /* PGUT_H */

31
bin/sql/reorg.sql
View File

@ -6,7 +6,7 @@ CREATE TABLE tbl_cluster (
col1 int,
col2 timestamp,
":-)" text,
primary key(":-)", col1)
PRIMARY KEY (":-)", col1)
) WITH (fillfactor = 70);
CREATE INDEX cidx_cluster ON tbl_cluster (col2, length(":-)"));
@ -34,6 +34,18 @@ CREATE TABLE tbl_gistkey (
CREATE INDEX cidx_circle ON tbl_gistkey USING gist (c);
ALTER TABLE tbl_gistkey CLUSTER ON cidx_circle;
CREATE TABLE tbl_with_dropped_column (
d1 text,
c1 text,
id integer PRIMARY KEY,
d2 text,
c2 text,
d3 text
);
ALTER TABLE tbl_with_dropped_column CLUSTER ON tbl_with_dropped_column_pkey;
CREATE INDEX idx_c1c2 ON tbl_with_dropped_column (c1, c2);
CREATE INDEX idx_c2c1 ON tbl_with_dropped_column (c2, c1);
--
-- insert data
--
@ -53,6 +65,18 @@ INSERT INTO tbl_only_ckey VALUES(2, '2008-02-01 00:00:00', 'def');
INSERT INTO tbl_gistkey VALUES(1, '<(1,2),3>');
INSERT INTO tbl_gistkey VALUES(2, '<(4,5),6>');
INSERT INTO tbl_with_dropped_column VALUES('d1', 'c1', 2, 'd2', 'c2', 'd3');
INSERT INTO tbl_with_dropped_column VALUES('d1', 'c1', 1, 'd2', 'c2', 'd3');
ALTER TABLE tbl_with_dropped_column DROP COLUMN d1;
ALTER TABLE tbl_with_dropped_column DROP COLUMN d2;
ALTER TABLE tbl_with_dropped_column DROP COLUMN d3;
ALTER TABLE tbl_with_dropped_column ADD COLUMN c3 text;
--
-- before
--
SELECT * FROM tbl_with_dropped_column;
--
-- do reorg
--
@ -62,18 +86,20 @@ INSERT INTO tbl_gistkey VALUES(2, '<(4,5),6>');
\! pg_reorg --dbname=contrib_regression --table=tbl_cluster
--
-- results
-- after
--
\d tbl_cluster
\d tbl_gistkey
\d tbl_only_ckey
\d tbl_only_pkey
\d tbl_with_dropped_column
SELECT col1, to_char(col2, 'YYYY-MM-DD HH24:MI:SS'), ":-)" FROM tbl_cluster;
SELECT * FROM tbl_only_ckey ORDER BY 1;
SELECT * FROM tbl_only_pkey ORDER BY 1;
SELECT * FROM tbl_gistkey ORDER BY 1;
SELECT * FROM tbl_with_dropped_column;
--
-- clean up
@ -83,4 +109,5 @@ DROP TABLE tbl_cluster;
DROP TABLE tbl_only_pkey;
DROP TABLE tbl_only_ckey;
DROP TABLE tbl_gistkey;
DROP TABLE tbl_with_dropped_column;
RESET client_min_messages;