diff --git a/META.json b/META.json index ca4c4b1..454e3a5 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.2dev0", + "version": "1.2dev1", "maintainer": [ "Josh Kupershmidt ", "Daniele Varrazzo " @@ -13,7 +13,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.2dev0", + "version": "1.2dev1", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, diff --git a/bin/Makefile b/bin/Makefile index 3e9342b..24ae1cd 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -8,7 +8,7 @@ SRCS = pg_repack.c pgut/pgut.c pgut/pgut-fe.c OBJS = $(SRCS:.c=.o) PROGRAM = pg_repack -REGRESS = init repack +REGRESS = init repack tablespace EXTRA_CLEAN = sql/init-$(MAJORVERSION).sql sql/init.sql diff --git a/bin/expected/tablespace.out b/bin/expected/tablespace.out new file mode 100644 index 0000000..e6daf5c --- /dev/null +++ b/bin/expected/tablespace.out @@ -0,0 +1,76 @@ +SET client_min_messages = warning; +-- +-- Tablespace features tests +-- +-- Note: in order to pass this test you must create a tablespace called 'testts' +-- +SELECT spcname FROM pg_tablespace WHERE spcname = 'testts'; + spcname +--------- + testts +(1 row) + +-- If the query above failed you must create the 'testts' tablespace; +CREATE TABLE testts1 (id serial primary key, data text); +CREATE INDEX testts1_partial_idx on testts1 (id) where (id > 0); +INSERT INTO testts1 (data) values ('a'); +INSERT INTO testts1 (data) values ('b'); +INSERT INTO testts1 (data) values ('c'); +-- can move the tablespace from default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- + testts1 | testts +(1 row) + +SELECT * from testts1 order by id; + id | data +----+------ + 1 | a + 2 | b + 3 | c +(3 rows) + +-- tablespace stays where it is +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- + testts1 | testts +(1 row) + +-- can move the ts back to default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -s pg_default +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- +(0 rows) + +-- can move the table together with the indexes +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts --moveidx +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------------------+--------- + testts1 | testts + testts1_partial_idx | testts + testts1_pkey | testts +(3 rows) + +-- can't specify --moveidx without --tablespace +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --moveidx +ERROR: cannot specify --moveidx (-S) without --tablespace (-s) +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -S +ERROR: cannot specify --moveidx (-S) without --tablespace (-s) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index d25d3d7..6569bc3 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -172,6 +172,7 @@ typedef struct repack_index } repack_index; static bool is_superuser(void); +static void check_tablespace(void); static void repack_all_databases(const char *order_by); static bool repack_one_database(const char *order_by, char *errbuf, size_t errsize); static void repack_one_table(const repack_table *table, const char *order_by); @@ -197,6 +198,8 @@ static bool alldb = false; static bool noorder = false; static SimpleStringList table_list = {NULL, NULL}; static char *orderby = NULL; +static char *tablespace = NULL; +static bool moveidx = false; static int wait_timeout = 60; /* in seconds */ static int jobs = 0; /* number of concurrent worker conns. */ @@ -214,6 +217,8 @@ static pgut_option options[] = { 'l', 't', "table", &table_list }, { 'b', 'n', "no-order", &noorder }, { 's', 'o', "order-by", &orderby }, + { 's', 's', "tablespace", &tablespace }, + { 'b', 'S', "moveidx", &moveidx }, { 'i', 'T', "wait-timeout", &wait_timeout }, { 'B', 'Z', "no-analyze", &analyze }, { 'i', 'j', "jobs", &jobs }, @@ -234,6 +239,8 @@ main(int argc, char *argv[]) (errcode(EINVAL), errmsg("too many arguments"))); + check_tablespace(); + if (noorder) orderby = ""; @@ -281,6 +288,56 @@ is_superuser(void) return false; } +/* + * Check if the tablespace requested exists. + * + * Raise an exception on error. + */ +void +check_tablespace() +{ + PGresult *res = NULL; + const char *params[1]; + + if (tablespace == NULL) + { + /* nothing to check, but let's see the options */ + if (moveidx) + { + ereport(ERROR, + (errcode(EINVAL), + errmsg("cannot specify --moveidx (-S) without --tablespace (-s)"))); + } + return; + } + + /* check if the tablespace exists */ + reconnect(ERROR); + params[0] = tablespace; + res = execute_elevel( + "select spcname from pg_tablespace where spcname = $1", + 1, params, DEBUG2); + + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + if (PQntuples(res) == 0) + { + ereport(ERROR, + (errcode(EINVAL), + errmsg("the tablespace \"%s\" doesn't exist", tablespace))); + } + } + else + { + ereport(ERROR, + (errcode(EINVAL), + errmsg("error checking the namespace: %s", + PQerrorMessage(connection)))); + } + + CLEARPGRES(res); +} + /* * Call repack_one_database for each database. @@ -360,10 +417,15 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) StringInfoData sql; SimpleStringListCell *cell; const char **params = NULL; - size_t num_params = simple_string_list_size(table_list); + int iparam = 0; + size_t num_tables; + size_t num_params; - if (num_params) - params = pgut_malloc(num_params * sizeof(char *)); + num_tables = simple_string_list_size(table_list); + + /* 1st param is the user-specified tablespace */ + num_params = num_tables + 1; + params = pgut_malloc(num_params * sizeof(char *)); initStringInfo(&sql); @@ -442,29 +504,46 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) command("SET client_min_messages = warning", 0, NULL); /* acquire target tables */ - appendStringInfoString(&sql, "SELECT * FROM repack.tables WHERE "); - if (num_params) + appendStringInfoString(&sql, + "SELECT t.*," + " coalesce(v.tablespace, t.tablespace_orig) as tablespace_dest" + " FROM repack.tables t, " + " (VALUES (quote_ident($1::text))) as v (tablespace)" + " WHERE "); + + params[iparam++] = tablespace; + if (num_tables) { appendStringInfoString(&sql, "("); - for (i = 0, cell = table_list.head; cell; cell = cell->next, i++) + for (cell = table_list.head; cell; cell = cell->next) { /* Construct table name placeholders to be used by PQexecParams */ - appendStringInfo(&sql, "relid = $%d::regclass", i + 1); - params[i] = cell->val; + appendStringInfo(&sql, "relid = $%d::regclass", iparam + 1); + params[iparam++] = cell->val; if (cell->next) appendStringInfoString(&sql, " OR "); } appendStringInfoString(&sql, ")"); - res = execute_elevel(sql.data, (int) num_params, params, DEBUG2); } else { appendStringInfoString(&sql, "pkid IS NOT NULL"); if (!orderby) appendStringInfoString(&sql, " AND ckid IS NOT NULL"); - res = execute_elevel(sql.data, 0, NULL, DEBUG2); } + /* double check the parameters array is sane */ + if (iparam != num_params) + { + if (errbuf) + snprintf(errbuf, errsize, + "internal error: bad parameters count: %i instead of %zi", + iparam, num_params); + goto cleanup; + } + + res = execute_elevel(sql.data, (int) num_params, params, DEBUG2); + /* on error skip the database */ if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -489,7 +568,9 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) for (i = 0; i < num; i++) { repack_table table; - const char *create_table; + const char *create_table_1; + const char *create_table_2; + const char *tablespace; const char *ckey; int c = 0; @@ -512,44 +593,50 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) table.create_trigger = getstr(res, i, c++); table.enable_trigger = getstr(res, i, c++); - create_table = getstr(res, i, c++); + create_table_1 = getstr(res, i, c++); + tablespace = getstr(res, i, c++); /* to be clobbered */ + create_table_2 = getstr(res, i, c++); table.drop_columns = getstr(res, i, c++); table.delete_log = getstr(res, i, c++); table.lock_table = getstr(res, i, c++); ckey = getstr(res, i, c++); - - resetStringInfo(&sql); - if (!orderby) - { - if (ckey != NULL) - { - /* CLUSTER mode */ - appendStringInfo(&sql, "%s ORDER BY %s", create_table, ckey); - table.create_table = sql.data; - } - else - { - /* VACUUM FULL mode (non-clustered tables) */ - table.create_table = create_table; - } - } - else if (!orderby[0]) - { - /* VACUUM FULL mode (for clustered tables too) */ - table.create_table = create_table; - } - else - { - /* User specified ORDER BY */ - appendStringInfo(&sql, "%s ORDER BY %s", create_table, orderby); - table.create_table = sql.data; - } - table.sql_peek = getstr(res, i, c++); table.sql_insert = getstr(res, i, c++); table.sql_delete = getstr(res, i, c++); table.sql_update = getstr(res, i, c++); table.sql_pop = getstr(res, i, c++); + tablespace = getstr(res, i, c++); + + resetStringInfo(&sql); + appendStringInfoString(&sql, create_table_1); + appendStringInfoString(&sql, tablespace); + appendStringInfoString(&sql, create_table_2); + if (!orderby) + { + if (ckey != NULL) + { + /* CLUSTER mode */ + appendStringInfoString(&sql, " ORDER BY "); + appendStringInfoString(&sql, ckey); + table.create_table = sql.data; + } + else + { + /* VACUUM FULL mode (non-clustered tables) */ + table.create_table = sql.data; + } + } + else if (!orderby[0]) + { + /* VACUUM FULL mode (for clustered tables too) */ + table.create_table = sql.data; + } + else + { + /* User specified ORDER BY */ + appendStringInfoString(&sql, " ORDER BY "); + appendStringInfoString(&sql, orderby); + } repack_one_table(&table, orderby); } @@ -594,7 +681,7 @@ static bool rebuild_indexes(const repack_table *table) { PGresult *res; - const char *params[1]; + const char *params[2]; int num_indexes; int i; int num_active_workers; @@ -606,6 +693,7 @@ rebuild_indexes(const repack_table *table) elog(DEBUG2, "---- create indexes ----"); params[0] = utoa(table->target_oid, buffer); + params[1] = moveidx ? tablespace : NULL; /* First, just display a warning message for any invalid indexes * which may be on the table (mostly to match the behavior of 1.1.8). @@ -621,8 +709,9 @@ rebuild_indexes(const repack_table *table) } res = execute("SELECT indexrelid," - " repack.repack_indexdef(indexrelid, indrelid) " - " FROM pg_index WHERE indrelid = $1 AND indisvalid", 1, params); + " repack.repack_indexdef(indexrelid, indrelid, $2) " + " FROM pg_index WHERE indrelid = $1 AND indisvalid", + 2, params); num_indexes = PQntuples(res); @@ -1453,6 +1542,8 @@ pgut_help(bool details) printf("Options:\n"); printf(" -a, --all repack all databases\n"); printf(" -t, --table=TABLE repack specific table only\n"); + printf(" -s, --tablespace=TABLESPC move repacked tables to a new tablespace\n"); + printf(" -S, --moveidx move repacked indexes to TABLESPC too\n"); printf(" -o, --order-by=COLUMNS order by columns instead of cluster keys\n"); printf(" -n, --no-order do vacuum full instead of cluster\n"); printf(" -j --jobs Use this many parallel jobs for each table\n"); diff --git a/bin/sql/tablespace.sql b/bin/sql/tablespace.sql new file mode 100644 index 0000000..73400e7 --- /dev/null +++ b/bin/sql/tablespace.sql @@ -0,0 +1,55 @@ +SET client_min_messages = warning; + +-- +-- Tablespace features tests +-- +-- Note: in order to pass this test you must create a tablespace called 'testts' +-- + +SELECT spcname FROM pg_tablespace WHERE spcname = 'testts'; +-- If the query above failed you must create the 'testts' tablespace; + +CREATE TABLE testts1 (id serial primary key, data text); +CREATE INDEX testts1_partial_idx on testts1 (id) where (id > 0); +INSERT INTO testts1 (data) values ('a'); +INSERT INTO testts1 (data) values ('b'); +INSERT INTO testts1 (data) values ('c'); + +-- can move the tablespace from default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts + +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + +SELECT * from testts1 order by id; + +-- tablespace stays where it is +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 + +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + +-- can move the ts back to default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -s pg_default + +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + +-- can move the table together with the indexes +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts --moveidx + +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + +-- can't specify --moveidx without --tablespace +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --moveidx +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -S + diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index 4d2e38e..d066eb5 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -118,6 +118,8 @@ The following options can be specified in ``OPTIONS``. Options: -a, --all repack all databases -t, --table=TABLE repack specific table only + -s, --tablespace=TABLESPC move repacked tables to a new tablespace + -S, --moveidx move repacked indexes to TABLESPC too -o, --order-by=COLUMNS order by columns instead of cluster keys -n, --no-order do vacuum full instead of cluster -j, --jobs Use this many parallel jobs for each table @@ -163,6 +165,15 @@ Reorg Options on each table. If your PostgreSQL server has extra cores and disk I/O available, this can be a useful way to speed up pg_repack. +``-s TABLESPC``, ``--tablespace=TABLESPC`` + Move the repacked tables to the specified tablespace: essentially an + online version of ``ALTER TABLE ... SET TABLESPACE``. The tables indexes + are left on the original tablespace unless ``--moveidx`` is specified too. + +``-S``, ``--moveidx`` + Move the indexes too of the repacked tables to the tablespace specified + by the option ``--tablespace``. + ``-T SECS``, ``--wait-timeout=SECS`` pg_repack needs to take an exclusive lock at the end of the reorganization. This setting controls how many seconds pg_repack will @@ -405,6 +416,7 @@ Releases * pg_repack 1.2 + * Added --tablespace and --moveidx options to perform online SET TABLESPACE. * Added --jobs option for parallel operation. * Don't require --no-order to perform a VACUUM FULL on non-clustered tables (pg_repack issue #6). diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index cfc9512..a223976 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -179,7 +179,9 @@ CREATE VIEW repack.tables AS 'CREATE TABLE repack.log_' || R.oid || ' (id bigserial PRIMARY KEY, pk repack.pk_' || R.oid || ', row ' || repack.oid2text(R.oid) || ')' AS create_log, repack.get_create_trigger(R.oid, PK.indexrelid) AS create_trigger, repack.get_enable_trigger(R.oid) as enable_trigger, - 'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE ' || coalesce(quote_ident(S.spcname), 'pg_default') || ' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table, + 'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE ' AS create_table_1, + coalesce(quote_ident(S.spcname), 'pg_default') tablespace_orig, + ' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table_2, repack.get_drop_columns(R.oid, 'repack.table_' || R.oid) AS drop_columns, 'DELETE FROM repack.log_' || R.oid AS delete_log, 'LOCK TABLE ' || repack.oid2text(R.oid) || ' IN ACCESS EXCLUSIVE MODE' AS lock_table, @@ -205,9 +207,9 @@ CREATE VIEW repack.tables AS AND N.nspname NOT IN ('pg_catalog', 'information_schema') AND N.nspname NOT LIKE E'pg\\_temp\\_%'; -CREATE FUNCTION repack.repack_indexdef(oid, oid) RETURNS text AS +CREATE FUNCTION repack.repack_indexdef(oid, oid, name) RETURNS text AS 'MODULE_PATHNAME', 'repack_indexdef' -LANGUAGE C STABLE STRICT; +LANGUAGE C STABLE; CREATE FUNCTION repack.repack_trigger() RETURNS trigger AS 'MODULE_PATHNAME', 'repack_trigger' diff --git a/lib/repack.c b/lib/repack.c index 1bfec43..19cb089 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -618,20 +618,59 @@ repack_get_order_by(PG_FUNCTION_ARGS) * * @param index Oid of target index. * @param table Oid of table of the index. + * @param tablespace Namespace for the index. If NULL keep the original. * @retval Create index DDL for temp table. */ Datum repack_indexdef(PG_FUNCTION_ARGS) { - Oid index = PG_GETARG_OID(0); - Oid table = PG_GETARG_OID(1); + Oid index; + Oid table; + Name tablespace = NULL; IndexDef stmt; StringInfoData str; + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + PG_RETURN_NULL(); + + index = PG_GETARG_OID(0); + table = PG_GETARG_OID(1); + + if (!PG_ARGISNULL(2)) + tablespace = PG_GETARG_NAME(2); + parse_indexdef(&stmt, index, table); + initStringInfo(&str); - appendStringInfo(&str, "%s index_%u ON repack.table_%u USING %s (%s)%s", - stmt.create, index, table, stmt.type, stmt.columns, stmt.options); + appendStringInfo(&str, "%s index_%u ON repack.table_%u USING %s (%s)", + stmt.create, index, table, stmt.type, stmt.columns); + + /* Replace the tablespace in the index options */ + if (tablespace == NULL) + { + /* tablespace is just fine */ + appendStringInfoString(&str, stmt.options); + } + else + { + if (NULL == strstr(stmt.options, "TABLESPACE")) + { + /* tablespace is to append */ + appendStringInfoString(&str, " TABLESPACE "); + appendStringInfoString(&str, NameStr(*tablespace)); + } + else + { + /* tablespace is to replace */ + char *tmp; + tmp = skip_const(index, stmt.options, " TABLESPACE", NULL); + appendStringInfoString(&str, stmt.options); + appendStringInfo(&str, " %s", NameStr(*tablespace)); + tmp = skip_ident(index, tmp); + if (*tmp) + appendStringInfo(&str, " %s", tmp); + } + } PG_RETURN_TEXT_P(cstring_to_text(str.data)); }