Merge branch 'change-tablespace'

Conflicts: bin/pg_repack.c doc/pg_repack.rst
2013-04-17 09:07:09 +01:00 · 2013-04-17 09:07:09 +01:00 · 42c65d16f8
commit 42c65d16f8
parent 7617e07f10 b6bd078e92
8 changed files with 328 additions and 53 deletions
--- a/META.json
+++ b/META.json
@ -2,7 +2,7 @@
   "name": "pg_repack",
   "abstract": "PostgreSQL module for data reorganization",
   "description": "Reorganize tables in PostgreSQL databases with minimal locks",
-   "version": "1.2dev0",
+   "version": "1.2dev1",
   "maintainer": [
       "Josh Kupershmidt <schmiddy@gmail.com>",
       "Daniele Varrazzo <daniele.varrazzo@gmail.com>"
@ -13,7 +13,7 @@
   "provides": {
      "pg_repack": {
         "file": "lib/pg_repack.sql",
-         "version": "1.2dev0",
+         "version": "1.2dev1",
         "abstract": "Reorganize tables in PostgreSQL databases with minimal locks"
      }
   },
--- a/bin/Makefile
+++ b/bin/Makefile
@ -8,7 +8,7 @@
 SRCS = pg_repack.c pgut/pgut.c pgut/pgut-fe.c
 OBJS = $(SRCS:.c=.o)
 PROGRAM = pg_repack
-REGRESS = init repack
+REGRESS = init repack tablespace

 EXTRA_CLEAN = sql/init-$(MAJORVERSION).sql sql/init.sql

--- a/bin/expected/tablespace.out
+++ b/bin/expected/tablespace.out
@ -0,0 +1,76 @@
+SET client_min_messages = warning;
+--
+-- Tablespace features tests
+--
+-- Note: in order to pass this test you must create a tablespace called 'testts'
+--
+SELECT spcname FROM pg_tablespace WHERE spcname = 'testts';
+ spcname 
+---------
+ testts
+(1 row)
+
+-- If the query above failed you must create the 'testts' tablespace;
+CREATE TABLE testts1 (id serial primary key, data text);
+CREATE INDEX testts1_partial_idx on testts1 (id) where (id > 0);
+INSERT INTO testts1 (data) values ('a');
+INSERT INTO testts1 (data) values ('b');
+INSERT INTO testts1 (data) values ('c');
+-- can move the tablespace from default
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+ relname | spcname 
+---------+---------
+ testts1 | testts
+(1 row)
+
+SELECT * from testts1 order by id;
+ id | data 
+----+------
+  1 | a
+  2 | b
+  3 | c
+(3 rows)
+
+-- tablespace stays where it is
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+ relname | spcname 
+---------+---------
+ testts1 | testts
+(1 row)
+
+-- can move the ts back to default
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -s pg_default
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+ relname | spcname 
+---------+---------
+(0 rows)
+
+-- can move the table together with the indexes
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts --moveidx
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+       relname       | spcname 
+---------------------+---------
+ testts1             | testts
+ testts1_partial_idx | testts
+ testts1_pkey        | testts
+(3 rows)
+
+-- can't specify --moveidx without --tablespace
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --moveidx
+ERROR: cannot specify --moveidx (-S) without --tablespace (-s)
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -S
+ERROR: cannot specify --moveidx (-S) without --tablespace (-s)
--- a/bin/pg_repack.c
+++ b/bin/pg_repack.c
@ -172,6 +172,7 @@ typedef struct repack_index
 } repack_index;

 static bool is_superuser(void);
+static void check_tablespace(void);
 static void repack_all_databases(const char *order_by);
 static bool repack_one_database(const char *order_by, char *errbuf, size_t errsize);
 static void repack_one_table(const repack_table *table, const char *order_by);
@ -197,6 +198,8 @@ static bool				alldb = false;
 static bool				noorder = false;
 static SimpleStringList	table_list = {NULL, NULL};
 static char				*orderby = NULL;
+static char				*tablespace = NULL;
+static bool				moveidx = false;
 static int				wait_timeout = 60;	/* in seconds */
 static int				jobs = 0;	/* number of concurrent worker conns. */

@ -214,6 +217,8 @@ static pgut_option options[] =
 	{ 'l', 't', "table", &table_list },
 	{ 'b', 'n', "no-order", &noorder },
 	{ 's', 'o', "order-by", &orderby },
+	{ 's', 's', "tablespace", &tablespace },
+	{ 'b', 'S', "moveidx", &moveidx },
 	{ 'i', 'T', "wait-timeout", &wait_timeout },
 	{ 'B', 'Z', "no-analyze", &analyze },
 	{ 'i', 'j', "jobs", &jobs },
@ -234,6 +239,8 @@ main(int argc, char *argv[])
 			(errcode(EINVAL),
 			 errmsg("too many arguments")));

+	check_tablespace();
+
 	if (noorder)
 		orderby = "";

@ -281,6 +288,56 @@ is_superuser(void)
 	return false;
 }

+/*
+ * Check if the tablespace requested exists.
+ *
+ * Raise an exception on error.
+ */
+void
+check_tablespace()
+{
+	PGresult		*res = NULL;
+	const char *params[1];
+
+	if (tablespace == NULL)
+	{
+		/* nothing to check, but let's see the options */
+		if (moveidx)
+		{
+			ereport(ERROR,
+				(errcode(EINVAL),
+				 errmsg("cannot specify --moveidx (-S) without --tablespace (-s)")));
+		}
+		return;
+	}
+
+	/* check if the tablespace exists */
+	reconnect(ERROR);
+	params[0] = tablespace;
+	res = execute_elevel(
+		"select spcname from pg_tablespace where spcname = $1",
+		1, params, DEBUG2);
+
+	if (PQresultStatus(res) == PGRES_TUPLES_OK)
+	{
+		if (PQntuples(res) == 0)
+		{
+			ereport(ERROR,
+				(errcode(EINVAL),
+				 errmsg("the tablespace \"%s\" doesn't exist", tablespace)));
+		}
+	}
+	else
+	{
+		ereport(ERROR,
+			(errcode(EINVAL),
+			 errmsg("error checking the namespace: %s",
+				 PQerrorMessage(connection))));
+	}
+
+	CLEARPGRES(res);
+}
+

 /*
 * Call repack_one_database for each database.
@ -360,10 +417,15 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
 	StringInfoData			sql;
 	SimpleStringListCell   *cell;
 	const char			  **params = NULL;
-	size_t					num_params = simple_string_list_size(table_list);
+	int						iparam = 0;
+	size_t					num_tables;
+	size_t					num_params;

-	if (num_params)
-		params = pgut_malloc(num_params * sizeof(char *));
+	num_tables = simple_string_list_size(table_list);
+
+	/* 1st param is the user-specified tablespace */
+	num_params = num_tables + 1;
+	params = pgut_malloc(num_params * sizeof(char *));

 	initStringInfo(&sql);

@ -442,29 +504,46 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
 	command("SET client_min_messages = warning", 0, NULL);

 	/* acquire target tables */
-	appendStringInfoString(&sql, "SELECT * FROM repack.tables WHERE ");
-	if (num_params)
+	appendStringInfoString(&sql,
+		"SELECT t.*,"
+		" coalesce(v.tablespace, t.tablespace_orig) as tablespace_dest"
+		" FROM repack.tables t, "
+		" (VALUES (quote_ident($1::text))) as v (tablespace)"
+		" WHERE ");
+
+	params[iparam++] = tablespace;
+	if (num_tables)
 	{
 		appendStringInfoString(&sql, "(");
-		for (i = 0, cell = table_list.head; cell; cell = cell->next, i++)
+		for (cell = table_list.head; cell; cell = cell->next)
 		{
 			/* Construct table name placeholders to be used by PQexecParams */
-			appendStringInfo(&sql, "relid = $%d::regclass", i + 1);
-			params[i] = cell->val;
+			appendStringInfo(&sql, "relid = $%d::regclass", iparam + 1);
+			params[iparam++] = cell->val;
 			if (cell->next)
 				appendStringInfoString(&sql, " OR ");
 		}
 		appendStringInfoString(&sql, ")");
-		res = execute_elevel(sql.data, (int) num_params, params, DEBUG2);
 	}
 	else
 	{
 		appendStringInfoString(&sql, "pkid IS NOT NULL");
 		if (!orderby)
 			appendStringInfoString(&sql, " AND ckid IS NOT NULL");
-		res = execute_elevel(sql.data, 0, NULL, DEBUG2);
 	}

+	/* double check the parameters array is sane */
+	if (iparam != num_params)
+	{
+		if (errbuf)
+			snprintf(errbuf, errsize,
+				"internal error: bad parameters count: %i instead of %zi",
+				 iparam, num_params);
+		goto cleanup;
+	}
+
+	res = execute_elevel(sql.data, (int) num_params, params, DEBUG2);
+
 	/* on error skip the database */
 	if (PQresultStatus(res) != PGRES_TUPLES_OK)
 	{
@ -489,7 +568,9 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
 	for (i = 0; i < num; i++)
 	{
 		repack_table	table;
-		const char *create_table;
+		const char *create_table_1;
+		const char *create_table_2;
+		const char *tablespace;
 		const char *ckey;
 		int			c = 0;

@ -512,44 +593,50 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize)
 		table.create_trigger = getstr(res, i, c++);
 		table.enable_trigger = getstr(res, i, c++);

-		create_table = getstr(res, i, c++);
+		create_table_1 = getstr(res, i, c++);
+		tablespace = getstr(res, i, c++);	/* to be clobbered */
+		create_table_2 = getstr(res, i, c++);
 		table.drop_columns = getstr(res, i, c++);
 		table.delete_log = getstr(res, i, c++);
 		table.lock_table = getstr(res, i, c++);
 		ckey = getstr(res, i, c++);
-
-		resetStringInfo(&sql);
-		if (!orderby)
-		{
-			if (ckey != NULL)
-			{
-				/* CLUSTER mode */
-				appendStringInfo(&sql, "%s ORDER BY %s", create_table, ckey);
-				table.create_table = sql.data;
-			}
-			else
-			{
-				/* VACUUM FULL mode (non-clustered tables) */
-				table.create_table = create_table;
-			}
-		}
-		else if (!orderby[0])
-		{
-			/* VACUUM FULL mode (for clustered tables too) */
-			table.create_table = create_table;
-		}
-		else
-		{
-			/* User specified ORDER BY */
-			appendStringInfo(&sql, "%s ORDER BY %s", create_table, orderby);
-			table.create_table = sql.data;
-		}
-
 		table.sql_peek = getstr(res, i, c++);
 		table.sql_insert = getstr(res, i, c++);
 		table.sql_delete = getstr(res, i, c++);
 		table.sql_update = getstr(res, i, c++);
 		table.sql_pop = getstr(res, i, c++);
+		tablespace = getstr(res, i, c++);
+
+		resetStringInfo(&sql);
+		appendStringInfoString(&sql, create_table_1);
+		appendStringInfoString(&sql, tablespace);
+		appendStringInfoString(&sql, create_table_2);
+		if (!orderby)
+		{
+			if (ckey != NULL)
+			{
+				/* CLUSTER mode */
+				appendStringInfoString(&sql, " ORDER BY ");
+				appendStringInfoString(&sql, ckey);
+				table.create_table = sql.data;
+			}
+			else
+			{
+				/* VACUUM FULL mode (non-clustered tables) */
+				table.create_table = sql.data;
+			}
+		}
+		else if (!orderby[0])
+		{
+			/* VACUUM FULL mode (for clustered tables too) */
+			table.create_table = sql.data;
+		}
+		else
+		{
+			/* User specified ORDER BY */
+			appendStringInfoString(&sql, " ORDER BY ");
+			appendStringInfoString(&sql, orderby);
+		}

 		repack_one_table(&table, orderby);
 	}
@ -594,7 +681,7 @@ static bool
 rebuild_indexes(const repack_table *table)
 {
 	PGresult	   *res;
-	const char	   *params[1];
+	const char	   *params[2];
 	int			    num_indexes;
 	int				i;
 	int				num_active_workers;
@ -606,6 +693,7 @@ rebuild_indexes(const repack_table *table)
 	elog(DEBUG2, "---- create indexes ----");

 	params[0] = utoa(table->target_oid, buffer);
+	params[1] = moveidx ? tablespace : NULL;

 	/* First, just display a warning message for any invalid indexes
 	 * which may be on the table (mostly to match the behavior of 1.1.8).
@ -621,8 +709,9 @@ rebuild_indexes(const repack_table *table)
 	}

 	res = execute("SELECT indexrelid,"
-		" repack.repack_indexdef(indexrelid, indrelid) "
-		" FROM pg_index WHERE indrelid = $1 AND indisvalid", 1, params);
+		" repack.repack_indexdef(indexrelid, indrelid, $2) "
+		" FROM pg_index WHERE indrelid = $1 AND indisvalid",
+		2, params);

 	num_indexes = PQntuples(res);

@ -1453,6 +1542,8 @@ pgut_help(bool details)
 	printf("Options:\n");
 	printf("  -a, --all                 repack all databases\n");
 	printf("  -t, --table=TABLE         repack specific table only\n");
+	printf("  -s, --tablespace=TABLESPC move repacked tables to a new tablespace\n");
+	printf("  -S, --moveidx             move repacked indexes to TABLESPC too\n");
 	printf("  -o, --order-by=COLUMNS    order by columns instead of cluster keys\n");
 	printf("  -n, --no-order            do vacuum full instead of cluster\n");
 	printf("  -j --jobs                 Use this many parallel jobs for each table\n");
--- a/bin/sql/tablespace.sql
+++ b/bin/sql/tablespace.sql
@ -0,0 +1,55 @@
+SET client_min_messages = warning;
+
+--
+-- Tablespace features tests
+--
+-- Note: in order to pass this test you must create a tablespace called 'testts'
+--
+
+SELECT spcname FROM pg_tablespace WHERE spcname = 'testts';
+-- If the query above failed you must create the 'testts' tablespace;
+
+CREATE TABLE testts1 (id serial primary key, data text);
+CREATE INDEX testts1_partial_idx on testts1 (id) where (id > 0);
+INSERT INTO testts1 (data) values ('a');
+INSERT INTO testts1 (data) values ('b');
+INSERT INTO testts1 (data) values ('c');
+
+-- can move the tablespace from default
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts
+
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+
+SELECT * from testts1 order by id;
+
+-- tablespace stays where it is
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1
+
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+
+-- can move the ts back to default
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -s pg_default
+
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+
+-- can move the table together with the indexes
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts --moveidx
+
+SELECT relname, spcname
+FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace
+WHERE relname ~ '^testts1'
+ORDER BY relname;
+
+-- can't specify --moveidx without --tablespace
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --moveidx
+\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -S
+
--- a/doc/pg_repack.rst
+++ b/doc/pg_repack.rst
@ -118,6 +118,8 @@ The following options can be specified in ``OPTIONS``.
 Options:
  -a, --all                 repack all databases
  -t, --table=TABLE         repack specific table only
+  -s, --tablespace=TABLESPC move repacked tables to a new tablespace
+  -S, --moveidx             move repacked indexes to TABLESPC too
  -o, --order-by=COLUMNS    order by columns instead of cluster keys
  -n, --no-order            do vacuum full instead of cluster
  -j, --jobs                Use this many parallel jobs for each table
@ -163,6 +165,15 @@ Reorg Options
    on each table. If your PostgreSQL server has extra cores and disk
    I/O available, this can be a useful way to speed up pg_repack.

+``-s TABLESPC``, ``--tablespace=TABLESPC``
+    Move the repacked tables to the specified tablespace: essentially an
+    online version of ``ALTER TABLE ... SET TABLESPACE``. The tables indexes
+    are left on the original tablespace unless ``--moveidx`` is specified too.
+
+``-S``, ``--moveidx``
+    Move the indexes too of the repacked tables to the tablespace specified
+    by the option ``--tablespace``.
+
 ``-T SECS``, ``--wait-timeout=SECS``
    pg_repack needs to take an exclusive lock at the end of the
    reorganization.  This setting controls how many seconds pg_repack will
@ -405,6 +416,7 @@ Releases

 * pg_repack 1.2

+  * Added --tablespace and --moveidx options to perform online SET TABLESPACE.
  * Added --jobs option for parallel operation.
  * Don't require --no-order to perform a VACUUM FULL on non-clustered tables
    (pg_repack issue #6).
--- a/lib/pg_repack.sql.in
+++ b/lib/pg_repack.sql.in
@ -179,7 +179,9 @@ CREATE VIEW repack.tables AS
         'CREATE TABLE repack.log_' || R.oid || ' (id bigserial PRIMARY KEY, pk repack.pk_' || R.oid || ', row ' || repack.oid2text(R.oid) || ')' AS create_log,
         repack.get_create_trigger(R.oid, PK.indexrelid) AS create_trigger,
         repack.get_enable_trigger(R.oid) as enable_trigger,
-         'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE ' || coalesce(quote_ident(S.spcname), 'pg_default') || ' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table,
+         'CREATE TABLE repack.table_' || R.oid || ' WITH (' || array_to_string(array_append(R.reloptions, 'oids=' || CASE WHEN R.relhasoids THEN 'true' ELSE 'false' END), ',') || ') TABLESPACE '  AS create_table_1,
+         coalesce(quote_ident(S.spcname), 'pg_default') tablespace_orig,
+         ' AS SELECT ' || repack.get_columns_for_create_as(R.oid) || ' FROM ONLY ' || repack.oid2text(R.oid) AS create_table_2,
         repack.get_drop_columns(R.oid, 'repack.table_' || R.oid) AS drop_columns,
         'DELETE FROM repack.log_' || R.oid AS delete_log,
         'LOCK TABLE ' || repack.oid2text(R.oid) || ' IN ACCESS EXCLUSIVE MODE' AS lock_table,
@ -205,9 +207,9 @@ CREATE VIEW repack.tables AS
     AND N.nspname NOT IN ('pg_catalog', 'information_schema')
     AND N.nspname NOT LIKE E'pg\\_temp\\_%';

-CREATE FUNCTION repack.repack_indexdef(oid, oid) RETURNS text AS
+CREATE FUNCTION repack.repack_indexdef(oid, oid, name) RETURNS text AS
 'MODULE_PATHNAME', 'repack_indexdef'
-LANGUAGE C STABLE STRICT;
+LANGUAGE C STABLE;

 CREATE FUNCTION repack.repack_trigger() RETURNS trigger AS
 'MODULE_PATHNAME', 'repack_trigger'
--- a/lib/repack.c
+++ b/lib/repack.c
@ -618,20 +618,59 @@ repack_get_order_by(PG_FUNCTION_ARGS)
 *
 * @param	index	Oid of target index.
 * @param	table	Oid of table of the index.
+ * @param	tablespace	Namespace for the index. If NULL keep the original.
 * @retval			Create index DDL for temp table.
 */
 Datum
 repack_indexdef(PG_FUNCTION_ARGS)
 {
-	Oid				index = PG_GETARG_OID(0);
-	Oid				table = PG_GETARG_OID(1);
+	Oid				index;
+	Oid				table;
+	Name			tablespace = NULL;
 	IndexDef		stmt;
 	StringInfoData	str;

+	if (PG_ARGISNULL(0) || PG_ARGISNULL(1))
+		PG_RETURN_NULL();
+
+	index = PG_GETARG_OID(0);
+	table = PG_GETARG_OID(1);
+
+	if (!PG_ARGISNULL(2))
+		tablespace = PG_GETARG_NAME(2);
+
 	parse_indexdef(&stmt, index, table);
+
 	initStringInfo(&str);
-	appendStringInfo(&str, "%s index_%u ON repack.table_%u USING %s (%s)%s",
-		stmt.create, index, table, stmt.type, stmt.columns, stmt.options);
+	appendStringInfo(&str, "%s index_%u ON repack.table_%u USING %s (%s)",
+		stmt.create, index, table, stmt.type, stmt.columns);
+
+	/* Replace the tablespace in the index options */
+	if (tablespace == NULL)
+	{
+		/* tablespace is just fine */
+		appendStringInfoString(&str, stmt.options);
+	}
+	else
+	{
+		if (NULL == strstr(stmt.options, "TABLESPACE"))
+		{
+			/* tablespace is to append */
+			appendStringInfoString(&str, " TABLESPACE ");
+			appendStringInfoString(&str, NameStr(*tablespace));
+		}
+		else
+		{
+			/* tablespace is to replace */
+			char *tmp;
+			tmp = skip_const(index, stmt.options, " TABLESPACE", NULL);
+			appendStringInfoString(&str, stmt.options);
+			appendStringInfo(&str, " %s", NameStr(*tablespace));
+			tmp = skip_ident(index, tmp);
+			if (*tmp)
+				appendStringInfo(&str, " %s", tmp);
+		}
+	}

 	PG_RETURN_TEXT_P(cstring_to_text(str.data));
 }