From 506104686b374f0be0294afe83b3f82692f5f8d1 Mon Sep 17 00:00:00 2001 From: Josh Kupershmidt Date: Tue, 22 Apr 2014 10:56:24 -0400 Subject: [PATCH 1/9] Fix a nasty data loss/corruption bug from the sql_pop query. It is not safe to assume that we can bulk-delete all entries from the log table based on their "id" being less than the largest "id" we have processed so far, as we may unintentionally delete some tuples from the log which we haven't actually processed yet in the case of concurrent transactions adding tuples into the log table. --- lib/pg_repack.sql.in | 2 +- lib/repack.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index eb42436..80bc2d4 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -190,7 +190,7 @@ CREATE VIEW repack.tables AS 'INSERT INTO repack.table_' || R.oid || ' VALUES ($1.*)' AS sql_insert, 'DELETE FROM repack.table_' || R.oid || ' WHERE ' || repack.get_compare_pkey(PK.indexrelid, '$1') AS sql_delete, 'UPDATE repack.table_' || R.oid || ' SET ' || repack.get_assign(R.oid, '$2') || ' WHERE ' || repack.get_compare_pkey(PK.indexrelid, '$1') AS sql_update, - 'DELETE FROM repack.log_' || R.oid || ' WHERE id <= $1' AS sql_pop + 'DELETE FROM repack.log_' || R.oid || ' WHERE id = $1' AS sql_pop FROM pg_class R LEFT JOIN pg_class T ON R.reltoastrelid = T.oid LEFT JOIN repack.primary_keys PK diff --git a/lib/repack.c b/lib/repack.c index c2c128f..eae361a 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -282,13 +282,16 @@ repack_apply(PG_FUNCTION_ARGS) plan_update = repack_prepare(sql_update, 2, &argtypes[1]); execute_plan(SPI_OK_UPDATE, plan_update, &values[1], &nulls[1]); } + /* Delete tuple in log. + * XXX It would be a lot more efficient to perform + * this DELETE in bulk, but be careful to only + * delete log entries we have actually processed. + */ + if (plan_pop == NULL) + plan_pop = repack_prepare(sql_pop, 1, argtypes); + execute_plan(SPI_OK_DELETE, plan_pop, values, nulls); } - /* delete tuple in log */ - if (plan_pop == NULL) - plan_pop = repack_prepare(sql_pop, 1, argtypes); - execute_plan(SPI_OK_DELETE, plan_pop, values, nulls); - SPI_freetuptable(tuptable); } From 5e9d596bb7f4f2b73de951212fd330802d370908 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 May 2014 10:41:44 +0100 Subject: [PATCH 2/9] Document issue #23 fixed --- doc/pg_repack.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index ccc4978..514b6f5 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -442,6 +442,7 @@ Releases * Don't wait for locks held in other databases (pg_repack issue #11). * Bugfix: correctly handle key indexes with options such as DESC, NULL FIRST/LAST, COLLATE (pg_repack issue #3). + * Fixed data corruption bug on delete (pg_repack issue #23). * More helpful program output and error messages. * pg_repack 1.1.8 From 3555c6b81a9299ad7d07993a4f5de488bc81e58c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 14 Nov 2013 23:42:53 +0000 Subject: [PATCH 3/9] Added makefile target to create source distribution --- Makefile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6b9b8d6..95a0e5c 100644 --- a/Makefile +++ b/Makefile @@ -7,19 +7,26 @@ # PG_CONFIG ?= pg_config +EXTENSION = pg_repack -# Pull out the version number from pg_config +.PHONY: dist/$(EXTENSION)-$(EXTVERSION).zip + +# Pull out PostgreSQL version number from pg_config VERSION := $(shell $(PG_CONFIG) --version | awk '{print $$2}') ifeq ("$(VERSION)","") $(error pg_config not found) endif -# version as a number, e.g. 9.1.4 -> 901 +# PostgreSQL version as a number, e.g. 9.1.4 -> 901 INTVERSION := $(shell echo $$(($$(echo $(VERSION) | sed 's/\([[:digit:]]\{1,\}\)\.\([[:digit:]]\{1,\}\).*/\1*100+\2/')))) +# The version number of the library +EXTVERSION = $(shell grep '"version":' META.json | head -1 \ + | sed -e 's/[ ]*"version":[ ]*"\(.*\)",/\1/') + # We support PostgreSQL 8.3 and later. ifeq ($(shell echo $$(($(INTVERSION) < 803))),1) -$(error pg_repack requires PostgreSQL 8.3 or later. This is $(VERSION)) +$(error $(EXTENSION) requires PostgreSQL 8.3 or later. This is $(VERSION)) endif @@ -36,3 +43,9 @@ check installcheck: $(MAKE) -C $$dir $@ || CHECKERR=$$?; \ done; \ exit $$CHECKERR + +# Prepare the package for PGXN submission +package: $(EXTENSION)-$(EXTVERSION).zip + +$(EXTENSION)-$(EXTVERSION).zip: + git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output $@ master From cdcb69a237f1d6e5e4498ee5d50a4b1e9fc2a41c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Jan 2014 12:54:46 +0000 Subject: [PATCH 4/9] Source packages go in the dist directory --- .gitignore | 1 + Makefile | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bffb51b..fc936f5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.so regress/regression.diffs regress/regression.out +dist/*.zip diff --git a/Makefile b/Makefile index 95a0e5c..51956e6 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,10 @@ check installcheck: exit $$CHECKERR # Prepare the package for PGXN submission -package: $(EXTENSION)-$(EXTVERSION).zip +package: dist dist/$(EXTENSION)-$(EXTVERSION).zip -$(EXTENSION)-$(EXTVERSION).zip: +dist: + mkdir -p dist + +dist/$(EXTENSION)-$(EXTVERSION).zip: git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output $@ master From ae6638b581f27aac5067ad884554e1775dfdf068 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Jan 2014 12:55:22 +0000 Subject: [PATCH 5/9] Releasing version 1.2 --- META.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/META.json b/META.json index feaff93..87f4a48 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.2.0-beta1", + "version": "1.2.0", "maintainer": [ "Josh Kupershmidt ", "Daniele Varrazzo " @@ -13,7 +13,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.2.0-beta1", + "version": "1.2.0", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, From a9f3682645e5181b342fe5f113b77d85b7515e59 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 May 2014 12:43:35 +0100 Subject: [PATCH 6/9] Setting release status to stable Grr... can't remove the wrong package from PGXN --- META.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/META.json b/META.json index 87f4a48..8e3b64d 100644 --- a/META.json +++ b/META.json @@ -2,18 +2,18 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.2.0", + "version": "1.2.1", "maintainer": [ "Josh Kupershmidt ", "Daniele Varrazzo " ], "tags": [ "bloat", "maintenance", "vacuum", "cluster" ], - "release_status": "testing", + "release_status": "stable", "license": "bsd", "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.2.0", + "version": "1.2.1", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, From 3ca9c39dff94979c2fc64b9eaa3acb835e1823f7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 May 2014 13:45:29 +0100 Subject: [PATCH 7/9] Added documentation about the release process --- doc/release.rst | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/release.rst diff --git a/doc/release.rst b/doc/release.rst new file mode 100644 index 0000000..fddb736 --- /dev/null +++ b/doc/release.rst @@ -0,0 +1,63 @@ +What to do to release pg_repack +=============================== + +This document is the list of operations to do to release a new pg_repack +version. The version number in this document is indicated by ``$VER``: it +should be a three-digit dot-separated version, eventually followed by a +pre-release string: ``1.2.0``, ``1.2.1``, ``1.2-dev0``, ``1.2.0-beta1`` are +valid version numbers. + +In order to release the package you will accounts on Github, Freecode and PGXN +with the right privileges: contact Daniele Varrazzo to obtain them. + +- Set the right version number in ``META.json`` (note: it's in two different + places). +- Set the right release_status in ``META.json``: ``testing`` or ``stable``. +- Commit the above metadata changes. +- Create a tag, signed if possible:: + + git tag -a -s ver_$VER + +- Create a package running ``make package``. The package will be called + ``dist/pg_repack-$VER.zip``. + +- Check the packages installs and tests ok with `pgxn client`__:: + + pgxn install --sudo -- dist/pg_repack-$VER.zip + pgxn check dist/pg_repack-$VER.zip + + (note that ``check`` may require the Postgres bin directory to be added to + the path; check the ``install`` log to see where ``pg_repack`` executable + was installed). + + .. __: http://pgxnclient.projects.pgfoundry.org/ + +- Push the code changes and tags on github:: + + git push + git push --tags + +- Upload the package on http://manager.pgxn.org/. + +- Check the uploaded package works as expected:: + + pgxn install --sudo -- pg_repack + pgxn check pg_repack + +- Upload the docs by pushing in the repos at + http://reorg.github.io/pg_repack/. The operations are roughly:: + + git clone git@github.com:reorg/reorg.github.com.git + cd reorg.github.com.git + git submodule init + git submodule update + make + git commit -a -m "Docs upload for release $VER" + git push + +- Check the page http://reorg.github.io/pg_repack/ is right. + +- Announce the package on reorg-general@pgfoundry.org and + pgsql-announce@postgresql.org. + +- Announce the package on http://freecode.com/. From 9916c481383418e40f1e6bce398fdd26f190f53d Mon Sep 17 00:00:00 2001 From: schmiddy Date: Thu, 22 May 2014 15:08:39 -0400 Subject: [PATCH 8/9] A few tweaks to the README, addressing the state of pg_reorg. --- README.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 76429d1..87c496e 100644 --- a/README.rst +++ b/README.rst @@ -28,20 +28,18 @@ What about pg_reorg? -------------------- pg_repack is a fork of the pg_reorg_ project, which has proven hugely -successful. Unfortunately development appears to have stopped after the -release of the version 1.1.7, around August 2011. +successful. Unfortunately new feature development on pg_reorg_ has slowed +or stopped since late 2011. -pg_repack 1.1.8 was released as a drop-in replacement for pg_reorg, addressing -some of the shortcomings of the last pg_reorg version (such as support for -PostgreSQL 9.2 and EXTENSION packaging) and known bugs. Shortly after the -first pg_repack release, pg_reorg 1.1.8 was released too, merging all the -pg_repack changes. Version 1.1.8 is the last pg_reorg release at the time of -writing. +pg_repack was initially released as a drop-in replacement for pg_reorg, +addressing some of the shortcomings of the last pg_reorg version (such as +support for PostgreSQL 9.2 and EXTENSION packaging) and known bugs. -pg_repack 1.2 is a new development line based on the original pg_reorg -codebase and offering new features. Its behaviour may be different from the -1.1.x release so it shouldn't be considered a drop-in replacement: you are -advised to check the documentation__ before upgrading from previous versions. +pg_repack 1.2 introduces further new features (parallel index builds, +ability to rebuild only indexes) and bugfixes. In some cases its behaviour +may be different from the 1.1.x release so it shouldn't be considered a +drop-in replacement: you are advised to check the documentation__ before +upgrading from previous versions. .. __: pg_repack_ .. _pg_reorg: http://reorg.projects.pgfoundry.org/ From 08d28df63a35680d2c04f6a05a8c1825f20195d1 Mon Sep 17 00:00:00 2001 From: schmiddy Date: Thu, 22 May 2014 23:36:06 -0400 Subject: [PATCH 9/9] Add note urging users to upgrade due to Issue #23. --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 87c496e..c5299be 100644 --- a/README.rst +++ b/README.rst @@ -17,11 +17,16 @@ CLUSTER directly. Please check the documentation (in the ``doc`` directory or online_) for installation and usage instructions. +All users of pg_reorg 1.1.9 or earlier, and pg_repack 1.2.0-beta1 or earlier, +are **urged to upgrade** to the latest pg_repack version to fix a serious +data corruption issue_. + .. _pg_repack: http://reorg.github.com/pg_repack .. _CLUSTER: http://www.postgresql.org/docs/current/static/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: http://www.postgresql.org/docs/current/static/sql-vacuum.html .. _online: pg_repack_ +.. _issue: https://github.com/reorg/pg_repack/issues/23 What about pg_reorg?