Merge branch 'REL_2_5' into REL_2_6

Author: funny-falcon

src/dir.c

@@ -1742,13 +1742,16 @@ is_forkname(char *name, size_t *pos, const char *forkname)
 }
 
 #define OIDCHARS 10
+#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1)
+#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */
 
 /* Set forkName if possible */
 bool
 set_forkname(pgFile *file)
 {
 	size_t i = 0;
 	uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */
+	uint64_t segno = 0;
 
 	/* pretend it is not relation file */
 	file->relOid = 0;
@@ -1779,8 +1782,15 @@ set_forkname(pgFile *file)
 	/* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */
 	if (file->name[i] == '.' && isdigit(file->name[i+1]))
 	{
+		size_t start = i+1;
 		for (i++; isdigit(file->name[i]); i++)
-			;
+		{
+			if (i == start && file->name[i] == '0')
+				return false;
+			segno = segno * 10 + file->name[i] - '0';
+		}
+		if (i - start > SEGNOCHARS || segno > MAXSEGNO)
+			return false;
 	}
 
 	/* CFS "fork name" */
@@ -1799,6 +1809,7 @@ set_forkname(pgFile *file)
 	}
 
 	file->relOid = oid;
+	file->segno = segno;
 	file->is_datafile = file->forkName == none;
 	return true;
 }

tests/archive.py

@@ -981,7 +981,7 @@ def test_concurrent_archiving(self):
         """
 
         if self.pg_config_version < self.version_to_num('11.0'):
-            return unittest.skip('You need PostgreSQL >= 11 for this test')
+            self.skipTest('You need PostgreSQL >= 11 for this test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/backup.py

@@ -13,6 +13,38 @@
 
 class BackupTest(ProbackupTest, unittest.TestCase):
 
+    def test_basic_full_backup(self):
+        """
+        Just test full backup with at least two segments
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            initdb_params=['--data-checksums'],
+            # we need to write a lot. Lets speedup a bit.
+            pg_options={"fsync": "off", "synchronous_commit": "off"})
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # Fill with data
+        # Have to use scale=100 to create second segment.
+        node.pgbench_init(scale=100, no_vacuum=True)
+
+        # FULL
+        backup_id = self.backup_node(backup_dir, 'node', node)
+
+        out = self.validate_pb(backup_dir, 'node', backup_id)
+        self.assertIn(
+            "INFO: Backup {0} is valid".format(backup_id),
+            out)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
     # @unittest.skip("skip")
     # @unittest.expectedFailure
     # PGPRO-707
@@ -1497,7 +1529,7 @@ def test_backup_concurrent_drop_table(self):
     def test_pg_11_adjusted_wal_segment_size(self):
         """"""
         if self.pg_config_version < self.version_to_num('11.0'):
-            return unittest.skip('You need PostgreSQL >= 11 for this test')
+            self.skipTest('You need PostgreSQL >= 11 for this test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1740,7 +1772,7 @@ def test_drop_table(self):
     def test_basic_missing_file_permissions(self):
         """"""
         if os.name == 'nt':
-            return unittest.skip('Skipped because it is POSIX only test')
+            self.skipTest('Skipped because it is POSIX only test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1787,7 +1819,7 @@ def test_basic_missing_file_permissions(self):
     def test_basic_missing_dir_permissions(self):
         """"""
         if os.name == 'nt':
-            return unittest.skip('Skipped because it is POSIX only test')
+            self.skipTest('Skipped because it is POSIX only test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/catchup.py

@@ -190,7 +190,7 @@ def test_basic_ptrack_catchup(self):
         Test ptrack catchup
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         # preparation 1: source
         src_pg = self.make_simple_node(
@@ -336,7 +336,7 @@ def test_tli_ptrack_catchup(self):
         Test that we correctly follow timeline change with ptrack catchup
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         # preparation 1: source
         src_pg = self.make_simple_node(
@@ -475,7 +475,7 @@ def test_table_drop_with_ptrack(self):
         Test that dropped table in source will be dropped in ptrack catchup'ed instance too
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         # preparation 1: source
         src_pg = self.make_simple_node(
@@ -590,7 +590,7 @@ def test_tablefile_truncation_with_ptrack(self):
         Test that truncated table in source will be truncated in ptrack catchup'ed instance too
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         # preparation 1: source
         src_pg = self.make_simple_node(
@@ -655,7 +655,7 @@ def test_local_tablespace_without_mapping(self):
         Test that we detect absence of needed --tablespace-mapping option
         """
         if self.remote:
-            return unittest.skip('Skipped because this test tests local catchup error handling')
+            self.skipTest('Skipped because this test tests local catchup error handling')
 
         src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'src'))
         src_pg.slow_start()
@@ -1035,7 +1035,7 @@ def test_unclean_ptrack_catchup(self):
         Test that we correctly recover uncleanly shutdowned destination
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         # preparation 1: source
         src_pg = self.make_simple_node(
@@ -1506,7 +1506,7 @@ def test_dry_run_catchup_ptrack(self):
         Test dry-run option for catchup in incremental ptrack mode
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
          # preparation 1: source
         src_pg = self.make_simple_node(

tests/compatibility.py

@@ -366,7 +366,7 @@ def test_backward_compatibility_ptrack(self):
         """Description in jira issue PGPRO-434"""
 
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/delete.py

@@ -189,7 +189,7 @@ def test_delete_increment_page(self):
     def test_delete_increment_ptrack(self):
         """delete increment and all after him"""
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(

tests/external.py

@@ -1535,7 +1535,7 @@ def test_external_dir_is_symlink(self):
         but restored as directory
         """
         if os.name == 'nt':
-            return unittest.skip('Skipped for Windows')
+            self.skipTest('Skipped for Windows')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1618,7 +1618,7 @@ def test_external_dir_contain_symlink_on_dir(self):
         but restored as directory
         """
         if os.name == 'nt':
-            return unittest.skip('Skipped for Windows')
+            self.skipTest('Skipped for Windows')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1703,7 +1703,7 @@ def test_external_dir_contain_symlink_on_file(self):
         but restored as directory
         """
         if os.name == 'nt':
-            return unittest.skip('Skipped for Windows')
+            self.skipTest('Skipped for Windows')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/helpers/__init__.py

@@ -1,2 +1,9 @@
 __all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors']
-#from . import *
+
+import unittest
+
+# python 2.7 compatibility
+if not hasattr(unittest.TestCase, "skipTest"):
+    def skipTest(self, reason):
+        raise unittest.SkipTest(reason)
+    unittest.TestCase.skipTest = skipTest

tests/helpers/ptrack_helpers.py

@@ -242,10 +242,7 @@ def __init__(self, *args, **kwargs):
         self.user = self.get_username()
         self.probackup_path = None
         if 'PGPROBACKUPBIN' in self.test_env:
-            if (
-                os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and
-                os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK)
-            ):
+            if shutil.which(self.test_env["PGPROBACKUPBIN"]):
                 self.probackup_path = self.test_env["PGPROBACKUPBIN"]
             else:
                 if self.verbose:

tests/merge.py

@@ -796,7 +796,7 @@ def test_merge_ptrack_truncate(self):
         restore last page backup and check data correctness
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/option.py

@@ -239,5 +239,5 @@ def test_help_6(self):
                     help_out.read().decode("utf-8")
                 )
         else:
-            return unittest.skip(
+            self.skipTest(
                 'You need configure PostgreSQL with --enabled-nls option for this test')

tests/ptrack.py

@@ -16,7 +16,7 @@
 class PtrackTest(ProbackupTest, unittest.TestCase):
     def setUp(self):
         if self.pg_config_version < self.version_to_num('11.0'):
-            return unittest.skip('You need PostgreSQL >= 11 for this test')
+            self.skipTest('You need PostgreSQL >= 11 for this test')
         self.fname = self.id().split('.')[3]
 
     # @unittest.skip("skip")

tests/replica.py

@@ -98,7 +98,7 @@ def test_replica_stream_ptrack_backup(self):
         take full stream backup from replica
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/restore.py

@@ -492,7 +492,7 @@ def test_restore_to_lsn_not_inclusive(self):
     def test_restore_full_ptrack_archive(self):
         """recovery to latest from archive full+ptrack backups"""
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -546,7 +546,7 @@ def test_restore_full_ptrack_archive(self):
     def test_restore_ptrack(self):
         """recovery to latest from archive full+ptrack+ptrack backups"""
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -607,7 +607,7 @@ def test_restore_ptrack(self):
     def test_restore_full_ptrack_stream(self):
         """recovery in stream mode to latest from full + ptrack backups"""
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -665,7 +665,7 @@ def test_restore_full_ptrack_under_load(self):
         with loads when ptrack backup do
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -734,7 +734,7 @@ def test_restore_full_under_load_ptrack(self):
         with loads when full backup do
         """
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -2333,7 +2333,7 @@ def test_pg_11_group_access(self):
         test group access for PG >= 11
         """
         if self.pg_config_version < self.version_to_num('11.0'):
-            return unittest.skip('You need PostgreSQL >= 11 for this test')
+            self.skipTest('You need PostgreSQL >= 11 for this test')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -3595,7 +3595,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self):
             self.skipTest("You must specify PGPROBACKUPBIN_OLD"
                           " for run this test")
         if self.pg_config_version < self.version_to_num('12.0'):
-           return unittest.skip('You need PostgreSQL >= 12 for this test')
+           self.skipTest('You need PostgreSQL >= 12 for this test')
 
         if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'):
             self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test')
@@ -3667,7 +3667,7 @@ def test_drop_postgresql_auto_conf(self):
         """
 
         if self.pg_config_version < self.version_to_num('12.0'):
-           return unittest.skip('You need PostgreSQL >= 12 for this test')
+           self.skipTest('You need PostgreSQL >= 12 for this test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -3712,7 +3712,7 @@ def test_truncate_postgresql_auto_conf(self):
         """
 
         if self.pg_config_version < self.version_to_num('12.0'):
-           return unittest.skip('You need PostgreSQL >= 12 for this test')
+           self.skipTest('You need PostgreSQL >= 12 for this test')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')

tests/time_consuming.py

@@ -16,9 +16,9 @@ def test_pbckp150(self):
         """
         # init node
         if self.pg_config_version < self.version_to_num('11.0'):
-            return unittest.skip('You need PostgreSQL >= 11 for this test')
+            self.skipTest('You need PostgreSQL >= 11 for this test')
         if not self.ptrack:
-            return unittest.skip('Skipped because ptrack support is disabled')
+            self.skipTest('Skipped because ptrack support is disabled')
 
         fname = self.id().split('.')[3]
         node = self.make_simple_node(