Merge branch 'REL_2_6' into release_2_6-pbckp-232

Author: kulaginm

Makefile

@@ -33,7 +33,6 @@ PROGRAM := pg_probackup
 # pg_probackup sources
 OBJS := src/utils/configuration.o src/utils/json.o src/utils/logger.o \
 	src/utils/parray.o src/utils/pgut.o src/utils/thread.o src/utils/remote.o src/utils/file.o
-OBJS += src/fu_util/impl/ft_impl.o src/fu_util/impl/fo_impl.o
 OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \
 	src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \
 	src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \
@@ -52,6 +51,8 @@ BORROWED_C_SRC := \
 	src/bin/pg_basebackup/streamutil.c \
 	src/bin/pg_basebackup/walmethods.c
 
+OBJS += src/fu_util/impl/ft_impl.o src/fu_util/impl/fo_impl.o
+
 BORROW_DIR := src/borrowed
 BORROWED_H := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_H_SRC)))
 BORROWED_C := $(addprefix $(BORROW_DIR)/, $(notdir $(BORROWED_C_SRC)))

doc/pgprobackup.xml

@@ -3958,6 +3958,7 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
 [--restore-command=<replaceable>cmdline</replaceable>]
 [--primary-conninfo=<replaceable>primary_conninfo</replaceable>]
 [-S | --primary-slot-name=<replaceable>slot_name</replaceable>]
+[-X <replaceable>wal_dir</replaceable> | --waldir=<replaceable>wal_dir</replaceable>]
 [<replaceable>recovery_target_options</replaceable>] [<replaceable>logging_options</replaceable>] [<replaceable>remote_options</replaceable>]
 [<replaceable>partial_restore_options</replaceable>] [<replaceable>remote_wal_archive_options</replaceable>]
 </programlisting>
@@ -4125,6 +4126,17 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
       </para>
       </listitem>
       </varlistentry>
+
+      <varlistentry>
+<term><option>-X <replaceable>wal_dir</replaceable></option></term>
+<term><option>--waldir=<replaceable>wal_dir</replaceable></option></term>
+      <listitem>
+      <para>
+        Specifies the directory where WAL should be stored.
+      </para>
+      </listitem>
+    </varlistentry>
+
     </variablelist>
     </para>
       <para>
@@ -5150,6 +5162,60 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
       </varlistentry>
 
       <varlistentry>
+<term><option>--log-format-console=<replaceable>log_format</replaceable></option></term>
+        <listitem>
+        <para>
+          Defines the format of the console log. Only set from the command line. Note that you cannot
+          specify this option in the <filename>pg_probackup.conf</filename> configuration file through
+          the <xref linkend="pbk-set-config"/> command and that the <xref linkend="pbk-backup"/>
+          command also treats this option specified in the configuration file as an error.
+          Possible values are:
+
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              <literal>plain</literal> — sets the plain-text format of the console log.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>json</literal> — sets the <acronym>JSON</acronym> format of the console log.
+            </para>
+          </listitem>
+        </itemizedlist>
+        </para>
+        <para>
+         Default: <literal>plain</literal>
+        </para>
+        </listitem>
+        </varlistentry>
+
+      <varlistentry>
+<term><option>--log-format-file=<replaceable>log_format</replaceable></option></term>
+        <listitem>
+        <para>
+          Defines the format of log files used. Possible values are:
+
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              <literal>plain</literal> — sets the plain-text format of log files.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>json</literal> — sets the <acronym>JSON</acronym> format of log files.
+            </para>
+          </listitem>
+        </itemizedlist>
+        </para>
+        <para>
+         Default: <literal>plain</literal>
+        </para>
+        </listitem>
+        </varlistentry>
+
+        <varlistentry>
 <term><option>--log-rotation-size=<replaceable>log_rotation_size</replaceable></option></term>
       <listitem>
       <para>
@@ -6021,6 +6087,8 @@ archive-timeout = 5min
 # Logging parameters
 log-level-console = INFO
 log-level-file = OFF
+log-format-console = PLAIN
+log-format-file = PLAIN
 log-filename = pg_probackup.log
 log-rotation-size = 0
 log-rotation-age = 0

src/archive.c

@@ -347,7 +347,7 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
 		canonicalize_path(wal_file_ready);
 		canonicalize_path(wal_file_done);
 		/* It is ok to rename status file in archive_status directory */
-		elog(VERBOSE, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done);
+		elog(LOG, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done);
 
 		/* do not error out, if rename failed */
 		if (fio_rename(FIO_DB_HOST, wal_file_ready, wal_file_done) < 0)
@@ -502,7 +502,7 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
         elog(ERROR, "Temp WAL: %s", $errmsg(err));
 
     part_opened:
-    elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_part);
+    elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_part);
 
     if ($i(pioExists, backup_drive, to_fullpath, &err))
     {
@@ -872,7 +872,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha
 		if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false))
 		{
 			fail_count = 0;
-			elog(INFO, "pg_probackup archive-get copied WAL file %s", wal_file_name);
+			elog(LOG, "pg_probackup archive-get copied WAL file %s", wal_file_name);
 			n_fetched++;
 			break;
 		}

src/backup.c

@@ -113,7 +113,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 	char		pretty_time[20];
 	char		pretty_bytes[20];
 
-	elog(LOG, "Database backup start");
+	elog(INFO, "Database backup start");
 	if(current.external_dir_str)
 	{
 		external_dirs = make_external_directory_list(current.external_dir_str,
@@ -329,11 +329,11 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 	/* Extract information about files in backup_list parsing their names:*/
 	parse_filelist_filenames(backup_files_list, instance_config.pgdata);
 
-	elog(LOG, "Current Start LSN: %X/%X, TLI: %X",
+	elog(INFO, "Current Start LSN: %X/%X, TLI: %X",
 			(uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn),
 			current.tli);
 	if (current.backup_mode != BACKUP_MODE_FULL)
-		elog(LOG, "Parent Start LSN: %X/%X, TLI: %X",
+		elog(INFO, "Parent Start LSN: %X/%X, TLI: %X",
 			 (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn),
 			 prev_backup->tli);
 
@@ -405,7 +405,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 			else
 				join_path_components(dirpath, current.database_dir, file->rel_path);
 
-			elog(VERBOSE, "Create directory '%s'", dirpath);
+			elog(LOG, "Create directory '%s'", dirpath);
 			fio_mkdir(FIO_BACKUP_HOST, dirpath, DIR_PERMISSION, false);
 		}
 
@@ -666,7 +666,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
 	nodeInfo->checksum_version = current.checksum_version;
 
 	if (current.checksum_version)
-		elog(LOG, "This PostgreSQL instance was initialized with data block checksums. "
+		elog(INFO, "This PostgreSQL instance was initialized with data block checksums. "
 					"Data block corruption will be detected");
 	else
 		elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. "
@@ -688,7 +688,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
  */
 int
 do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
-		  bool no_validate, bool no_sync, bool backup_logs)
+		  bool no_validate, bool no_sync, bool backup_logs, time_t start_time)
 {
 	PGconn		*backup_conn = NULL;
 	PGNodeInfo	nodeInfo;
@@ -703,7 +703,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
 		current.external_dir_str = instance_config.external_dir_str;
 
 	/* Create backup directory and BACKUP_CONTROL_FILE */
-	pgBackupCreateDir(&current, instanceState->instance_backup_subdir_path);
+	pgBackupCreateDir(&current, instanceState, start_time);
 
 	if (!instance_config.pgdata)
 		elog(ERROR, "required parameter not specified: PGDATA "
@@ -1473,7 +1473,7 @@ wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBa
 		stop_lsn_exists = true;
 	}
 
-	elog(LOG, "stop_lsn: %X/%X",
+	elog(INFO, "stop_lsn: %X/%X",
 		(uint32) (stop_lsn >> 32), (uint32) (stop_lsn));
 
 	/*
@@ -1811,15 +1811,15 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb
 
 	backup->recovery_xid = stop_backup_result.snapshot_xid;
 
-	elog(LOG, "Getting the Recovery Time from WAL");
+	elog(INFO, "Getting the Recovery Time from WAL");
 
 	/* iterate over WAL from stop_backup lsn to start_backup lsn */
 	if (!read_recovery_info(xlog_path, backup->tli,
 						instance_config.xlog_seg_size,
 						backup->start_lsn, backup->stop_lsn,
 						&backup->recovery_time))
 	{
-		elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp");
+		elog(INFO, "Failed to find Recovery Time in WAL, forced to trust current_timestamp");
 		backup->recovery_time = stop_backup_result.invocation_time;
 	}
 
@@ -1901,9 +1901,8 @@ backup_files(void *arg)
 		if (interrupted || thread_interrupted)
 			elog(ERROR, "interrupted during backup");
 
-		if (progress)
-			elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
-				 i + 1, n_backup_files_list, file->rel_path);
+		elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"",
+			 i + 1, n_backup_files_list, file->rel_path);
 
 		/* Handle zero sized files */
 		if (file->size == 0)
@@ -1973,11 +1972,11 @@ backup_files(void *arg)
 
 		if (file->write_size == BYTES_INVALID)
 		{
-			elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath);
+			elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath);
 			continue;
 		}
 
-		elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",
+		elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes",
 						from_fullpath, file->write_size);
 	}
 
@@ -2095,26 +2094,26 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
 		elog(ERROR, "Out of memory");
 	len = strlen("/pg_compression");
 	cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0;
-	elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative);
+	elog(LOG, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative);
 
 	for (p = (int) i; p >= 0; p--)
 	{
 		prev_file = (pgFile *) parray_get(files, (size_t) p);
 
-		elog(VERBOSE, "Checking file in cfs tablespace %s", prev_file->rel_path);
+		elog(LOG, "Checking file in cfs tablespace %s", prev_file->rel_path);
 
 		if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL)
 		{
 			if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
 			{
-				elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s",
+				elog(LOG, "Setting 'is_cfs' on file %s, name %s",
 					prev_file->rel_path, prev_file->name);
 				prev_file->is_cfs = true;
 			}
 		}
 		else
 		{
-			elog(VERBOSE, "Breaking on %s", prev_file->rel_path);
+			elog(LOG, "Breaking on %s", prev_file->rel_path);
 			break;
 		}
 	}

src/catalog.c

@@ -23,7 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo);
 static pgBackup* get_oldest_backup(timelineInfo *tlinfo);
 static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
 static pgBackup *readBackupControlFile(const char *path);
-static time_t create_backup_dir(pgBackup *backup, const char *backup_instance_path);
+static void create_backup_dir(pgBackup *backup, const char *backup_instance_path);
 
 static bool backup_lock_exit_hook_registered = false;
 static parray *locks = NULL;
@@ -1426,10 +1426,12 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list,
  * to fix it or let it be.
  */
 void
-pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
+pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time)
 {
 	int		i;
 	parray *subdirs = parray_new();
+	parray * backups;
+	pgBackup *target_backup;
 
 	parray_append(subdirs, pg_strdup(DATABASE_DIR));
 
@@ -1450,7 +1452,26 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
 		free_dir_list(external_list);
 	}
 
-	backup->backup_id = create_backup_dir(backup, backup_instance_path);
+	/* Get list of all backups*/
+	backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
+	if (parray_num(backups) > 0)
+	{
+		target_backup = (pgBackup *) parray_get(backups, 0);
+		if (start_time > target_backup->backup_id)
+		{
+			backup->backup_id = start_time;
+			create_backup_dir(backup, instanceState->instance_backup_subdir_path);
+		}
+		else
+		{
+			elog(ERROR, "Cannot create directory for older backup");
+		}
+	}
+	else
+	{
+		backup->backup_id = start_time;
+		create_backup_dir(backup, instanceState->instance_backup_subdir_path);
+	}
 
 	if (backup->backup_id == 0)
 		elog(ERROR, "Cannot create backup directory: %s", strerror(errno));
@@ -1477,7 +1498,7 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path)
  * Create root directory for backup,
  * update pgBackup.root_dir if directory creation was a success
  */
-time_t
+void
 create_backup_dir(pgBackup *backup, const char *backup_instance_path)
 {
 	int     attempts = 10;
@@ -1486,16 +1507,15 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path)
 	{
 		int    rc;
 		char   path[MAXPGPATH];
-		time_t backup_id = time(NULL);
 
-		join_path_components(path, backup_instance_path, base36enc(backup_id));
+		join_path_components(path, backup_instance_path, base36enc(backup->backup_id));
 
 		rc = fio_mkdir(FIO_BACKUP_HOST, path, DIR_PERMISSION, true);
 
 		if (rc == 0)
 		{
 			backup->root_dir = pgut_strdup(path);
-			return backup_id;
+			return;
 		}
 		else
 		{
@@ -1504,7 +1524,6 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path)
 		}
 	}
 
-	return 0;
 }
 
 /*