Add --uniform-snapshot-timestamp option (#146)

This option apply a single consistent timestamp to all filenames in a snapshot instead of using individual file creation times.

Author: stephane-klein

config.go

@@ -47,40 +47,41 @@ var defaultCfg string
 
 // options struct holds command line and configuration file options
 type options struct {
-	NoConfigFile      bool
-	BinDirectory      string
-	Directory         string
-	Host              string
-	Port              int
-	Username          string
-	ConnDb            string
-	ExcludeDbs        []string
-	Dbnames           []string
-	WithTemplates     bool
-	Format            rune
-	DirJobs           int
-	CompressLevel     int
-	Jobs              int
-	PauseTimeout      int
-	PurgeInterval     time.Duration
-	PurgeKeep         int
-	SumAlgo           string
-	PreHook           string
-	PostHook          string
-	PgDumpOpts        []string
-	PerDbOpts         map[string]*dbOpts
-	CfgFile           string
-	TimeFormat        string
-	Verbose           bool
-	Quiet             bool
-	Encrypt           bool
-	EncryptKeepSrc    bool
-	CipherPassphrase  string
-	CipherPublicKey   string
-	CipherPrivateKey  string
-	Decrypt           bool
-	WithRolePasswords bool
-	DumpOnly          bool
+	NoConfigFile             bool
+	BinDirectory             string
+	Directory                string
+	Host                     string
+	Port                     int
+	Username                 string
+	ConnDb                   string
+	ExcludeDbs               []string
+	Dbnames                  []string
+	WithTemplates            bool
+	Format                   rune
+	DirJobs                  int
+	CompressLevel            int
+	Jobs                     int
+	PauseTimeout             int
+	PurgeInterval            time.Duration
+	PurgeKeep                int
+	SumAlgo                  string
+	PreHook                  string
+	PostHook                 string
+	PgDumpOpts               []string
+	PerDbOpts                map[string]*dbOpts
+	CfgFile                  string
+	TimeFormat               string
+	Verbose                  bool
+	Quiet                    bool
+	Encrypt                  bool
+	EncryptKeepSrc           bool
+	CipherPassphrase         string
+	CipherPublicKey          string
+	CipherPrivateKey         string
+	Decrypt                  bool
+	WithRolePasswords        bool
+	DumpOnly                 bool
+	UniformSnapshotTimestamp bool
 
 	Upload       string // values are none, b2, s3, sftp, gcs
 	UploadPrefix string
@@ -285,6 +286,7 @@ func parseCli(args []string) (options, []string, error) {
 	pflag.StringVarP(&purgeKeep, "purge-min-keep", "K", "0", "minimum number of dumps to keep when purging or 'all' to keep\neverything")
 	pflag.StringVar(&opts.PreHook, "pre-backup-hook", "", "command to run before taking dumps")
 	pflag.StringVar(&opts.PostHook, "post-backup-hook", "", "command to run after taking dumps\n")
+	pflag.BoolVar(&opts.UniformSnapshotTimestamp, "uniform-snapshot-timestamp", false, "Apply a single consistent timestamp to all filenames in a snapshot instead of using individual file creation times")
 
 	pflag.BoolVar(&opts.Encrypt, "encrypt", false, "encrypt the dumps")
 	NoEncrypt := pflag.Bool("no-encrypt", false, "do not encrypt the dumps")
@@ -532,7 +534,7 @@ func validateConfigurationFile(cfg *ini.File) error {
 		"sftp_port", "sftp_user", "sftp_password", "sftp_directory", "sftp_identity",
 		"sftp_ignore_hostkey", "gcs_bucket", "gcs_endpoint", "gcs_keyfile",
 		"azure_container", "azure_account", "azure_key", "azure_endpoint", "pg_dump_options",
-		"dump_role_passwords", "dump_only", "upload_prefix",
+		"dump_role_passwords", "dump_only", "upload_prefix", "uniform_snapshot_timestamp",
 	}
 
 gkLoop:
@@ -625,6 +627,7 @@ func loadConfigurationFile(path string) (options, error) {
 	opts.CipherPublicKey = s.Key("cipher_public_key").MustString("")
 	opts.CipherPrivateKey = s.Key("cipher_private_key").MustString("")
 	opts.EncryptKeepSrc = s.Key("encrypt_keep_source").MustBool(false)
+	opts.UniformSnapshotTimestamp = s.Key("uniform_snapshot_timestamp").MustBool(false)
 
 	opts.Upload = s.Key("upload").MustString("none")
 	opts.UploadPrefix = s.Key("upload_prefix").MustString("")
@@ -950,6 +953,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin
 			opts.Username = cliOpts.Username
 		case "dbname":
 			opts.ConnDb = cliOpts.ConnDb
+		case "uniform-snapshot-timestamp":
+			opts.UniformSnapshotTimestamp = cliOpts.UniformSnapshotTimestamp
 		}
 	}
 

main.go

@@ -302,6 +302,12 @@ func run() (retVal error) {
 	}
 	defer db.Close()
 
+	// Generate a single datetime that will be used in all generated snapshot names
+	var snapshotTime time.Time
+	if opts.UniformSnapshotTimestamp {
+		snapshotTime = time.Now()
+	}
+
 	if !opts.DumpOnly {
 		if !db.superuser {
 			l.Infoln("connection user is not superuser, some information will not be dumped")
@@ -315,7 +321,7 @@ func run() (retVal error) {
 		} else {
 			l.Infoln("dumping globals without role passwords")
 		}
-		if err := dumpGlobals(opts.Directory, opts.TimeFormat, dumpRolePasswords, conninfo, producedFiles); err != nil {
+		if err := dumpGlobals(opts.Directory, opts.TimeFormat, dumpRolePasswords, conninfo, producedFiles, snapshotTime); err != nil {
 			return fmt.Errorf("pg_dumpall of globals failed: %w", err)
 		}
 
@@ -325,15 +331,15 @@ func run() (retVal error) {
 			perr *pgPrivError
 		)
 
-		if err := dumpSettings(opts.Directory, opts.TimeFormat, db, producedFiles); err != nil {
+		if err := dumpSettings(opts.Directory, opts.TimeFormat, db, producedFiles, snapshotTime); err != nil {
 			if errors.As(err, &verr) || errors.As(err, &perr) {
 				l.Warnln(err)
 			} else {
 				return fmt.Errorf("could not dump configuration parameters: %w", err)
 			}
 		}
 
-		if err := dumpConfigFiles(opts.Directory, opts.TimeFormat, db, producedFiles); err != nil {
+		if err := dumpConfigFiles(opts.Directory, opts.TimeFormat, db, producedFiles, snapshotTime); err != nil {
 			return fmt.Errorf("could not dump configuration files: %w", err)
 		}
 	}
@@ -384,6 +390,7 @@ func run() (retVal error) {
 			CipherPassphrase: passphrase,
 			CipherPublicKey:  publicKey,
 			EncryptKeepSrc:   opts.EncryptKeepSrc,
+			When:             snapshotTime,
 			ExitCode:         -1,
 			PgDumpVersion:    pgDumpVersion,
 		}
@@ -612,8 +619,6 @@ func (d *dump) dump(fc chan<- sumFileJob) error {
 		return fmt.Errorf("could not acquire lock for %s", dbname)
 	}
 
-	d.When = time.Now()
-
 	var fileEnd string
 	switch d.Options.Format {
 	case 'p':
@@ -630,6 +635,10 @@ func (d *dump) dump(fc chan<- sumFileJob) error {
 		fileEnd = "d"
 	}
 
+	if d.When.IsZero() {
+		d.When = time.Now()
+	}
+
 	file := formatDumpPath(d.Directory, d.TimeFormat, fileEnd, dbname, d.When, d.Options.CompressLevel)
 	formatOpt := fmt.Sprintf("-F%c", d.Options.Format)
 
@@ -898,7 +907,7 @@ func pgToolVersion(tool string) int {
 	return numver
 }
 
-func dumpGlobals(dir string, timeFormat string, withRolePasswords bool, conninfo *ConnInfo, fc chan<- sumFileJob) error {
+func dumpGlobals(dir string, timeFormat string, withRolePasswords bool, conninfo *ConnInfo, fc chan<- sumFileJob, snapshotTime time.Time) error {
 	command := execPath("pg_dumpall")
 	args := []string{"-g", "-w"}
 
@@ -930,7 +939,11 @@ func dumpGlobals(dir string, timeFormat string, withRolePasswords bool, conninfo
 		args = append(args, "--no-role-passwords")
 	}
 
-	file := formatDumpPath(dir, timeFormat, "sql", "pg_globals", time.Now(), 0)
+	if snapshotTime.IsZero() {
+		snapshotTime = time.Now()
+	}
+
+	file := formatDumpPath(dir, timeFormat, "sql", "pg_globals", snapshotTime, 0)
 	args = append(args, "-f", file)
 
 	if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
@@ -970,9 +983,12 @@ func dumpGlobals(dir string, timeFormat string, withRolePasswords bool, conninfo
 	return nil
 }
 
-func dumpSettings(dir string, timeFormat string, db *pg, fc chan<- sumFileJob) error {
+func dumpSettings(dir string, timeFormat string, db *pg, fc chan<- sumFileJob, snapshotTime time.Time) error {
+	if snapshotTime.IsZero() {
+		snapshotTime = time.Now()
+	}
 
-	file := formatDumpPath(dir, timeFormat, "out", "pg_settings", time.Now(), 0)
+	file := formatDumpPath(dir, timeFormat, "out", "pg_settings", snapshotTime, 0)
 
 	if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
 		return err
@@ -1000,9 +1016,12 @@ func dumpSettings(dir string, timeFormat string, db *pg, fc chan<- sumFileJob) e
 	return nil
 }
 
-func dumpConfigFiles(dir string, timeFormat string, db *pg, fc chan<- sumFileJob) error {
+func dumpConfigFiles(dir string, timeFormat string, db *pg, fc chan<- sumFileJob, snapshotTime time.Time) error {
 	for _, param := range []string{"hba_file", "ident_file"} {
-		file := formatDumpPath(dir, timeFormat, "out", param, time.Now(), 0)
+		if snapshotTime.IsZero() {
+			snapshotTime = time.Now()
+		}
+		file := formatDumpPath(dir, timeFormat, "out", param, snapshotTime, 0)
 
 		if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
 			return err

pg_back.conf

@@ -43,6 +43,10 @@ with_templates = false
 # Dump only databases, excluding configuration and globals
 dump_only = false
 
+# Apply a single consistent timestamp to all filenames in a snapshot
+# instead of using individual file creation times
+uniform_snapshot_timestamp = false
+
 # Format of the dump, understood by pg_dump. Possible values are
 # plain, custom, tar or directory.
 format = custom