snapshotting: ability to specify timestamp location != UTC (#801)
This PR adds a new field optional field `timestamp_location` that allows the user to specify a timezone different than the default UTC for use in the snapshot suffix. I took @mjasnik 's PR https://github.com/zrepl/zrepl/pull/785 and refactored+extended it as follows: * move all formatting logic into its own package * disallow `dense` and `human` with formats != UTC to protect users from stupidity * document behavior more clearly * regression test for existing users
This commit is contained in:
Christian Schwarz
committed by
GitHub
GitHub
parent
904c1512a3
commit
b9b9ad10cf
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/daemon/snapper/snapname"
|
||||
"github.com/zrepl/zrepl/util/suspendresumesafetimer"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
@@ -20,10 +21,15 @@ func cronFromConfig(fsf zfs.DatasetFilter, in config.SnapshottingCron) (*Cron, e
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "hook config error")
|
||||
}
|
||||
|
||||
formatter, err := snapname.New(in.Prefix, in.TimestampFormat, in.TimestampLocation)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "build snapshot name formatter")
|
||||
}
|
||||
|
||||
planArgs := planArgs{
|
||||
prefix: in.Prefix,
|
||||
timestampFormat: in.TimestampFormat,
|
||||
hooks: hooksList,
|
||||
formatter: formatter,
|
||||
hooks: hooksList,
|
||||
}
|
||||
return &Cron{config: in, fsf: fsf, planArgs: planArgs}, nil
|
||||
}
|
||||
|
||||
+4
-21
@@ -4,20 +4,19 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/snapper/snapname"
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type planArgs struct {
|
||||
prefix string
|
||||
timestampFormat string
|
||||
hooks *hooks.List
|
||||
formatter *snapname.Formatter
|
||||
hooks *hooks.List
|
||||
}
|
||||
|
||||
type plan struct {
|
||||
@@ -60,21 +59,6 @@ type snapProgress struct {
|
||||
runResults hooks.PlanReport
|
||||
}
|
||||
|
||||
func (plan *plan) formatNow(format string) string {
|
||||
now := time.Now().UTC()
|
||||
switch strings.ToLower(format) {
|
||||
case "dense":
|
||||
format = "20060102_150405_000"
|
||||
case "human":
|
||||
format = "2006-01-02_15:04:05"
|
||||
case "iso-8601":
|
||||
format = "2006-01-02T15:04:05.000Z"
|
||||
case "unix-seconds":
|
||||
return strconv.FormatInt(now.Unix(), 10)
|
||||
}
|
||||
return now.Format(format)
|
||||
}
|
||||
|
||||
func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
|
||||
|
||||
hookMatchCount := make(map[hooks.Hook]int, len(*plan.args.hooks))
|
||||
@@ -85,8 +69,7 @@ func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
|
||||
anyFsHadErr := false
|
||||
// TODO channel programs -> allow a little jitter?
|
||||
for fs, progress := range plan.snaps {
|
||||
suffix := plan.formatNow(plan.args.timestampFormat)
|
||||
snapname := fmt.Sprintf("%s%s", plan.args.prefix, suffix)
|
||||
snapname := plan.args.formatter.Format(time.Now())
|
||||
|
||||
ctx := logging.WithInjectedField(ctx, "fs", fs.ToString())
|
||||
ctx = logging.WithInjectedField(ctx, "snap", snapname)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/daemon/snapper/snapname"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
@@ -32,13 +33,17 @@ func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.Snap
|
||||
return nil, errors.Wrap(err, "hook config error")
|
||||
}
|
||||
|
||||
formatter, err := snapname.New(in.Prefix, in.TimestampFormat, in.TimestampLocation)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "build snapshot name formatter")
|
||||
}
|
||||
|
||||
args := periodicArgs{
|
||||
interval: in.Interval.Duration(),
|
||||
fsf: fsf,
|
||||
planArgs: planArgs{
|
||||
prefix: in.Prefix,
|
||||
timestampFormat: in.TimestampFormat,
|
||||
hooks: hookList,
|
||||
formatter: formatter,
|
||||
hooks: hookList,
|
||||
},
|
||||
// ctx and log is set in Run()
|
||||
}
|
||||
@@ -166,7 +171,7 @@ func periodicStateSyncUp(a periodicArgs, u updater) state {
|
||||
if err != nil {
|
||||
return onErr(err, u)
|
||||
}
|
||||
syncPoint, err := findSyncPoint(a.ctx, fss, a.planArgs.prefix, a.interval)
|
||||
syncPoint, err := findSyncPoint(a.ctx, fss, a.planArgs.formatter.Prefix(), a.interval)
|
||||
if err != nil {
|
||||
return onErr(err, u)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package snapname
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/snapper/snapname/timestamp"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type Formatter struct {
|
||||
prefix string
|
||||
timestamp *timestamp.Formatter
|
||||
}
|
||||
|
||||
func New(prefix, tsFormat, tsLocation string) (*Formatter, error) {
|
||||
timestamp, err := timestamp.New(tsFormat, tsLocation)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "build timestamp formatter")
|
||||
}
|
||||
formatter := &Formatter{
|
||||
prefix: prefix,
|
||||
timestamp: timestamp,
|
||||
}
|
||||
// Best-effort check to detect whether the result would be an invalid name.
|
||||
// Test two dates that in most places have will have different time zone offsets due to DST.
|
||||
check := func(t time.Time) error {
|
||||
testFormat := formatter.Format(t)
|
||||
if err := zfs.ComponentNamecheck(testFormat); err != nil {
|
||||
// testFormat last, can be quite long
|
||||
return fmt.Errorf("`invalid snapshot name would result from `prefix+$timestamp`: %s: %q", err, testFormat)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := check(time.Date(2020, 6, 1, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := check(time.Date(2020, 12, 1, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return formatter, nil
|
||||
}
|
||||
|
||||
func (f *Formatter) Format(now time.Time) string {
|
||||
return f.prefix + f.timestamp.Format(now)
|
||||
}
|
||||
|
||||
func (f *Formatter) Prefix() string {
|
||||
return f.prefix
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package timestamp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type Formatter struct {
|
||||
format func(time.Time) string
|
||||
location *time.Location
|
||||
}
|
||||
|
||||
func New(formatString string, locationString string) (*Formatter, error) {
|
||||
location, err := time.LoadLocation(locationString) // no shadow
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "load location from string %q", locationString)
|
||||
}
|
||||
makeFormatFunc := func(formatString string) (func(time.Time) string, error) {
|
||||
// NB: we use zfs.EntityNamecheck in higher-level code to filter out all invalid characters.
|
||||
// This check here is specifically so that we know for sure that the `+`=>`_` replacement
|
||||
// that we do in the returned func replaces exactly the timezone offset `+` and not some other `+`.
|
||||
if strings.Contains(formatString, "+") {
|
||||
return nil, fmt.Errorf("character '+' is not allowed in ZFS snapshot names and has special handling")
|
||||
}
|
||||
return func(t time.Time) string {
|
||||
res := t.Format(formatString)
|
||||
// if the formatString contains a time zone specifier
|
||||
// and the location would result in a positive offset to UTC
|
||||
// then the result of t.Format would contain a '+' sign.
|
||||
if isLocationPositiveOffsetToUTC(location) {
|
||||
// the only source of `+` can be the positive time zone offset because we disallowed `+` as a character in the format string
|
||||
res = strings.Replace(res, "+", "_", 1)
|
||||
}
|
||||
if strings.Contains(res, "+") {
|
||||
panic(fmt.Sprintf("format produced a string containing illegal character '+' that wasn't the expected case of positive time zone offset: format=%q location=%q unix=%q result=%q", formatString, location, t.Unix(), res))
|
||||
}
|
||||
return res
|
||||
}, nil
|
||||
}
|
||||
var formatFunc func(time.Time) string
|
||||
mustUseUtcError := func() error {
|
||||
return fmt.Errorf("format string requires UTC location")
|
||||
}
|
||||
switch strings.ToLower(formatString) {
|
||||
case "dense":
|
||||
if location != time.UTC {
|
||||
err = mustUseUtcError()
|
||||
} else {
|
||||
formatFunc, err = makeFormatFunc("20060102_150405_000")
|
||||
}
|
||||
case "human":
|
||||
if location != time.UTC {
|
||||
err = mustUseUtcError()
|
||||
} else {
|
||||
formatFunc, err = makeFormatFunc("2006-01-02_15:04:05")
|
||||
}
|
||||
case "iso-8601":
|
||||
formatFunc, err = makeFormatFunc("2006-01-02T15:04:05.000Z0700")
|
||||
case "unix-seconds":
|
||||
if location != time.UTC {
|
||||
// Technically not required because unix time is by definition in UTC
|
||||
// but let's make that clear to confused users...
|
||||
err = mustUseUtcError()
|
||||
} else {
|
||||
formatFunc = func(t time.Time) string {
|
||||
return strconv.FormatInt(t.Unix(), 10)
|
||||
}
|
||||
}
|
||||
default:
|
||||
formatFunc, err = makeFormatFunc(formatString)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "invalid format string %q or location %q", formatString, locationString)
|
||||
}
|
||||
return &Formatter{
|
||||
format: formatFunc,
|
||||
location: location,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isLocationPositiveOffsetToUTC(location *time.Location) bool {
|
||||
_, offsetSeconds := time.Now().In(location).Zone()
|
||||
return offsetSeconds > 0
|
||||
}
|
||||
|
||||
func (f *Formatter) Format(t time.Time) string {
|
||||
t = t.In(f.location)
|
||||
return f.format(t)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package timestamp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var utc, _ = time.LoadLocation("UTC")
|
||||
var berlin, _ = time.LoadLocation("Europe/Berlin")
|
||||
var nyc, _ = time.LoadLocation("America/New_York")
|
||||
|
||||
func TestAssumptionsAboutTimePackage(t *testing.T) {
|
||||
now := time.Now()
|
||||
assert.Equal(t, now.In(utc).Unix(), now.In(berlin).Unix(), "unix timestamp is always in UTC")
|
||||
}
|
||||
|
||||
func TestLegacyIso8601Format(t *testing.T) {
|
||||
// Before we allowed users to specify the location of the time to be formatted,
|
||||
// we always used UTC.
|
||||
// At the time, the `iso-8601` format used the following format string
|
||||
// "2006-01-02T15:04:05.000Z"
|
||||
// That format string's `Z` was never identified by the Go time package as a time zone specifier.
|
||||
// The correct way would have been
|
||||
// "2006-01-02T15:04:05.000Z07"
|
||||
// "2006-01-02T15:04:05.000Z0700"
|
||||
// "2006-01-02T15:04:05.000Z070000"
|
||||
// or any variation of how the minute / second offsets are displayed.
|
||||
// However, because of the forced location UTC, it didn't matter, because both format strings
|
||||
// evaluate to the same time string.
|
||||
//
|
||||
// This test is here to ensure that the legacy behavior is preserved for users who don't specify a non-UTC location
|
||||
// (UTC location is the default location at this time)
|
||||
|
||||
oldFormatString := "2006-01-02T15:04:05.000Z"
|
||||
now := time.Now()
|
||||
oldOutput := now.In(utc).Format(oldFormatString)
|
||||
|
||||
currentImpl, err := New("iso-8601", "UTC")
|
||||
require.NoError(t, err)
|
||||
currentOutput := currentImpl.Format(now)
|
||||
|
||||
assert.Equal(t, oldOutput, currentOutput, "legacy behavior of iso-8601 format is preserved")
|
||||
}
|
||||
|
||||
func TestIso8601PrintsTimeZoneOffset(t *testing.T) {
|
||||
f, err := New("iso-8601", "Europe/Berlin")
|
||||
require.NoError(t, err)
|
||||
out := f.Format(time.Date(2024, 5, 14, 21, 16, 23, 0, time.UTC))
|
||||
require.Equal(t, "2024-05-14T23:16:23.000_0200", out, "time zone offset is printed")
|
||||
}
|
||||
|
||||
func TestBuiltinFormatsErrorOnNonUtcLocation(t *testing.T) {
|
||||
var err error
|
||||
|
||||
expectMsg := `^.*: format string requires UTC location$`
|
||||
|
||||
_, err = New("dense", "Europe/Berlin")
|
||||
require.Regexp(t, expectMsg, err)
|
||||
|
||||
_, err = New("human", "Europe/Berlin")
|
||||
require.Regexp(t, expectMsg, err)
|
||||
|
||||
_, err = New("unix-seconds", "Europe/Berlin")
|
||||
require.Regexp(t, expectMsg, err)
|
||||
|
||||
_, err = New("iso-8601", "Europe/Berlin")
|
||||
require.NoError(t, err, "iso-8601 prints time zone, so non-UTC locations are allowed, see test TestIso8601PrintsTimeZoneOffset")
|
||||
}
|
||||
|
||||
func TestPositiveUtcOffsetDetection(t *testing.T) {
|
||||
assert.True(t, isLocationPositiveOffsetToUTC(berlin), "Berlin is UTC+1/UTC+2")
|
||||
assert.False(t, isLocationPositiveOffsetToUTC(utc), "UTC is UTC+0")
|
||||
assert.False(t, isLocationPositiveOffsetToUTC(nyc), "New York is UTC-5/UTC-4")
|
||||
}
|
||||
|
||||
func TestFormatCanReplacePlusWithUnderscore(t *testing.T) {
|
||||
f, err := New("2006-01-02_15:04:05-07:00:00", "Europe/Berlin")
|
||||
require.NoError(t, err)
|
||||
out := f.Format(time.Date(2024, 5, 14, 21, 16, 23, 0, time.UTC))
|
||||
require.Equal(t, "2024-05-14_23:16:23_02:00:00", out, "+ is replaced with _ so we can use the string in ZFS snapshots")
|
||||
}
|
||||
Reference in New Issue
Block a user