move implementation to internal/ directory (#828)

This commit is contained in:
Christian Schwarz
2024-10-18 19:21:17 +02:00
committed by GitHub
parent b9b9ad10cf
commit 908807bd59
360 changed files with 507 additions and 507 deletions
+158
View File
@@ -0,0 +1,158 @@
package snapper
import (
"context"
"fmt"
"sync"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon/hooks"
"github.com/zrepl/zrepl/internal/daemon/snapper/snapname"
"github.com/zrepl/zrepl/internal/util/suspendresumesafetimer"
"github.com/zrepl/zrepl/internal/zfs"
)
func cronFromConfig(fsf zfs.DatasetFilter, in config.SnapshottingCron) (*Cron, error) {
hooksList, err := hooks.ListFromConfig(&in.Hooks)
if err != nil {
return nil, errors.Wrap(err, "hook config error")
}
formatter, err := snapname.New(in.Prefix, in.TimestampFormat, in.TimestampLocation)
if err != nil {
return nil, errors.Wrap(err, "build snapshot name formatter")
}
planArgs := planArgs{
formatter: formatter,
hooks: hooksList,
}
return &Cron{config: in, fsf: fsf, planArgs: planArgs}, nil
}
type Cron struct {
config config.SnapshottingCron
fsf zfs.DatasetFilter
planArgs planArgs
mtx sync.RWMutex
running bool
wakeupTime time.Time // zero value means uninit
lastError error
lastPlan *plan
wakeupWhileRunningCount int
}
func (s *Cron) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
for {
now := time.Now()
s.mtx.Lock()
s.wakeupTime = s.config.Cron.Schedule.Next(now)
s.mtx.Unlock()
ctxDone := suspendresumesafetimer.SleepUntil(ctx, s.wakeupTime)
if ctxDone != nil {
return
}
getLogger(ctx).Debug("cron timer fired")
s.mtx.Lock()
if s.running {
getLogger(ctx).Warn("snapshotting triggered according to cron rules but previous snapshotting is not done; not taking a snapshot this time")
s.wakeupWhileRunningCount++
s.mtx.Unlock()
continue
}
s.lastError = nil
s.lastPlan = nil
s.wakeupWhileRunningCount = 0
s.running = true
s.mtx.Unlock()
go func() {
err := s.do(ctx)
s.mtx.Lock()
s.lastError = err
s.running = false
s.mtx.Unlock()
select {
case snapshotsTaken <- struct{}{}:
default:
if snapshotsTaken != nil {
getLogger(ctx).Warn("callback channel is full, discarding snapshot update event")
}
}
}()
}
}
func (s *Cron) do(ctx context.Context) error {
fss, err := zfs.ZFSListMapping(ctx, s.fsf)
if err != nil {
return errors.Wrap(err, "cannot list filesystems")
}
p := makePlan(s.planArgs, fss)
s.mtx.Lock()
s.lastPlan = p
s.lastError = nil
s.mtx.Unlock()
ok := p.execute(ctx, false)
if !ok {
return errors.New("one or more snapshots could not be created, check logs for details")
} else {
return nil
}
}
type CronState string
const (
CronStateRunning CronState = "running"
CronStateWaiting CronState = "waiting"
)
type CronReport struct {
State CronState
WakeupTime time.Time
Errors []string
Progress []*ReportFilesystem
}
func (s *Cron) Report() Report {
s.mtx.Lock()
defer s.mtx.Unlock()
r := CronReport{}
r.WakeupTime = s.wakeupTime
if s.running {
r.State = CronStateRunning
} else {
r.State = CronStateWaiting
}
if s.lastError != nil {
r.Errors = append(r.Errors, s.lastError.Error())
}
if s.wakeupWhileRunningCount > 0 {
r.Errors = append(r.Errors, fmt.Sprintf("cron frequency is too high; snapshots were not taken %d times", s.wakeupWhileRunningCount))
}
r.Progress = nil
if s.lastPlan != nil {
r.Progress = s.lastPlan.report()
}
return Report{Type: TypeCron, Cron: &r}
}
+68
View File
@@ -0,0 +1,68 @@
package snapper
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/internal/config"
)
func TestCronLibraryWorks(t *testing.T) {
type testCase struct {
spec string
in time.Time
expect time.Time
}
dhm := func(day, hour, minutes int) time.Time {
return time.Date(2022, 7, day, hour, minutes, 0, 0, time.UTC)
}
hm := func(hour, minutes int) time.Time {
return dhm(23, hour, minutes)
}
tcs := []testCase{
{"0-10 * * * *", dhm(17, 1, 10), dhm(17, 2, 0)},
{"0-10 * * * *", dhm(17, 23, 10), dhm(18, 0, 0)},
{"0-10 * * * *", hm(1, 9), hm(1, 10)},
{"0-10 * * * *", hm(1, 9), hm(1, 10)},
{"1,3,5 * * * *", hm(1, 1), hm(1, 3)},
{"1,3,5 * * * *", hm(1, 2), hm(1, 3)},
{"1,3,5 * * * *", hm(1, 3), hm(1, 5)},
{"1,3,5 * * * *", hm(1, 5), hm(2, 1)},
{"* 0-5,8,12 * * *", hm(0, 0), hm(0, 1)},
{"* 0-5,8,12 * * *", hm(4, 59), hm(5, 0)},
{"* 0-5,8,12 * * *", hm(5, 0), hm(5, 1)},
{"* 0-5,8,12 * * *", hm(5, 59), hm(8, 0)},
{"* 0-5,8,12 * * *", hm(8, 59), hm(12, 0)},
// https://github.com/zrepl/zrepl/pull/614#issuecomment-1188358989
{"53 17,18,19 * * *", dhm(23, 17, 52), dhm(23, 17, 53)},
{"53 17,18,19 * * *", dhm(23, 17, 53), dhm(23, 18, 53)},
{"53 17,18,19 * * *", dhm(23, 18, 53), dhm(23, 19, 53)},
{"53 17,18,19 * * *", dhm(23, 19, 53), dhm(24 /* ! */, 17, 53)},
}
for i, tc := range tcs {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var s struct {
Cron config.CronSpec `yaml:"cron"`
}
inp := fmt.Sprintf("cron: %q", tc.spec)
fmt.Println("spec is ", inp)
err := yaml.UnmarshalStrict([]byte(inp), &s)
require.NoError(t, err)
actual := s.Cron.Schedule.Next(tc.in)
assert.Equal(t, tc.expect, actual)
})
}
}
+251
View File
@@ -0,0 +1,251 @@
package snapper
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/zrepl/zrepl/internal/daemon/hooks"
"github.com/zrepl/zrepl/internal/daemon/logging"
"github.com/zrepl/zrepl/internal/daemon/snapper/snapname"
"github.com/zrepl/zrepl/internal/util/chainlock"
"github.com/zrepl/zrepl/internal/zfs"
)
type planArgs struct {
formatter *snapname.Formatter
hooks *hooks.List
}
type plan struct {
mtx chainlock.L
args planArgs
snaps map[*zfs.DatasetPath]*snapProgress
}
func makePlan(args planArgs, fss []*zfs.DatasetPath) *plan {
snaps := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
for _, fs := range fss {
snaps[fs] = &snapProgress{state: SnapPending}
}
return &plan{snaps: snaps, args: args}
}
//go:generate stringer -type=SnapState
type SnapState uint
const (
SnapPending SnapState = 1 << iota
SnapStarted
SnapDone
SnapError
)
// All fields protected by Snapper.mtx
type snapProgress struct {
state SnapState
// SnapStarted, SnapDone, SnapError
name string
startAt time.Time
hookPlan *hooks.Plan
// SnapDone
doneAt time.Time
// SnapErr TODO disambiguate state
runResults hooks.PlanReport
}
func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
hookMatchCount := make(map[hooks.Hook]int, len(*plan.args.hooks))
for _, h := range *plan.args.hooks {
hookMatchCount[h] = 0
}
anyFsHadErr := false
// TODO channel programs -> allow a little jitter?
for fs, progress := range plan.snaps {
snapname := plan.args.formatter.Format(time.Now())
ctx := logging.WithInjectedField(ctx, "fs", fs.ToString())
ctx = logging.WithInjectedField(ctx, "snap", snapname)
hookEnvExtra := hooks.Env{
hooks.EnvFS: fs.ToString(),
hooks.EnvSnapshot: snapname,
}
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
l := getLogger(ctx)
l.Debug("create snapshot")
err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
if err != nil {
l.WithError(err).Error("cannot create snapshot")
}
return
})
fsHadErr := false
var hookPlanReport hooks.PlanReport
var hookPlan *hooks.Plan
{
filteredHooks, err := plan.args.hooks.CopyFilteredForFilesystem(fs)
if err != nil {
getLogger(ctx).WithError(err).Error("unexpected filter error")
fsHadErr = true
goto updateFSState
}
// account for running hooks
for _, h := range filteredHooks {
hookMatchCount[h] = hookMatchCount[h] + 1
}
var planErr error
hookPlan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
if planErr != nil {
fsHadErr = true
getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
goto updateFSState
}
}
plan.mtx.HoldWhile(func() {
progress.name = snapname
progress.startAt = time.Now()
progress.hookPlan = hookPlan
progress.state = SnapStarted
})
{
getLogger(ctx).WithField("report", hookPlan.Report().String()).Debug("begin run job plan")
hookPlan.Run(ctx, dryRun)
hookPlanReport = hookPlan.Report()
fsHadErr = hookPlanReport.HadError() // not just fatal errors
if fsHadErr {
getLogger(ctx).WithField("report", hookPlanReport.String()).Error("end run job plan with error")
} else {
getLogger(ctx).WithField("report", hookPlanReport.String()).Info("end run job plan successful")
}
}
updateFSState:
anyFsHadErr = anyFsHadErr || fsHadErr
plan.mtx.HoldWhile(func() {
progress.doneAt = time.Now()
progress.state = SnapDone
if fsHadErr {
progress.state = SnapError
}
progress.runResults = hookPlanReport
})
}
for h, mc := range hookMatchCount {
if mc == 0 {
hookIdx := -1
for idx, ah := range *plan.args.hooks {
if ah == h {
hookIdx = idx
break
}
}
getLogger(ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
}
}
return !anyFsHadErr
}
type ReportFilesystem struct {
Path string
State SnapState
// Valid in SnapStarted and later
SnapName string
StartAt time.Time
Hooks string
HooksHadError bool
// Valid in SnapDone | SnapError
DoneAt time.Time
}
func (plan *plan) report() []*ReportFilesystem {
plan.mtx.Lock()
defer plan.mtx.Unlock()
pReps := make([]*ReportFilesystem, 0, len(plan.snaps))
for fs, p := range plan.snaps {
var hooksStr string
var hooksHadError bool
if p.hookPlan != nil {
hooksStr, hooksHadError = p.report()
}
pReps = append(pReps, &ReportFilesystem{
Path: fs.ToString(),
State: p.state,
SnapName: p.name,
StartAt: p.startAt,
DoneAt: p.doneAt,
Hooks: hooksStr,
HooksHadError: hooksHadError,
})
}
sort.Slice(pReps, func(i, j int) bool {
return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
})
return pReps
}
func (p *snapProgress) report() (hooksStr string, hooksHadError bool) {
hr := p.hookPlan.Report()
// FIXME: technically this belongs into client
// but we can't serialize hooks.Step ATM
rightPad := func(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
hooksHadError = hr.HadError()
rows := make([][]string, len(hr))
const numCols = 4
lens := make([]int, numCols)
for i, e := range hr {
rows[i] = make([]string, numCols)
rows[i][0] = fmt.Sprintf("%d", i+1)
rows[i][1] = e.Status.String()
runTime := "..."
if e.Status != hooks.StepPending {
runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
}
rows[i][2] = runTime
rows[i][3] = ""
if e.Report != nil {
rows[i][3] = e.Report.String()
}
for j, col := range lens {
if len(rows[i][j]) > col {
lens[j] = len(rows[i][j])
}
}
}
rowsFlat := make([]string, len(hr))
for i, r := range rows {
colsPadded := make([]string, len(r))
for j, c := range r[:len(r)-1] {
colsPadded[j] = rightPad(c, lens[j], " ")
}
colsPadded[len(r)-1] = r[len(r)-1]
rowsFlat[i] = strings.Join(colsPadded, " ")
}
hooksStr = strings.Join(rowsFlat, "\n")
return hooksStr, hooksHadError
}
+15
View File
@@ -0,0 +1,15 @@
package snapper
import (
"context"
)
type manual struct{}
func (s *manual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
// nothing to do
}
func (s *manual) Report() Report {
return Report{Type: TypeManual, Manual: &struct{}{}}
}
+399
View File
@@ -0,0 +1,399 @@
package snapper
import (
"context"
"fmt"
"sort"
"sync"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
"github.com/zrepl/zrepl/internal/daemon/snapper/snapname"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon/hooks"
"github.com/zrepl/zrepl/internal/daemon/logging"
"github.com/zrepl/zrepl/internal/util/envconst"
"github.com/zrepl/zrepl/internal/util/suspendresumesafetimer"
"github.com/zrepl/zrepl/internal/zfs"
)
func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.SnapshottingPeriodic) (*Periodic, error) {
if in.Prefix == "" {
return nil, errors.New("prefix must not be empty")
}
if in.Interval.Duration() <= 0 {
return nil, errors.New("interval must be positive")
}
hookList, err := hooks.ListFromConfig(&in.Hooks)
if err != nil {
return nil, errors.Wrap(err, "hook config error")
}
formatter, err := snapname.New(in.Prefix, in.TimestampFormat, in.TimestampLocation)
if err != nil {
return nil, errors.Wrap(err, "build snapshot name formatter")
}
args := periodicArgs{
interval: in.Interval.Duration(),
fsf: fsf,
planArgs: planArgs{
formatter: formatter,
hooks: hookList,
},
// ctx and log is set in Run()
}
return &Periodic{state: SyncUp, args: args}, nil
}
type periodicArgs struct {
ctx context.Context
interval time.Duration
fsf zfs.DatasetFilter
planArgs planArgs
snapshotsTaken chan<- struct{}
dryRun bool
}
type Periodic struct {
args periodicArgs
mtx sync.Mutex
state State
// set in state Plan, used in Waiting
lastInvocation time.Time
// valid for state Snapshotting
plan *plan
// valid for state SyncUp and Waiting
sleepUntil time.Time
// valid for state Err
err error
}
//go:generate stringer -type=State
type State uint
const (
SyncUp State = 1 << iota
SyncUpErrWait
Planning
Snapshotting
Waiting
ErrorWait
Stopped
)
func (s State) sf() state {
m := map[State]state{
SyncUp: periodicStateSyncUp,
SyncUpErrWait: periodicStateWait,
Planning: periodicStatePlan,
Snapshotting: periodicStateSnapshot,
Waiting: periodicStateWait,
ErrorWait: periodicStateWait,
Stopped: nil,
}
return m[s]
}
type updater func(u func(*Periodic)) State
type state func(a periodicArgs, u updater) state
func (s *Periodic) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
getLogger(ctx).Debug("start")
defer getLogger(ctx).Debug("stop")
s.args.snapshotsTaken = snapshotsTaken
s.args.ctx = ctx
s.args.dryRun = false // for future expansion
u := func(u func(*Periodic)) State {
s.mtx.Lock()
defer s.mtx.Unlock()
if u != nil {
u(s)
}
return s.state
}
var st state = periodicStateSyncUp
for st != nil {
pre := u(nil)
st = st(s.args, u)
post := u(nil)
getLogger(ctx).
WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
Debug("state transition")
}
}
func onErr(err error, u updater) state {
return u(func(s *Periodic) {
s.err = err
preState := s.state
switch s.state {
case SyncUp:
s.state = SyncUpErrWait
case Planning:
fallthrough
case Snapshotting:
s.state = ErrorWait
}
getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
}).sf()
}
func onMainCtxDone(ctx context.Context, u updater) state {
return u(func(s *Periodic) {
s.err = ctx.Err()
s.state = Stopped
}).sf()
}
func periodicStateSyncUp(a periodicArgs, u updater) state {
u(func(snapper *Periodic) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
syncPoint, err := findSyncPoint(a.ctx, fss, a.planArgs.formatter.Prefix(), a.interval)
if err != nil {
return onErr(err, u)
}
u(func(s *Periodic) {
s.sleepUntil = syncPoint
})
ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, syncPoint)
if ctxDone != nil {
return onMainCtxDone(a.ctx, u)
}
return u(func(s *Periodic) {
s.state = Planning
}).sf()
}
func periodicStatePlan(a periodicArgs, u updater) state {
u(func(snapper *Periodic) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
p := makePlan(a.planArgs, fss)
return u(func(s *Periodic) {
s.state = Snapshotting
s.plan = p
s.err = nil
}).sf()
}
func periodicStateSnapshot(a periodicArgs, u updater) state {
var plan *plan
u(func(snapper *Periodic) {
plan = snapper.plan
})
ok := plan.execute(a.ctx, false)
select {
case a.snapshotsTaken <- struct{}{}:
default:
if a.snapshotsTaken != nil {
getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
}
}
return u(func(snapper *Periodic) {
if !ok {
snapper.state = ErrorWait
snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
} else {
snapper.state = Waiting
snapper.err = nil
}
}).sf()
}
func periodicStateWait(a periodicArgs, u updater) state {
var sleepUntil time.Time
u(func(snapper *Periodic) {
lastTick := snapper.lastInvocation
snapper.sleepUntil = lastTick.Add(a.interval)
sleepUntil = snapper.sleepUntil
log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
logFunc := log.Debug
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
logFunc = log.Error
}
logFunc("enter wait-state after error")
})
ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, sleepUntil)
if ctxDone != nil {
return onMainCtxDone(a.ctx, u)
}
return u(func(snapper *Periodic) {
snapper.state = Planning
}).sf()
}
func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {
return zfs.ZFSListMapping(ctx, mf)
}
var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
// see docs/snapshotting.rst
func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
const (
prioHasVersions int = iota
prioNoVersions
)
type snapTime struct {
ds *zfs.DatasetPath
prio int // lower is higher
time time.Time
}
if len(fss) == 0 {
return time.Now(), nil
}
snaptimes := make([]snapTime, 0, len(fss))
hardErrs := 0
now := time.Now()
getLogger(ctx).Debug("examine filesystem state to find sync point")
for _, d := range fss {
ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
if err == findSyncPointFSNoFilesystemVersionsErr {
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioNoVersions,
time: now,
})
} else if err != nil {
hardErrs++
getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
} else {
getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioHasVersions,
time: syncPoint,
})
}
}
if hardErrs == len(fss) {
return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
}
if len(snaptimes) == 0 {
panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
}
// sort ascending by (prio,time)
// => those filesystems with versions win over those without any
sort.Slice(snaptimes, func(i, j int) bool {
if snaptimes[i].prio == snaptimes[j].prio {
return snaptimes[i].time.Before(snaptimes[j].time)
}
return snaptimes[i].prio < snaptimes[j].prio
})
winnerSyncPoint := snaptimes[0].time
l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
l.Info("determined sync point")
if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
for _, st := range snaptimes {
if st.prio == prioNoVersions {
l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
}
}
}
return snaptimes[0].time, nil
}
var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
Types: zfs.Snapshots,
ShortnamePrefix: prefix,
})
if err != nil {
return time.Time{}, errors.Wrap(err, "list filesystem versions")
}
if len(fsvs) <= 0 {
return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
}
// Sort versions by creation
sort.SliceStable(fsvs, func(i, j int) bool {
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
})
latest := fsvs[len(fsvs)-1]
getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
since := now.Sub(latest.Creation)
if since < 0 {
return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
}
return latest.Creation.Add(interval), nil
}
type PeriodicReport struct {
State State
// valid in state SyncUp and Waiting
SleepUntil time.Time
// valid in state Err
Error string
// valid in state Snapshotting
Progress []*ReportFilesystem
}
func (s *Periodic) Report() Report {
s.mtx.Lock()
defer s.mtx.Unlock()
var progress []*ReportFilesystem = nil
if s.plan != nil {
progress = s.plan.report()
}
r := &PeriodicReport{
State: s.state,
SleepUntil: s.sleepUntil,
Error: errOrEmptyString(s.err),
Progress: progress,
}
return Report{Type: TypePeriodic, Periodic: r}
}
@@ -0,0 +1,52 @@
package snapname
import (
"fmt"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/daemon/snapper/snapname/timestamp"
"github.com/zrepl/zrepl/internal/zfs"
)
type Formatter struct {
prefix string
timestamp *timestamp.Formatter
}
func New(prefix, tsFormat, tsLocation string) (*Formatter, error) {
timestamp, err := timestamp.New(tsFormat, tsLocation)
if err != nil {
return nil, errors.Wrap(err, "build timestamp formatter")
}
formatter := &Formatter{
prefix: prefix,
timestamp: timestamp,
}
// Best-effort check to detect whether the result would be an invalid name.
// Test two dates that in most places have will have different time zone offsets due to DST.
check := func(t time.Time) error {
testFormat := formatter.Format(t)
if err := zfs.ComponentNamecheck(testFormat); err != nil {
// testFormat last, can be quite long
return fmt.Errorf("`invalid snapshot name would result from `prefix+$timestamp`: %s: %q", err, testFormat)
}
return nil
}
if err := check(time.Date(2020, 6, 1, 0, 0, 0, 0, time.UTC)); err != nil {
return nil, err
}
if err := check(time.Date(2020, 12, 1, 0, 0, 0, 0, time.UTC)); err != nil {
return nil, err
}
return formatter, nil
}
func (f *Formatter) Format(now time.Time) string {
return f.prefix + f.timestamp.Format(now)
}
func (f *Formatter) Prefix() string {
return f.prefix
}
@@ -0,0 +1,93 @@
package timestamp
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
)
type Formatter struct {
format func(time.Time) string
location *time.Location
}
func New(formatString string, locationString string) (*Formatter, error) {
location, err := time.LoadLocation(locationString) // no shadow
if err != nil {
return nil, errors.Wrapf(err, "load location from string %q", locationString)
}
makeFormatFunc := func(formatString string) (func(time.Time) string, error) {
// NB: we use zfs.EntityNamecheck in higher-level code to filter out all invalid characters.
// This check here is specifically so that we know for sure that the `+`=>`_` replacement
// that we do in the returned func replaces exactly the timezone offset `+` and not some other `+`.
if strings.Contains(formatString, "+") {
return nil, fmt.Errorf("character '+' is not allowed in ZFS snapshot names and has special handling")
}
return func(t time.Time) string {
res := t.Format(formatString)
// if the formatString contains a time zone specifier
// and the location would result in a positive offset to UTC
// then the result of t.Format would contain a '+' sign.
if isLocationPositiveOffsetToUTC(location) {
// the only source of `+` can be the positive time zone offset because we disallowed `+` as a character in the format string
res = strings.Replace(res, "+", "_", 1)
}
if strings.Contains(res, "+") {
panic(fmt.Sprintf("format produced a string containing illegal character '+' that wasn't the expected case of positive time zone offset: format=%q location=%q unix=%q result=%q", formatString, location, t.Unix(), res))
}
return res
}, nil
}
var formatFunc func(time.Time) string
mustUseUtcError := func() error {
return fmt.Errorf("format string requires UTC location")
}
switch strings.ToLower(formatString) {
case "dense":
if location != time.UTC {
err = mustUseUtcError()
} else {
formatFunc, err = makeFormatFunc("20060102_150405_000")
}
case "human":
if location != time.UTC {
err = mustUseUtcError()
} else {
formatFunc, err = makeFormatFunc("2006-01-02_15:04:05")
}
case "iso-8601":
formatFunc, err = makeFormatFunc("2006-01-02T15:04:05.000Z0700")
case "unix-seconds":
if location != time.UTC {
// Technically not required because unix time is by definition in UTC
// but let's make that clear to confused users...
err = mustUseUtcError()
} else {
formatFunc = func(t time.Time) string {
return strconv.FormatInt(t.Unix(), 10)
}
}
default:
formatFunc, err = makeFormatFunc(formatString)
}
if err != nil {
return nil, errors.Wrapf(err, "invalid format string %q or location %q", formatString, locationString)
}
return &Formatter{
format: formatFunc,
location: location,
}, nil
}
func isLocationPositiveOffsetToUTC(location *time.Location) bool {
_, offsetSeconds := time.Now().In(location).Zone()
return offsetSeconds > 0
}
func (f *Formatter) Format(t time.Time) string {
t = t.In(f.location)
return f.format(t)
}
@@ -0,0 +1,84 @@
package timestamp
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var utc, _ = time.LoadLocation("UTC")
var berlin, _ = time.LoadLocation("Europe/Berlin")
var nyc, _ = time.LoadLocation("America/New_York")
func TestAssumptionsAboutTimePackage(t *testing.T) {
now := time.Now()
assert.Equal(t, now.In(utc).Unix(), now.In(berlin).Unix(), "unix timestamp is always in UTC")
}
func TestLegacyIso8601Format(t *testing.T) {
// Before we allowed users to specify the location of the time to be formatted,
// we always used UTC.
// At the time, the `iso-8601` format used the following format string
// "2006-01-02T15:04:05.000Z"
// That format string's `Z` was never identified by the Go time package as a time zone specifier.
// The correct way would have been
// "2006-01-02T15:04:05.000Z07"
// "2006-01-02T15:04:05.000Z0700"
// "2006-01-02T15:04:05.000Z070000"
// or any variation of how the minute / second offsets are displayed.
// However, because of the forced location UTC, it didn't matter, because both format strings
// evaluate to the same time string.
//
// This test is here to ensure that the legacy behavior is preserved for users who don't specify a non-UTC location
// (UTC location is the default location at this time)
oldFormatString := "2006-01-02T15:04:05.000Z"
now := time.Now()
oldOutput := now.In(utc).Format(oldFormatString)
currentImpl, err := New("iso-8601", "UTC")
require.NoError(t, err)
currentOutput := currentImpl.Format(now)
assert.Equal(t, oldOutput, currentOutput, "legacy behavior of iso-8601 format is preserved")
}
func TestIso8601PrintsTimeZoneOffset(t *testing.T) {
f, err := New("iso-8601", "Europe/Berlin")
require.NoError(t, err)
out := f.Format(time.Date(2024, 5, 14, 21, 16, 23, 0, time.UTC))
require.Equal(t, "2024-05-14T23:16:23.000_0200", out, "time zone offset is printed")
}
func TestBuiltinFormatsErrorOnNonUtcLocation(t *testing.T) {
var err error
expectMsg := `^.*: format string requires UTC location$`
_, err = New("dense", "Europe/Berlin")
require.Regexp(t, expectMsg, err)
_, err = New("human", "Europe/Berlin")
require.Regexp(t, expectMsg, err)
_, err = New("unix-seconds", "Europe/Berlin")
require.Regexp(t, expectMsg, err)
_, err = New("iso-8601", "Europe/Berlin")
require.NoError(t, err, "iso-8601 prints time zone, so non-UTC locations are allowed, see test TestIso8601PrintsTimeZoneOffset")
}
func TestPositiveUtcOffsetDetection(t *testing.T) {
assert.True(t, isLocationPositiveOffsetToUTC(berlin), "Berlin is UTC+1/UTC+2")
assert.False(t, isLocationPositiveOffsetToUTC(utc), "UTC is UTC+0")
assert.False(t, isLocationPositiveOffsetToUTC(nyc), "New York is UTC-5/UTC-4")
}
func TestFormatCanReplacePlusWithUnderscore(t *testing.T) {
f, err := New("2006-01-02_15:04:05-07:00:00", "Europe/Berlin")
require.NoError(t, err)
out := f.Format(time.Date(2024, 5, 14, 21, 16, 23, 0, time.UTC))
require.Equal(t, "2024-05-14_23:16:23_02:00:00", out, "+ is replaced with _ so we can use the string in ZFS snapshots")
}
+42
View File
@@ -0,0 +1,42 @@
package snapper
import (
"context"
"fmt"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/zfs"
)
type Type string
const (
TypePeriodic Type = "periodic"
TypeCron Type = "cron"
TypeManual Type = "manual"
)
type Snapper interface {
Run(ctx context.Context, snapshotsTaken chan<- struct{})
Report() Report
}
type Report struct {
Type Type
Periodic *PeriodicReport
Cron *CronReport
Manual *struct{}
}
func FromConfig(g *config.Global, fsf zfs.DatasetFilter, in config.SnapshottingEnum) (Snapper, error) {
switch v := in.Ret.(type) {
case *config.SnapshottingPeriodic:
return periodicFromConfig(g, fsf, v)
case *config.SnapshottingCron:
return cronFromConfig(fsf, *v)
case *config.SnapshottingManual:
return &manual{}, nil
default:
return nil, fmt.Errorf("unknown snapshotting type %T", v)
}
}
@@ -0,0 +1,39 @@
// Code generated by "stringer -type=SnapState"; DO NOT EDIT.
package snapper
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[SnapPending-1]
_ = x[SnapStarted-2]
_ = x[SnapDone-4]
_ = x[SnapError-8]
}
const (
_SnapState_name_0 = "SnapPendingSnapStarted"
_SnapState_name_1 = "SnapDone"
_SnapState_name_2 = "SnapError"
)
var (
_SnapState_index_0 = [...]uint8{0, 11, 22}
)
func (i SnapState) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _SnapState_name_0[_SnapState_index_0[i]:_SnapState_index_0[i+1]]
case i == 4:
return _SnapState_name_1
case i == 8:
return _SnapState_name_2
default:
return "SnapState(" + strconv.FormatInt(int64(i), 10) + ")"
}
}
+51
View File
@@ -0,0 +1,51 @@
// Code generated by "stringer -type=State"; DO NOT EDIT.
package snapper
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[SyncUp-1]
_ = x[SyncUpErrWait-2]
_ = x[Planning-4]
_ = x[Snapshotting-8]
_ = x[Waiting-16]
_ = x[ErrorWait-32]
_ = x[Stopped-64]
}
const (
_State_name_0 = "SyncUpSyncUpErrWait"
_State_name_1 = "Planning"
_State_name_2 = "Snapshotting"
_State_name_3 = "Waiting"
_State_name_4 = "ErrorWait"
_State_name_5 = "Stopped"
)
var (
_State_index_0 = [...]uint8{0, 6, 19}
)
func (i State) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _State_name_0[_State_index_0[i]:_State_index_0[i+1]]
case i == 4:
return _State_name_1
case i == 8:
return _State_name_2
case i == 16:
return _State_name_3
case i == 32:
return _State_name_4
case i == 64:
return _State_name_5
default:
return "State(" + strconv.FormatInt(int64(i), 10) + ")"
}
}
+21
View File
@@ -0,0 +1,21 @@
package snapper
import (
"context"
"github.com/zrepl/zrepl/internal/daemon/logging"
"github.com/zrepl/zrepl/internal/logger"
)
type Logger = logger.Logger
func getLogger(ctx context.Context) Logger {
return logging.GetLogger(ctx, logging.SubsysSnapshot)
}
func errOrEmptyString(e error) string {
if e != nil {
return e.Error()
}
return ""
}