adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move implementation to internal/ directory (#828)

Browse Source
This commit is contained in:
Christian Schwarz
2024-10-18 19:21:17 +02:00
committed by GitHub
parent b9b9ad10cf
commit 908807bd59
360 changed files with 507 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+702
View File
@@ -0,0 +1,702 @@
package config
import (
"fmt"
"log/syslog"
"os"
"time"
"github.com/pkg/errors"
"github.com/robfig/cron/v3"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/internal/util/datasizeunit"
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
)
type ParseFlags uint
const (
ParseFlagsNone ParseFlags = 0
ParseFlagsNoCertCheck ParseFlags = 1 << iota
)
type Config struct {
Jobs []JobEnum `yaml:"jobs,optional"`
Global *Global `yaml:"global,optional,fromdefaults"`
}
func (c *Config) Job(name string) (*JobEnum, error) {
for _, j := range c.Jobs {
if j.Name() == name {
return &j, nil
}
}
return nil, fmt.Errorf("job %q not defined in config", name)
}
type JobEnum struct {
Ret interface{}
}
func (j JobEnum) Name() string {
var name string
switch v := j.Ret.(type) {
case *SnapJob:
name = v.Name
case *PushJob:
name = v.Name
case *SinkJob:
name = v.Name
case *PullJob:
name = v.Name
case *SourceJob:
name = v.Name
default:
panic(fmt.Sprintf("unknown job type %T", v))
}
return name
}
type ActiveJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Connect ConnectEnum `yaml:"connect"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Replication *Replication `yaml:"replication,optional,fromdefaults"`
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
}
type ConflictResolution struct {
InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
}
type PassiveJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Serve ServeEnum `yaml:"serve"`
}
type SnapJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Pruning PruningLocal `yaml:"pruning"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"`
}
type SendOptions struct {
Encrypted bool `yaml:"encrypted,optional,default=false"`
Raw bool `yaml:"raw,optional,default=false"`
SendProperties bool `yaml:"send_properties,optional,default=false"`
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
Compressed bool `yaml:"compressed,optional,default=false"`
EmbeddedData bool `yaml:"embedded_data,optional,default=false"`
Saved bool `yaml:"saved,optional,default=false"`
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
}
type RecvOptions struct {
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
// Encrypted bool `yaml:"may_encrypted"`
// Future:
// Reencrypt bool `yaml:"reencrypt"`
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
}
var _ yaml.Unmarshaler = &datasizeunit.Bits{}
type BandwidthLimit struct {
Max datasizeunit.Bits `yaml:"max,default=-1 B"`
BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
}
type Replication struct {
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
}
type ReplicationOptionsProtection struct {
Initial string `yaml:"initial,optional,default=guarantee_resumability"`
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
}
type ReplicationOptionsConcurrency struct {
Steps int `yaml:"steps,optional,default=1"`
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
}
type PropertyRecvOptions struct {
Inherit []zfsprop.Property `yaml:"inherit,optional"`
Override map[zfsprop.Property]string `yaml:"override,optional"`
}
type PlaceholderRecvOptions struct {
Encryption string `yaml:"encryption,default=unspecified"`
}
type PushJob struct {
ActiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"`
Send *SendOptions `yaml:"send,fromdefaults,optional"`
}
func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
type PullJob struct {
ActiveJob `yaml:",inline"`
RootFS string `yaml:"root_fs"`
Interval PositiveDurationOrManual `yaml:"interval"`
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
}
func (j *PullJob) GetRootFS() string { return j.RootFS }
func (j *PullJob) GetAppendClientIdentity() bool { return false }
func (j *PullJob) GetRecvOptions() *RecvOptions { return j.Recv }
type PositiveDurationOrManual struct {
Interval time.Duration
Manual bool
}
var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var s string
if err := u(&s, true); err != nil {
return err
}
switch s {
case "manual":
i.Manual = true
i.Interval = 0
case "":
return fmt.Errorf("value must not be empty")
default:
i.Manual = false
i.Interval, err = parsePositiveDuration(s)
if err != nil {
return err
}
}
return nil
}
type SinkJob struct {
PassiveJob `yaml:",inline"`
RootFS string `yaml:"root_fs"`
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
}
func (j *SinkJob) GetRootFS() string { return j.RootFS }
func (j *SinkJob) GetAppendClientIdentity() bool { return true }
func (j *SinkJob) GetRecvOptions() *RecvOptions { return j.Recv }
type SourceJob struct {
PassiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
Filesystems FilesystemsFilter `yaml:"filesystems"`
Send *SendOptions `yaml:"send,optional,fromdefaults"`
}
func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
func (j *SourceJob) GetSendOptions() *SendOptions { return j.Send }
type FilesystemsFilter map[string]bool
type SnapshottingEnum struct {
Ret interface{}
}
type SnapshottingPeriodic struct {
Type string `yaml:"type"`
Prefix string `yaml:"prefix"`
Interval *PositiveDuration `yaml:"interval"`
Hooks HookList `yaml:"hooks,optional"`
TimestampFormattingSpec `yaml:",inline"`
}
type TimestampFormattingSpec struct {
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
}
type CronSpec struct {
Schedule cron.Schedule
}
var _ yaml.Unmarshaler = &CronSpec{}
func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
var specString string
if err := unmarshal(&specString, false); err != nil {
return err
}
// Use standard cron format.
// Disable the various "descriptors" (@daily, etc)
// They are just aliases to "top of hour", "midnight", etc.
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
sched, err := parser.Parse(specString)
if err != nil {
return errors.Wrap(err, "cron syntax invalid")
}
s.Schedule = sched
return nil
}
type SnapshottingCron struct {
Type string `yaml:"type"`
Prefix string `yaml:"prefix"`
Cron CronSpec `yaml:"cron"`
Hooks HookList `yaml:"hooks,optional"`
TimestampFormattingSpec `yaml:",inline"`
}
type SnapshottingManual struct {
Type string `yaml:"type"`
}
type PruningSenderReceiver struct {
KeepSender []PruningEnum `yaml:"keep_sender"`
KeepReceiver []PruningEnum `yaml:"keep_receiver"`
}
type PruningLocal struct {
Keep []PruningEnum `yaml:"keep"`
}
type LoggingOutletEnumList []LoggingOutletEnum
func (l *LoggingOutletEnumList) SetDefault() {
def := `
type: "stdout"
time: true
level: "warn"
format: "human"
`
s := &StdoutLoggingOutlet{}
err := yaml.UnmarshalStrict([]byte(def), &s)
if err != nil {
panic(err)
}
*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
}
var _ yaml.Defaulter = &LoggingOutletEnumList{}
type Global struct {
Logging *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
Control *GlobalControl `yaml:"control,optional,fromdefaults"`
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
}
type ConnectEnum struct {
Ret interface{}
}
type ConnectCommon struct {
Type string `yaml:"type"`
}
type TCPConnect struct {
ConnectCommon `yaml:",inline"`
Address string `yaml:"address,hostport"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type TLSConnect struct {
ConnectCommon `yaml:",inline"`
Address string `yaml:"address,hostport"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ServerCN string `yaml:"server_cn"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type SSHStdinserverConnect struct {
ConnectCommon `yaml:",inline"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port uint16 `yaml:"port"`
IdentityFile string `yaml:"identity_file"`
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
Options []string `yaml:"options,optional"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type LocalConnect struct {
ConnectCommon `yaml:",inline"`
ListenerName string `yaml:"listener_name"`
ClientIdentity string `yaml:"client_identity"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
}
type ServeEnum struct {
Ret interface{}
}
type ServeCommon struct {
Type string `yaml:"type"`
}
type TCPServe struct {
ServeCommon `yaml:",inline"`
Listen string `yaml:"listen,hostport"`
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
Clients map[string]string `yaml:"clients"`
}
type TLSServe struct {
ServeCommon `yaml:",inline"`
Listen string `yaml:"listen,hostport"`
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ClientCNs []string `yaml:"client_cns"`
HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
}
type StdinserverServer struct {
ServeCommon `yaml:",inline"`
ClientIdentities []string `yaml:"client_identities"`
}
type LocalServe struct {
ServeCommon `yaml:",inline"`
ListenerName string `yaml:"listener_name"`
}
type PruningEnum struct {
Ret interface{}
}
type PruneKeepNotReplicated struct {
Type string `yaml:"type"`
KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"`
}
type PruneKeepLastN struct {
Type string `yaml:"type"`
Count int `yaml:"count"`
Regex string `yaml:"regex,optional"`
}
type PruneKeepRegex struct { // FIXME rename to KeepRegex
Type string `yaml:"type"`
Regex string `yaml:"regex"`
Negate bool `yaml:"negate,optional,default=false"`
}
type LoggingOutletEnum struct {
Ret interface{}
}
type LoggingOutletCommon struct {
Type string `yaml:"type"`
Level string `yaml:"level"`
Format string `yaml:"format"`
}
type StdoutLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Time bool `yaml:"time,default=true"`
Color bool `yaml:"color,default=true"`
}
type SyslogLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
}
type TCPLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Address string `yaml:"address,hostport"`
Net string `yaml:"net,default=tcp"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
TLS *TCPLoggingOutletTLS `yaml:"tls,optional"`
}
type TCPLoggingOutletTLS struct {
CA string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
}
type MonitoringEnum struct {
Ret interface{}
}
type PrometheusMonitoring struct {
Type string `yaml:"type"`
Listen string `yaml:"listen,hostport"`
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
}
type SyslogFacility syslog.Priority
func (f *SyslogFacility) SetDefault() {
*f = SyslogFacility(syslog.LOG_LOCAL0)
}
var _ yaml.Defaulter = (*SyslogFacility)(nil)
type GlobalControl struct {
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
}
type GlobalServe struct {
StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
}
type GlobalStdinServer struct {
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
}
type HookList []HookEnum
type HookEnum struct {
Ret interface{}
}
type HookCommand struct {
Path string `yaml:"path"`
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
Filesystems FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
HookSettingsCommon `yaml:",inline"`
}
type HookPostgresCheckpoint struct {
HookSettingsCommon `yaml:",inline"`
DSN string `yaml:"dsn"`
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
Filesystems FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
}
type HookMySQLLockTables struct {
HookSettingsCommon `yaml:",inline"`
DSN string `yaml:"dsn"`
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
Filesystems FilesystemsFilter `yaml:"filesystems"`
}
type HookSettingsCommon struct {
Type string `yaml:"type"`
ErrIsFatal bool `yaml:"err_is_fatal,optional,default=false"`
}
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
var in struct {
Type string
}
if err := u(&in, true); err != nil {
return nil, err
}
if in.Type == "" {
return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
}
v, ok := types[in.Type]
if !ok {
return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
}
if err := u(v, false); err != nil {
return nil, err
}
return v, nil
}
func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"snap": &SnapJob{},
"push": &PushJob{},
"sink": &SinkJob{},
"pull": &PullJob{},
"source": &SourceJob{},
})
return
}
func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPConnect{},
"tls": &TLSConnect{},
"ssh+stdinserver": &SSHStdinserverConnect{},
"local": &LocalConnect{},
})
return
}
func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPServe{},
"tls": &TLSServe{},
"stdinserver": &StdinserverServer{},
"local": &LocalServe{},
})
return
}
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"not_replicated": &PruneKeepNotReplicated{},
"last_n": &PruneKeepLastN{},
"grid": &PruneGrid{},
"regex": &PruneKeepRegex{},
})
return
}
func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"periodic": &SnapshottingPeriodic{},
"manual": &SnapshottingManual{},
"cron": &SnapshottingCron{},
})
return
}
func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"stdout": &StdoutLoggingOutlet{},
"syslog": &SyslogLoggingOutlet{},
"tcp": &TCPLoggingOutlet{},
})
return
}
func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"prometheus": &PrometheusMonitoring{},
})
return
}
func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var s string
if err := u(&s, true); err != nil {
return err
}
var level syslog.Priority
switch s {
case "kern":
level = syslog.LOG_KERN
case "user":
level = syslog.LOG_USER
case "mail":
level = syslog.LOG_MAIL
case "daemon":
level = syslog.LOG_DAEMON
case "auth":
level = syslog.LOG_AUTH
case "syslog":
level = syslog.LOG_SYSLOG
case "lpr":
level = syslog.LOG_LPR
case "news":
level = syslog.LOG_NEWS
case "uucp":
level = syslog.LOG_UUCP
case "cron":
level = syslog.LOG_CRON
case "authpriv":
level = syslog.LOG_AUTHPRIV
case "ftp":
level = syslog.LOG_FTP
case "local0":
level = syslog.LOG_LOCAL0
case "local1":
level = syslog.LOG_LOCAL1
case "local2":
level = syslog.LOG_LOCAL2
case "local3":
level = syslog.LOG_LOCAL3
case "local4":
level = syslog.LOG_LOCAL4
case "local5":
level = syslog.LOG_LOCAL5
case "local6":
level = syslog.LOG_LOCAL6
case "local7":
level = syslog.LOG_LOCAL7
default:
return fmt.Errorf("invalid syslog level: %q", s)
}
*t = SyslogFacility(level)
return nil
}
func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"command": &HookCommand{},
"postgres-checkpoint": &HookPostgresCheckpoint{},
"mysql-lock-tables": &HookMySQLLockTables{},
})
return
}
var ConfigFileDefaultLocations = []string{
"/etc/zrepl/zrepl.yml",
"/usr/local/etc/zrepl/zrepl.yml",
}
func ParseConfig(path string) (i *Config, err error) {
if path == "" {
// Try default locations
for _, l := range ConfigFileDefaultLocations {
stat, statErr := os.Stat(l)
if statErr != nil {
continue
}
if !stat.Mode().IsRegular() {
err = errors.Errorf("file at default location is not a regular file: %s", l)
return
}
path = l
break
}
}
var bytes []byte
if bytes, err = os.ReadFile(path); err != nil {
return
}
return ParseConfigBytes(bytes)
}
func ParseConfigBytes(bytes []byte) (*Config, error) {
var c *Config
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
return nil, err
}
if c != nil {
return c, nil
}
// There was no yaml document in the file, deserialize from default.
// => See TestFromdefaultsEmptyDoc in yaml-config package.
if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
return nil, err
}
if c == nil {
panic("the fallback to deserialize from `{}` should work")
}
return c, nil
}
internal/config/config_dep_test.go
+90
View File
@@ -0,0 +1,90 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
)
type A struct {
B *B `yaml:"b,optional,fromdefaults"`
A1 string `yaml:"a1,optional"`
}
type B struct {
C *C `yaml:"c,optional,fromdefaults"`
D string `yaml:"d,default=ddd"`
E string `yaml:"e,optional"`
}
type C struct {
Q string `yaml:"q,optional"`
R string `yaml:"r,default=r"`
}
func TestDepFromDefaults(t *testing.T) {
type testcase struct {
name string
yaml string
expect *A
}
tcs := []testcase{
{
name: "empty",
yaml: `{}`,
expect: &A{
B: &B{
C: &C{
R: "r",
},
D: "ddd",
},
},
},
{
name: "a1 set",
yaml: `{"a1":"blah"}`,
expect: &A{
A1: "blah",
B: &B{
C: &C{
R: "r",
},
D: "ddd",
},
},
},
{
name: "D set",
yaml: `
b:
d: 4d
`,
expect: &A{
B: &B{
D: "4d",
C: &C{
R: "r",
},
},
},
},
}
for tci := range tcs {
t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
tc := tcs[tci]
var a A
err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
require.NoError(t, err)
require.Equal(t, tc.expect, &a)
})
}
}
internal/config/config_duration.go
+106
View File
@@ -0,0 +1,106 @@
package config
import (
"errors"
"fmt"
"regexp"
"strconv"
"time"
"github.com/kr/pretty"
"github.com/zrepl/yaml-config"
)
type Duration struct{ d time.Duration }
func (d Duration) Duration() time.Duration { return d.d }
var _ yaml.Unmarshaler = &Duration{}
func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
var s string
err := unmarshal(&s, false)
if err != nil {
return err
}
d.d, err = parseDuration(s)
if err != nil {
d.d = 0
return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
}
return nil
}
type PositiveDuration struct{ d Duration }
var _ yaml.Unmarshaler = &PositiveDuration{}
func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
err := d.d.UnmarshalYAML(unmarshal)
if err != nil {
return err
}
if d.d.Duration() <= 0 {
return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
}
return nil
}
func parsePositiveDuration(e string) (time.Duration, error) {
d, err := parseDuration(e)
if err != nil {
return d, err
}
if d <= 0 {
return 0, errors.New("duration must be positive integer")
}
return d, err
}
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
func parseDuration(e string) (d time.Duration, err error) {
comps := durationStringRegex.FindStringSubmatch(e)
if comps == nil {
err = fmt.Errorf("must match %s", durationStringRegex)
return
}
if len(comps) != 3 {
panic(pretty.Sprint(comps))
}
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
if err != nil {
return 0, err
}
var durationUnit time.Duration
switch comps[2] {
case "":
if durationFactor != 0 {
err = fmt.Errorf("missing time unit")
return
} else {
// It's the case where user specified '0'.
// We want to allow this, just like time.ParseDuration.
}
case "s":
durationUnit = time.Second
case "m":
durationUnit = time.Minute
case "h":
durationUnit = time.Hour
case "d":
durationUnit = 24 * time.Hour
case "w":
durationUnit = 24 * 7 * time.Hour
default:
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
return
}
d = time.Duration(durationFactor) * durationUnit
return
}
internal/config/config_global_test.go
+115
View File
@@ -0,0 +1,115 @@
package config
import (
"fmt"
"log/syslog"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
)
func testValidGlobalSection(t *testing.T, s string) *Config {
jobdef := `
jobs:
- name: dummyjob
type: sink
serve:
type: tcp
listen: ":2342"
clients: {
"10.0.0.1":"foo"
}
root_fs: zroot/foo
`
_, err := ParseConfigBytes([]byte(jobdef))
require.NoError(t, err)
return testValidConfig(t, s+jobdef)
}
func TestOutletTypes(t *testing.T) {
conf := testValidGlobalSection(t, `
global:
logging:
- type: stdout
level: debug
format: human
- type: syslog
level: info
retry_interval: 20s
format: human
- type: tcp
level: debug
format: json
address: logserver.example.com:1234
- type: tcp
level: debug
format: json
address: encryptedlogserver.example.com:1234
retry_interval: 20s
tls:
ca: /etc/zrepl/log/ca.crt
cert: /etc/zrepl/log/key.pem
key: /etc/zrepl/log/cert.pem
`)
assert.Equal(t, 4, len(*conf.Global.Logging))
assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
}
func TestDefaultLoggingOutlet(t *testing.T) {
conf := testValidGlobalSection(t, "")
assert.Equal(t, 1, len(*conf.Global.Logging))
o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
assert.Equal(t, "warn", o.Level)
assert.Equal(t, "human", o.Format)
}
func TestPrometheusMonitoring(t *testing.T) {
conf := testValidGlobalSection(t, `
global:
monitoring:
- type: prometheus
listen: ':9811'
`)
assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
}
func TestSyslogLoggingOutletFacility(t *testing.T) {
type SyslogFacilityPriority struct {
Facility string
Priority syslog.Priority
}
syslogFacilitiesPriorities := []SyslogFacilityPriority{
{"", syslog.LOG_LOCAL0}, // default
{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
}
for _, sFP := range syslogFacilitiesPriorities {
logcfg := fmt.Sprintf(`
global:
logging:
- type: syslog
level: info
format: human
facility: %s
`, sFP.Facility)
conf := testValidGlobalSection(t, logcfg)
assert.Equal(t, 1, len(*conf.Global.Logging))
assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
}
}
func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
e := &LoggingOutletEnumList{}
var i yaml.Defaulter = e
require.NotPanics(t, func() {
i.SetDefault()
assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
})
}
internal/config/config_jobid_test.go
+1
View File
@@ -0,0 +1 @@
package config
internal/config/config_minimal_test.go
+32
View File
@@ -0,0 +1,32 @@
package config
import (
"testing"
)
func TestJobsOnlyWorks(t *testing.T) {
testValidConfig(t, `
jobs:
- name: push
type: push
# snapshot the filesystems matched by the left-hand-side of the mapping
# every 10m with zrepl_ as prefix
connect:
type: tcp
address: localhost:2342
filesystems: {
"pool1/var/db<": true,
"pool1/usr/home<": true,
"pool1/usr/home/paranoid": false, #don't backup paranoid user
"pool1/poudriere/ports<": false #don't backup the ports trees
}
snapshotting:
type: manual
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: last_n
count: 1
`)
}
internal/config/config_positiveintervalormanual_test.go
+41
View File
@@ -0,0 +1,41 @@
package config
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/zrepl/yaml-config"
)
func TestPositiveDurationOrManual(t *testing.T) {
cases := []struct {
Comment, Input string
Result *PositiveDurationOrManual
}{
{"empty is error", "", nil},
{"negative is error", "-1s", nil},
{"zero seconds is error", "0s", nil},
{"zero is error", "0", nil},
{"non-manual is error", "something", nil},
{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
}
for _, tc := range cases {
t.Run(tc.Comment, func(t *testing.T) {
var out struct {
FieldName PositiveDurationOrManual `yaml:"fieldname"`
}
input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
err := yaml.UnmarshalStrict([]byte(input), &out)
if tc.Result == nil {
assert.Error(t, err)
t.Logf("%#v", out)
} else {
assert.Equal(t, *tc.Result, out.FieldName)
}
})
}
}
internal/config/config_recv_test.go
+134
View File
@@ -0,0 +1,134 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
)
func TestRecvOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: pull
connect:
type: local
listener_name: foo
client_identity: bar
root_fs: "zreplplatformtest"
%s
interval: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
recv_properties_empty := `
recv:
properties:
`
recv_inherit_empty := `
recv:
properties:
inherit:
`
recv_inherit := `
recv:
properties:
inherit:
- testprop
`
recv_override_empty := `
recv:
properties:
override:
`
recv_override := `
recv:
properties:
override:
testprop2: "test123"
`
recv_override_and_inherit := `
recv:
properties:
inherit:
- testprop
override:
testprop2: "test123"
`
recv_empty := `
recv: {}
`
recv_not_specified := `
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
t.Run("recv_inherit_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit_empty))
assert.NotNil(t, c)
})
t.Run("recv_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
})
t.Run("recv_override_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_empty))
assert.NotNil(t, c)
})
t.Run("recv_override", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override))
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
require.Len(t, override, 1)
require.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_override_and_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_and_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
assert.NotEmpty(t, override)
assert.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_properties_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_properties_empty))
assert.NotNil(t, c)
})
t.Run("recv_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_empty))
assert.NotNil(t, c)
})
t.Run("send_not_specified", func(t *testing.T) {
c := testValidConfig(t, fill(recv_not_specified))
assert.NotNil(t, c)
})
}
internal/config/config_send_test.go
+146
View File
@@ -0,0 +1,146 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSendOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
type: local
listener_name: foo
client_identity: bar
filesystems: {"<": true}
%s
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
encrypted_false := `
send:
encrypted: false
`
encrypted_true := `
send:
encrypted: true
`
raw_true := `
send:
raw: true
`
raw_false := `
send:
raw: false
`
raw_and_encrypted := `
send:
encrypted: true
raw: true
`
properties_and_encrypted := `
send:
encrypted: true
send_properties: true
`
properties_true := `
send:
send_properties: true
`
properties_false := `
send:
send_properties: false
`
send_empty := `
send: {}
`
send_not_specified := `
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
t.Run("encrypted_false", func(t *testing.T) {
c := testValidConfig(t, fill(encrypted_false))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("encrypted_true", func(t *testing.T) {
c := testValidConfig(t, fill(encrypted_true))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, encrypted)
})
t.Run("send_empty", func(t *testing.T) {
c := testValidConfig(t, fill(send_empty))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("properties_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(properties_and_encrypted))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, encrypted)
assert.Equal(t, true, properties)
})
t.Run("properties_false", func(t *testing.T) {
c := testValidConfig(t, fill(properties_false))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, false, properties)
})
t.Run("properties_true", func(t *testing.T) {
c := testValidConfig(t, fill(properties_true))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, properties)
})
t.Run("raw_true", func(t *testing.T) {
c := testValidConfig(t, fill(raw_true))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, true, raw)
})
t.Run("raw_false", func(t *testing.T) {
c := testValidConfig(t, fill(raw_false))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, false, raw)
})
t.Run("raw_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(raw_and_encrypted))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, raw)
assert.Equal(t, true, encrypted)
})
t.Run("send_not_specified", func(t *testing.T) {
c := testValidConfig(t, fill(send_not_specified))
assert.NotNil(t, c)
})
}
internal/config/config_snapshotting_test.go
+178
View File
@@ -0,0 +1,178 @@
package config
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestSnapshotting(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
type: local
listener_name: foo
client_identity: bar
filesystems: {"<": true}
%s
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
manual := `
snapshotting:
type: manual
`
periodic := `
snapshotting:
type: periodic
prefix: zrepl_
timestamp_format: dense
interval: 10m
`
cron := `
snapshotting:
type: cron
prefix: zrepl_
timestamp_format: human
cron: "10 * * * *"
`
periodicDaily := `
snapshotting:
type: periodic
prefix: zrepl_
interval: 1d
`
hooks := `
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
hooks:
- type: command
path: /tmp/path/to/command
- type: command
path: /tmp/path/to/command
filesystems: { "zroot<": true, "<": false }
- type: postgres-checkpoint
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
filesystems: {
"tank/postgres/data11": true
}
- type: mysql-lock-tables
dsn: "root@tcp(localhost)/"
filesystems: {
"tank/mysql": true
}
`
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
var c *Config
t.Run("manual", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(manual))
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
assert.Equal(t, "manual", snm.Type)
})
t.Run("periodic", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(periodic))
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
assert.Equal(t, "periodic", snp.Type)
assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
assert.Equal(t, "zrepl_", snp.Prefix)
})
t.Run("periodicDaily", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(periodicDaily))
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
assert.Equal(t, "periodic", snp.Type)
assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
assert.Equal(t, "zrepl_", snp.Prefix)
assert.Equal(t, "dense", snp.TimestampFormat)
})
t.Run("cron", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(cron))
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
assert.Equal(t, "cron", snp.Type)
assert.Equal(t, "zrepl_", snp.Prefix)
assert.Equal(t, "human", snp.TimestampFormat)
})
t.Run("hooks", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(hooks))
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
})
}
func TestSnapshottingTimestampDefaults(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
type: local
listener_name: foo
client_identity: bar
filesystems: {"<": true}
%s
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
periodic := `
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
`
cron := `
snapshotting:
type: cron
prefix: zrepl_
cron: "10 * * * *"
`
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
var c *Config
t.Run("periodic", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(periodic))
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
assert.Equal(t, "periodic", snp.Type)
assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
assert.Equal(t, "zrepl_", snp.Prefix)
assert.Equal(t, snp.TimestampFormat, "dense")
assert.Equal(t, snp.TimestampLocation, "UTC")
})
t.Run("cron", func(t *testing.T) {
c = testValidConfig(t, fillSnapshotting(cron))
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
assert.Equal(t, "cron", snp.Type)
assert.Equal(t, "zrepl_", snp.Prefix)
assert.Equal(t, snp.TimestampFormat, "dense")
assert.Equal(t, snp.TimestampLocation, "UTC")
})
}
internal/config/config_test.go
+157
View File
@@ -0,0 +1,157 @@
package config
import (
"bufio"
"bytes"
"fmt"
"path"
"path/filepath"
"strings"
"testing"
"text/template"
"github.com/kr/pretty"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
paths, err := filepath.Glob("./samples/*")
if err != nil {
t.Errorf("glob failed: %+v", err)
}
paths = append(paths, "../../packaging/systemd-default-zrepl.yml")
for _, p := range paths {
if path.Ext(p) != ".yml" {
t.Logf("skipping file %s", p)
continue
}
t.Run(p, func(t *testing.T) {
c, err := ParseConfig(p)
if err != nil {
t.Errorf("error parsing %s:\n%+v", p, err)
}
t.Logf("file: %s", p)
t.Log(pretty.Sprint(c))
})
}
}
// template must be a template/text template with a single '{{ . }}' as placeholder for val
//
//nolint:deadcode,unused
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
tmp, err := template.New("master").Parse(tmpl)
if err != nil {
panic(err)
}
var buf bytes.Buffer
err = tmp.Execute(&buf, val)
if err != nil {
panic(err)
}
return testValidConfig(t, buf.String())
}
func testValidConfig(t *testing.T, input string) *Config {
t.Helper()
conf, err := testConfig(t, input)
require.NoError(t, err)
require.NotNil(t, conf)
return conf
}
func testConfig(t *testing.T, input string) (*Config, error) {
t.Helper()
return ParseConfigBytes([]byte(input))
}
func trimSpaceEachLineAndPad(s, pad string) string {
var out strings.Builder
scan := bufio.NewScanner(strings.NewReader(s))
for scan.Scan() {
fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
}
return out.String()
}
func TestTrimSpaceEachLineAndPad(t *testing.T) {
foo := `
foo
bar baz
`
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
}
func TestCronSpec(t *testing.T) {
expectAccept := []string{
`"* * * * *"`,
`"0-10 * * * *"`,
`"* 0-5,8,12 * * *"`,
}
expectFail := []string{
`* * * *`,
``,
`23`,
`"@reboot"`,
`"@every 1h30m"`,
`"@daily"`,
`* * * * * *`,
}
for _, input := range expectAccept {
t.Run(input, func(t *testing.T) {
s := fmt.Sprintf("spec: %s\n", input)
var v struct {
Spec CronSpec
}
v.Spec.Schedule = nil
t.Logf("input:\n%s", s)
err := yaml.UnmarshalStrict([]byte(s), &v)
t.Logf("error: %T %s", err, err)
require.NoError(t, err)
require.NotNil(t, v.Spec.Schedule)
})
}
for _, input := range expectFail {
t.Run(input, func(t *testing.T) {
s := fmt.Sprintf("spec: %s\n", input)
var v struct {
Spec CronSpec
}
v.Spec.Schedule = nil
t.Logf("input: %q", s)
err := yaml.UnmarshalStrict([]byte(s), &v)
t.Logf("error: %T %s", err, err)
require.Error(t, err)
require.Nil(t, v.Spec.Schedule)
})
}
}
func TestEmptyConfig(t *testing.T) {
cases := []string{
"",
"\n",
"---",
"---\n",
}
for _, input := range cases {
config := testValidConfig(t, input)
require.NotNil(t, config)
require.NotNil(t, config.Global)
require.Empty(t, config.Jobs)
}
}
internal/config/config_transport_test.go
+99
View File
@@ -0,0 +1,99 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestTransportConnect(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
%s
filesystems: {"<": true}
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
type test struct {
Name string
ExpectError bool
Connect string
}
testTable := []test{
{
Name: "tcp_with_address_and_port",
ExpectError: false,
Connect: `
type: tcp
address: 10.0.0.23:42
`,
},
{
Name: "tls_with_host_and_port",
ExpectError: false,
Connect: `
type: tls
address: "server1.foo.bar:8888"
ca: /etc/zrepl/ca.crt
cert: /etc/zrepl/backupserver.fullchain
key: /etc/zrepl/backupserver.key
server_cn: "server1"
`,
},
{
Name: "tcp_without_port",
ExpectError: true,
Connect: `
type: tcp
address: 10.0.0.23
`,
},
{
Name: "tls_without_port",
ExpectError: true,
Connect: `
type: tls
address: 10.0.0.23
ca: /etc/zrepl/ca.crt
cert: /etc/zrepl/backupserver.fullchain
key: /etc/zrepl/backupserver.key
server_cn: "server1"
`,
},
}
for _, tc := range testTable {
t.Run(tc.Name, func(t *testing.T) {
require.NotEmpty(t, tc.Connect)
connect := trimSpaceEachLineAndPad(tc.Connect, " ")
conf := mconf(connect)
config, err := testConfig(t, conf)
if tc.ExpectError && err == nil {
t.Errorf("expected test failure, but got valid config %v", config)
return
}
if !tc.ExpectError && err != nil {
t.Errorf("not expecint test failure but got error: %s", err)
return
}
t.Logf("error=%v", err)
t.Logf("config=%v", config)
})
}
}
internal/config/retentiongrid.go
+119
View File
@@ -0,0 +1,119 @@
package config
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
type RetentionIntervalList []RetentionInterval
type PruneGrid struct {
Type string `yaml:"type"`
Grid RetentionIntervalList `yaml:"grid"`
Regex string `yaml:"regex"`
}
type RetentionInterval struct {
length time.Duration
keepCount int
}
func (i *RetentionInterval) Length() time.Duration {
return i.length
}
func (i *RetentionInterval) KeepCount() int {
return i.keepCount
}
const RetentionGridKeepCountAll int = -1
func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var in string
if err := u(&in, true); err != nil {
return err
}
intervals, err := ParseRetentionIntervalSpec(in)
if err != nil {
return err
}
*t = intervals
return nil
}
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
if comps == nil {
err = fmt.Errorf("retention string does not match expected format")
return
}
times, err := strconv.Atoi(comps[1])
if err != nil {
return nil, err
} else if times <= 0 {
return nil, fmt.Errorf("contains factor <= 0")
}
duration, err := parsePositiveDuration(comps[2])
if err != nil {
return nil, err
}
keepCount := 1
if comps[3] != "" {
// Decompose key=value, comma separated
// For now, only keep_count is supported
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
res := re.FindStringSubmatch(comps[4])
if res == nil || len(res) != 2 {
err = fmt.Errorf("interval parameter contains unknown parameters")
return
}
if res[1] == "all" {
keepCount = RetentionGridKeepCountAll
} else {
keepCount, err = strconv.Atoi(res[1])
if err != nil {
err = fmt.Errorf("cannot parse keep_count value")
return
}
}
}
intervals = make([]RetentionInterval, times)
for i := range intervals {
intervals[i] = RetentionInterval{
length: duration,
keepCount: keepCount,
}
}
return
}
func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
ges := strings.Split(s, "|")
intervals = make([]RetentionInterval, 0, 7*len(ges))
for intervalIdx, e := range ges {
parsed, err := parseRetentionGridIntervalString(e)
if err != nil {
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
}
intervals = append(intervals, parsed...)
}
return
}
internal/config/samples/bandwidth_limit.yml
+41
View File
@@ -0,0 +1,41 @@
jobs:
- type: sink
name: "limited_sink"
root_fs: "fs0"
recv:
bandwidth_limit:
max: 12345 B
serve:
type: local
listener_name: localsink
- type: push
name: "limited_push"
connect:
type: local
listener_name: localsink
client_identity: local_backup
filesystems: {
"root<": true,
}
send:
bandwidth_limit:
max: 54321 B
bucket_capacity: 1024 B
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 1
keep_receiver:
- type: last_n
count: 1
- type: sink
name: "nolimit_sink"
root_fs: "fs1"
serve:
type: local
listener_name: localsink
internal/config/samples/empty.yml
View File
internal/config/samples/hooks/template.sh
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -e
[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
pre_snapshot() {
$DRYRUN date
}
post_snapshot() {
$DRYRUN date
}
case "$ZREPL_HOOKTYPE" in
pre_snapshot|post_snapshot)
"$ZREPL_HOOKTYPE"
;;
*)
printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
exit 255
;;
esac
internal/config/samples/local.yml
+31
View File
@@ -0,0 +1,31 @@
jobs:
- type: sink
name: "local_sink"
root_fs: "storage/zrepl/sink"
serve:
type: local
listener_name: localsink
- type: push
name: "backup_system"
connect:
type: local
listener_name: localsink
client_identity: local_backup
filesystems: {
"system<": true,
}
snapshotting:
type: periodic
interval: 10m
prefix: zrepl_
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
regex: "zrepl_.*"
internal/config/samples/pull.yml
+24
View File
@@ -0,0 +1,24 @@
jobs:
- name: pull_servers
type: pull
connect:
type: tls
address: "server1.foo.bar:8888"
ca: "/certs/ca.crt"
cert: "/certs/cert.crt"
key: "/certs/key.pem"
server_cn: "server1"
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
regex: "zrepl_.*"
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
regex: "zrepl_.*"
internal/config/samples/pull_ssh.yml
+28
View File
@@ -0,0 +1,28 @@
jobs:
- name: pull_servers
type: pull
connect:
type: ssh+stdinserver
host: app-srv.example.com
user: root
port: 22
identity_file: /etc/zrepl/ssh/identity
options: # optional, default [], `-o` arguments passed to ssh
- "Compression=yes"
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
regex: "^zrepl_.*"
keep_receiver:
- type: regex
regex: keep_
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
regex: "^zrepl_.*"
internal/config/samples/push.yml
+26
View File
@@ -0,0 +1,26 @@
jobs:
- type: push
name: "push"
filesystems: {
"<": true,
"tmp": false
}
connect:
type: tcp
address: "backup-server.foo.bar:8888"
snapshotting:
type: manual
send:
encrypted: false
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
regex: "^zrepl_.*"
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
regex: "^zrepl_.*"
internal/config/samples/quickstart_backup_to_external_disk.yml
+88
View File
@@ -0,0 +1,88 @@
# This config serves as an example for a local zrepl installation that
# backups the entire zpool `system` to `backuppool/zrepl/sink`
#
# The requirements covered by this setup are described in the zrepl documentation's
# quick start section which inlines this example.
#
# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
jobs:
# this job takes care of snapshot creation + pruning
- name: snapjob
type: snap
filesystems: {
"system<": true,
}
# create snapshots with prefix `zrepl_` every 15 minutes
snapshotting:
type: periodic
interval: 15m
prefix: zrepl_
pruning:
keep:
# fade-out scheme for snapshots starting with `zrepl_`
# - keep all created in the last hour
# - then destroy snapshots such that we keep 24 each 1 hour apart
# - then destroy snapshots such that we keep 14 each 1 day apart
# - then destroy all older snapshots
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
regex: "^zrepl_.*"
# keep all snapshots that don't have the `zrepl_` prefix
- type: regex
negate: true
regex: "^zrepl_.*"
# This job pushes to the local sink defined in job `backuppool_sink`.
# We trigger replication manually from the command line / udev rules using
# `zrepl signal wakeup push_to_drive`
- type: push
name: push_to_drive
connect:
type: local
listener_name: backuppool_sink
client_identity: myhostname
filesystems: {
"system<": true
}
send:
encrypted: true
replication:
protection:
initial: guarantee_resumability
# Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
# Thus, when we yank out the backup drive during replication
# - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
# - but in exchange we get back the disk space allocated by `to` when we prune it
# - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
incremental: guarantee_incremental
snapshotting:
type: manual
pruning:
# no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
keep_sender:
- type: regex
regex: ".*"
# retain
keep_receiver:
# longer retention on the backup drive, we have more space there
- type: grid
grid: 1x1h(keep=all) | 24x1h | 360x1d
regex: "^zrepl_.*"
# retain all non-zrepl snapshots on the backup drive
- type: regex
negate: true
regex: "^zrepl_.*"
# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
- type: sink
name: backuppool_sink
root_fs: "backuppool/zrepl/sink"
serve:
type: local
listener_name: backuppool_sink
internal/config/samples/quickstart_continuous_server_backup_receiver.yml
+12
View File
@@ -0,0 +1,12 @@
jobs:
- name: sink
type: sink
serve:
type: tls
listen: ":8888"
ca: "/etc/zrepl/prod.crt"
cert: "/etc/zrepl/backups.crt"
key: "/etc/zrepl/backups.key"
client_cns:
- "prod"
root_fs: "storage/zrepl/sink"
internal/config/samples/quickstart_continuous_server_backup_sender.yml
+28
View File
@@ -0,0 +1,28 @@
jobs:
- name: prod_to_backups
type: push
connect:
type: tls
address: "backups.example.com:8888"
ca: /etc/zrepl/backups.crt
cert: /etc/zrepl/prod.crt
key: /etc/zrepl/prod.key
server_cn: "backups"
filesystems: {
"zroot<": true,
"zroot/var/tmp<": false,
"zroot/usr/home/paranoid": false
}
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
regex: "^zrepl_"
internal/config/samples/quickstart_fan_out_replication_source.yml
+59
View File
@@ -0,0 +1,59 @@
jobs:
# Separate job for snapshots and pruning
- name: snapshots
type: snap
filesystems:
'tank<': true # all filesystems
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
pruning:
keep:
# Keep non-zrepl snapshots
- type: regex
negate: true
regex: '^zrepl_'
# Time-based snapshot retention
- type: grid
grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
regex: '^zrepl_'
# Source job for target B
- name: target_b
type: source
serve:
type: tls
listen: :8888
ca: /etc/zrepl/b.example.com.crt
cert: /etc/zrepl/a.example.com.crt
key: /etc/zrepl/a.example.com.key
client_cns:
- b.example.com
filesystems:
'tank<': true # all filesystems
# Snapshots are handled by the separate snap job
snapshotting:
type: manual
# Source job for target C
- name: target_c
type: source
serve:
type: tls
listen: :8889
ca: /etc/zrepl/c.example.com.crt
cert: /etc/zrepl/a.example.com.crt
key: /etc/zrepl/a.example.com.key
client_cns:
- c.example.com
filesystems:
'tank<': true # all filesystems
# Snapshots are handled by the separate snap job
snapshotting:
type: manual
# Source jobs for remaining targets. Each one should listen on a different port
# and reference the correct certificate and client CN.
# - name: target_c
# ...
internal/config/samples/quickstart_fan_out_replication_target.yml
+30
View File
@@ -0,0 +1,30 @@
jobs:
# Pull from source server A
- name: source_a
type: pull
connect:
type: tls
# Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
address: a.example.com:8888
ca: /etc/zrepl/a.example.com.crt
# Use the correct key pair for this specific client
cert: /etc/zrepl/b.example.com.crt
key: /etc/zrepl/b.example.com.key
server_cn: a.example.com
root_fs: pool0/backup
interval: 10m
pruning:
keep_sender:
# Source does the pruning in its snap job
- type: regex
regex: '.*'
# Receiver-side pruning can be configured as desired on each target server
keep_receiver:
# Keep non-zrepl snapshots
- type: regex
negate: true
regex: '^zrepl_'
# Time-based snapshot retention
- type: grid
grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
regex: '^zrepl_'
internal/config/samples/sink.yml
+13
View File
@@ -0,0 +1,13 @@
jobs:
- type: sink
name: "laptop_sink"
root_fs: "pool2/backup_laptops"
serve:
type: tls
listen: "192.168.122.189:8888"
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
client_cns:
- "laptop1"
- "homeserver"
internal/config/samples/snap.yml
+14
View File
@@ -0,0 +1,14 @@
jobs:
- name: snapjob
type: snap
filesystems: {
"tank/frequently_changed<": true,
}
snapshotting:
type: periodic
interval: 2m
prefix: zrepl_snapjob_
pruning:
keep:
- type: last_n
count: 60
internal/config/samples/snap_cron.yml
+14
View File
@@ -0,0 +1,14 @@
jobs:
- name: snapjob
type: snap
filesystems: {
"tank<": true,
}
snapshotting:
type: cron
prefix: zrepl_snapjob_
cron: "*/5 * * * *"
pruning:
keep:
- type: last_n
count: 60
internal/config/samples/source.yml
+21
View File
@@ -0,0 +1,21 @@
jobs:
- name: pull_source
type: source
serve:
type: tcp
listen: "0.0.0.0:8888"
clients: {
"192.168.122.123" : "mysql01",
"192.168.122.42" : "mx01",
"2001:0db8:85a3::8a2e:0370:7334": "gateway",
"10.23.42.0/24": "cluster-*",
"fde4:8dba:82e1::/64": "san-*",
}
filesystems: {
"<": true,
"secret": false
}
snapshotting:
type: periodic
interval: 10m
prefix: zrepl_
internal/config/samples/source_ssh.yml
+17
View File
@@ -0,0 +1,17 @@
jobs:
- name: pull_source
type: source
serve:
type: stdinserver
client_identities:
- "client1"
- "client2"
filesystems: {
"<": true,
"secret": false
}
snapshotting:
type: periodic
interval: 10m
prefix: zrepl_