WIP rewrite the daemon

cmd subdir does not build on purpose, it's only left in tree to grab old
code and move it to github.com/zrepl/zrepl/daemon
This commit is contained in:
Christian Schwarz
2018-08-27 22:21:45 +02:00
parent df6e1bc64d
commit c69ebd3806
55 changed files with 1133 additions and 84 deletions
+397
View File
@@ -0,0 +1,397 @@
package config
import (
"fmt"
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
"io/ioutil"
"os"
"regexp"
"strconv"
"time"
)
type Config struct {
Jobs []JobEnum `yaml:"jobs"`
Global Global `yaml:"global"`
}
type JobEnum struct {
Ret interface{}
}
type PushJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication PushReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type SinkJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication SinkReplication `yaml:"replication"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PullJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication PullReplication `yaml:"replication"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PullReplication struct {
Connect ConnectEnum `yaml:"connect"`
RootDataset string `yaml:"root_dataset"`
Interval time.Duration `yaml:"interval,positive"`
}
type SourceJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication SourceReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningLocal `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type LocalJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication LocalReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PushReplication struct {
Connect ConnectEnum `yaml:"connect"`
Filesystems map[string]bool `yaml:"filesystems"`
}
type SinkReplication struct {
RootDataset string `yaml:"root_dataset"`
Serve ServeEnum `yaml:"serve"`
}
type SourceReplication struct {
Serve ServeEnum `yaml:"serve"`
Filesystems map[string]bool `yaml:"filesystems"`
}
type LocalReplication struct {
Filesystems map[string]bool `yaml:"filesystems"`
RootDataset string `yaml:"root_dataset"`
}
type Snapshotting struct {
SnapshotPrefix string `yaml:"snapshot_prefix"`
Interval time.Duration `yaml:"interval,positive"`
}
type PruningSenderReceiver struct {
KeepSender []PruningEnum `yaml:"keep_sender"`
KeepReceiver []PruningEnum `yaml:"keep_receiver"`
}
type PruningLocal struct {
Keep []PruningEnum `yaml:"keep"`
}
type Global struct {
Logging []LoggingOutletEnum `yaml:"logging"`
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
Control GlobalControl `yaml:"control"`
Serve GlobalServe `yaml:"serve"`
}
type ConnectEnum struct {
Ret interface{}
}
type TCPConnect struct {
Type string `yaml:"type"`
Address string `yaml:"address"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type TLSConnect struct {
Type string `yaml:"type"`
Address string `yaml:"address"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ServerCN string `yaml:"server_cn"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type SSHStdinserverConnect struct {
Type string `yaml:"type"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port uint16 `yaml:"port"`
IdentityFile string `yaml:"identity_file"`
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
Options []string `yaml:"options"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type ServeEnum struct {
Ret interface{}
}
type TCPServe struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
Clients map[string]string `yaml:"clients"`
}
type TLSServe struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ClientCN string `yaml:"client_cn"`
HandshakeTimeout time.Duration `yaml:"handshake_timeout,positive,default=10s"`
}
type StdinserverServer struct {
Type string `yaml:"type"`
ClientIdentity string `yaml:"client_identity"`
}
type PruningEnum struct {
Ret interface{}
}
type PruneKeepNotReplicated struct {
Type string `yaml:"type"`
}
type PruneKeepLastN struct {
Type string `yaml:"type"`
Count int `yaml:"count"`
}
type PruneKeepPrefix struct { // FIXME rename to KeepPrefix
Type string `yaml:"type"`
Prefix string `yaml:"prefix"`
}
type LoggingOutletEnum struct {
Ret interface{}
}
type LoggingOutletCommon struct {
Type string `yaml:"type"`
Level string `yaml:"level"`
Format string `yaml:"format"`
}
type StdoutLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Time bool `yaml:"time,default=true"`
}
type SyslogLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
}
type TCPLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Address string `yaml:"address"`
Net string `yaml:"net,default=tcp"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
TLS *TCPLoggingOutletTLS `yaml:"tls,optional"`
}
type TCPLoggingOutletTLS struct {
CA string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
}
type MonitoringEnum struct {
Ret interface{}
}
type PrometheusMonitoring struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
}
type GlobalControl struct {
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
}
type GlobalServe struct {
StdinServer GlobalStdinServer `yaml:"stdinserver"`
}
type GlobalStdinServer struct {
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
}
type JobDebugSettings struct {
Conn struct {
ReadDump string `yaml:"read_dump"`
WriteDump string `yaml:"write_dump"`
} `yaml:"conn"`
}
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
var in struct {
Type string
}
if err := u(&in, true); err != nil {
return nil, err
}
if in.Type == "" {
return nil, &yaml.TypeError{[]string{"must specify type"}}
}
v, ok := types[in.Type]
if !ok {
return nil, &yaml.TypeError{[]string{fmt.Sprintf("invalid type name %q", in.Type)}}
}
if err := u(v, false); err != nil {
return nil, err
}
return v, nil
}
func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"push": &PushJob{},
"sink": &SinkJob{},
"pull": &PullJob{},
"source": &SourceJob{},
"local": &LocalJob{},
})
return
}
func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPConnect{},
"tls": &TLSConnect{},
"ssh+stdinserver": &SSHStdinserverConnect{},
})
return
}
func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPServe{},
"tls": &TLSServe{},
"stdinserver": &StdinserverServer{},
})
return
}
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"not_replicated": &PruneKeepNotReplicated{},
"last_n": &PruneKeepLastN{},
"grid": &PruneGrid{},
"prefix": &PruneKeepPrefix{},
})
return
}
func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"stdout": &StdoutLoggingOutlet{},
"syslog": &SyslogLoggingOutlet{},
"tcp": &TCPLoggingOutlet{},
})
return
}
func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"prometheus": &PrometheusMonitoring{},
})
return
}
var ConfigFileDefaultLocations = []string{
"/etc/zrepl/zrepl.yml",
"/usr/local/etc/zrepl/zrepl.yml",
}
func ParseConfig(path string) (i Config, err error) {
if path == "" {
// Try default locations
for _, l := range ConfigFileDefaultLocations {
stat, statErr := os.Stat(l)
if statErr != nil {
continue
}
if !stat.Mode().IsRegular() {
err = errors.Errorf("file at default location is not a regular file: %s", l)
return
}
path = l
break
}
}
var bytes []byte
if bytes, err = ioutil.ReadFile(path); err != nil {
return
}
if err = yaml.UnmarshalStrict(bytes, &i); err != nil {
return
}
return
}
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
func parsePostitiveDuration(e string) (d time.Duration, err error) {
comps := durationStringRegex.FindStringSubmatch(e)
if len(comps) != 3 {
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
return
}
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
if err != nil {
return 0, err
}
if durationFactor <= 0 {
return 0, errors.New("duration must be positive integer")
}
var durationUnit time.Duration
switch comps[2] {
case "s":
durationUnit = time.Second
case "m":
durationUnit = time.Minute
case "h":
durationUnit = time.Hour
case "d":
durationUnit = 24 * time.Hour
case "w":
durationUnit = 24 * 7 * time.Hour
default:
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
return
}
d = time.Duration(durationFactor) * durationUnit
return
}
+29
View File
@@ -0,0 +1,29 @@
package config
import (
"github.com/kr/pretty"
"path/filepath"
"testing"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
paths, err := filepath.Glob("./samples/*")
if err != nil {
t.Errorf("glob failed: %+v", err)
}
for _, p := range paths {
t.Run(p, func(t *testing.T) {
c, err := ParseConfig(p)
if err != nil {
t.Errorf("error parsing %s:\n%+v", p, err)
}
t.Logf("file: %s", p)
t.Log(pretty.Sprint(c))
})
}
}
+123
View File
@@ -0,0 +1,123 @@
package config
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
type RetentionIntervalList []RetentionInterval
type PruneGrid struct {
Type string `yaml:"type"`
Grid RetentionIntervalList `yaml:"grid"`
KeepBookmarks string `yaml:"keep_bookmarks"`
}
type RetentionInterval struct {
length time.Duration
keepCount int
}
func (i *RetentionInterval) Length() time.Duration {
return i.length
}
func (i *RetentionInterval) KeepCount() int {
return i.keepCount
}
const RetentionGridKeepCountAll int = -1
type RetentionGrid struct {
intervals []RetentionInterval
}
func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var in string
if err := u(&in, true); err != nil {
return err
}
intervals, err := parseRetentionGridIntervalsString(in)
if err != nil {
return err
}
*t = intervals
return nil
}
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
if comps == nil {
err = fmt.Errorf("retention string does not match expected format")
return
}
times, err := strconv.Atoi(comps[1])
if err != nil {
return nil, err
} else if times <= 0 {
return nil, fmt.Errorf("contains factor <= 0")
}
duration, err := parsePostitiveDuration(comps[2])
if err != nil {
return nil, err
}
keepCount := 1
if comps[3] != "" {
// Decompose key=value, comma separated
// For now, only keep_count is supported
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
res := re.FindStringSubmatch(comps[4])
if res == nil || len(res) != 2 {
err = fmt.Errorf("interval parameter contains unknown parameters")
return
}
if res[1] == "all" {
keepCount = RetentionGridKeepCountAll
} else {
keepCount, err = strconv.Atoi(res[1])
if err != nil {
err = fmt.Errorf("cannot parse keep_count value")
return
}
}
}
intervals = make([]RetentionInterval, times)
for i := range intervals {
intervals[i] = RetentionInterval{
length: duration,
keepCount: keepCount,
}
}
return
}
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
ges := strings.Split(s, "|")
intervals = make([]RetentionInterval, 0, 7*len(ges))
for intervalIdx, e := range ges {
parsed, err := parseRetentionGridIntervalString(e)
if err != nil {
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
}
intervals = append(intervals, parsed...)
}
return
}
+42
View File
@@ -0,0 +1,42 @@
jobs:
- name: mirror_local
type: local
# snapshot the filesystems matched by the left-hand-side of the mapping
# every 10m with zrepl_ as prefix
replication:
filesystems: {
"pool1/var/db<": true,
"pool1/usr/home<": true,
"pool1/usr/home/paranoid": false, #don't backup paranoid user
"pool1/poudriere/ports<": false #don't backup the ports trees
}
# TODO FIXME enforce that the tree under root_dataset and the trees allowed (true) by filesystems are non-overlapping
root_dataset: "pool2/backups/pool1"
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+40
View File
@@ -0,0 +1,40 @@
jobs:
- name: pull_servers
type: pull
replication:
connect:
type: tls
address: "server1.foo.bar:8888"
ca: "/certs/ca.crt"
cert: "/certs/cert.crt"
key: "/certs/key.pem"
server_cn: "server1"
root_dataset: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+44
View File
@@ -0,0 +1,44 @@
jobs:
- name: pull_servers
type: pull
replication:
connect:
type: ssh+stdinserver
host: app-srv.example.com
user: root
port: 22
identity_file: /etc/zrepl/ssh/identity
options: # optional, default [], `-o` arguments passed to ssh
- "Compression=on"
root_dataset: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: prefix
prefix: keep_
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+41
View File
@@ -0,0 +1,41 @@
jobs:
- type: push
name: "push"
replication:
connect:
type: tcp
address: "backup-server.foo.bar:8888"
filesystems: {
"<": true,
"tmp": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+28
View File
@@ -0,0 +1,28 @@
jobs:
- type: sink
name: "laptop_sink"
replication:
root_dataset: "pool2/backup_laptops"
serve:
type: tls
listen: "192.168.122.189:8888"
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
client_cn: "laptop1"
global:
logging:
- type: "tcp"
address: "123.123.123.123:1234"
retry_interval: 10s
tls:
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
level: "warn"
format: "human"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+40
View File
@@ -0,0 +1,40 @@
jobs:
- name: pull_source
type: source
replication:
serve:
type: tcp
listen: "0.0.0.0:8888"
clients: {
"192.168.122.123" : "client1"
}
filesystems: {
"<": true,
"secret": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
+37
View File
@@ -0,0 +1,37 @@
jobs:
- name: pull_source
type: source
replication:
serve:
type: stdinserver
client_identity: "client1"
filesystems: {
"<": true,
"secret": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver