adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP rewrite the daemon

Browse Source 
cmd subdir does not build on purpose, it's only left in tree to grab old
code and move it to github.com/zrepl/zrepl/daemon
This commit is contained in:
Christian Schwarz
2018-08-27 22:21:45 +02:00
parent df6e1bc64d
commit c69ebd3806
55 changed files with 1133 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/config/config.go
-396
View File
@@ -1,396 +0,0 @@
package config
import (
"fmt"
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
"io/ioutil"
"os"
"regexp"
"strconv"
"time"
)
type Config struct {
Jobs []JobEnum `yaml:"jobs"`
Global Global `yaml:"global"`
}
type JobEnum struct {
Ret interface{}
}
type PushJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication PushReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type SinkJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication SinkReplication `yaml:"replication"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PullJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication PullReplication `yaml:"replication"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PullReplication struct {
Connect ConnectEnum `yaml:"connect"`
RootDataset string `yaml:"root_dataset"`
Interval time.Duration `yaml:"interval,positive"`
}
type SourceJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication SourceReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningLocal `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type LocalJob struct {
Type string `yaml:"type"`
Name string `yaml:"name"`
Replication LocalReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
}
type PushReplication struct {
Connect ConnectEnum `yaml:"connect"`
Filesystems map[string]bool `yaml:"filesystems"`
}
type SinkReplication struct {
RootDataset string `yaml:"root_dataset"`
Serve ServeEnum `yaml:"serve"`
}
type SourceReplication struct {
Serve ServeEnum `yaml:"serve"`
Filesystems map[string]bool `yaml:"filesystems"`
}
type LocalReplication struct {
Filesystems map[string]bool `yaml:"filesystems"`
RootDataset string `yaml:"root_dataset"`
}
type Snapshotting struct {
SnapshotPrefix string `yaml:"snapshot_prefix"`
Interval time.Duration `yaml:"interval,positive"`
}
type PruningSenderReceiver struct {
KeepSender []PruningEnum `yaml:"keep_sender"`
KeepReceiver []PruningEnum `yaml:"keep_receiver"`
}
type PruningLocal struct {
Keep []PruningEnum `yaml:"keep"`
}
type Global struct {
Logging []LoggingOutletEnum `yaml:"logging"`
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
Control GlobalControl `yaml:"control"`
Serve GlobalServe `yaml:"serve"`
}
type ConnectEnum struct {
Ret interface{}
}
type TCPConnect struct {
Type string `yaml:"type"`
Address string `yaml:"address"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type TLSConnect struct {
Type string `yaml:"type"`
Address string `yaml:"address"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ServerCN string `yaml:"server_cn"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type SSHStdinserverConnect struct {
Type string `yaml:"type"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port uint16 `yaml:"port"`
IdentityFile string `yaml:"identity_file"`
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
Options []string `yaml:"options"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
}
type ServeEnum struct {
Ret interface{}
}
type TCPServe struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
Clients map[string]string `yaml:"clients"`
}
type TLSServe struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
Ca string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ClientCN string `yaml:"client_cn"`
}
type StdinserverServer struct {
Type string `yaml:"type"`
ClientIdentity string `yaml:"client_identity"`
}
type PruningEnum struct {
Ret interface{}
}
type PruneKeepNotReplicated struct {
Type string `yaml:"type"`
}
type PruneKeepLastN struct {
Type string `yaml:"type"`
Count int `yaml:"count"`
}
type PruneKeepPrefix struct { // FIXME rename to KeepPrefix
Type string `yaml:"type"`
Prefix string `yaml:"prefix"`
}
type LoggingOutletEnum struct {
Ret interface{}
}
type LoggingOutletCommon struct {
Type string `yaml:"type"`
Level string `yaml:"level"`
Format string `yaml:"format"`
}
type StdoutLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Time bool `yaml:"time"`
}
type SyslogLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
RetryInterval time.Duration `yaml:"retry_interval,positive"`
}
type TCPLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Address string `yaml:"address"`
Net string `yaml:"net,default=tcp"`
RetryInterval time.Duration `yaml:"retry_interval,positive"`
TLS *TCPLoggingOutletTLS `yaml:"tls,optional"`
}
type TCPLoggingOutletTLS struct {
CA string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
}
type MonitoringEnum struct {
Ret interface{}
}
type PrometheusMonitoring struct {
Type string `yaml:"type"`
Listen string `yaml:"listen"`
}
type GlobalControl struct {
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
}
type GlobalServe struct {
StdinServer GlobalStdinServer `yaml:"stdinserver"`
}
type GlobalStdinServer struct {
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
}
type JobDebugSettings struct {
Conn struct {
ReadDump string `yaml:"read_dump"`
WriteDump string `yaml:"write_dump"`
} `yaml:"conn"`
}
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
var in struct {
Type string
}
if err := u(&in, true); err != nil {
return nil, err
}
if in.Type == "" {
return nil, &yaml.TypeError{[]string{"must specify type"}}
}
v, ok := types[in.Type]
if !ok {
return nil, &yaml.TypeError{[]string{fmt.Sprintf("invalid type name %q", in.Type)}}
}
if err := u(v, false); err != nil {
return nil, err
}
return v, nil
}
func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"push": &PushJob{},
"sink": &SinkJob{},
"pull": &PullJob{},
"source": &SourceJob{},
"local": &LocalJob{},
})
return
}
func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPConnect{},
"tls": &TLSConnect{},
"ssh+stdinserver": &SSHStdinserverConnect{},
})
return
}
func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"tcp": &TCPServe{},
"tls": &TLSServe{},
"stdinserver": &StdinserverServer{},
})
return
}
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"not_replicated": &PruneKeepNotReplicated{},
"last_n": &PruneKeepLastN{},
"grid": &PruneGrid{},
"prefix": &PruneKeepPrefix{},
})
return
}
func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"stdout": &StdoutLoggingOutlet{},
"syslog": &SyslogLoggingOutlet{},
"tcp": &TCPLoggingOutlet{},
})
return
}
func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"prometheus": &PrometheusMonitoring{},
})
return
}
var ConfigFileDefaultLocations = []string{
"/etc/zrepl/zrepl.yml",
"/usr/local/etc/zrepl/zrepl.yml",
}
func ParseConfig(path string) (i Config, err error) {
if path == "" {
// Try default locations
for _, l := range ConfigFileDefaultLocations {
stat, statErr := os.Stat(l)
if statErr != nil {
continue
}
if !stat.Mode().IsRegular() {
err = errors.Errorf("file at default location is not a regular file: %s", l)
return
}
path = l
break
}
}
var bytes []byte
if bytes, err = ioutil.ReadFile(path); err != nil {
return
}
if err = yaml.UnmarshalStrict(bytes, &i); err != nil {
return
}
return
}
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
func parsePostitiveDuration(e string) (d time.Duration, err error) {
comps := durationStringRegex.FindStringSubmatch(e)
if len(comps) != 3 {
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
return
}
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
if err != nil {
return 0, err
}
if durationFactor <= 0 {
return 0, errors.New("duration must be positive integer")
}
var durationUnit time.Duration
switch comps[2] {
case "s":
durationUnit = time.Second
case "m":
durationUnit = time.Minute
case "h":
durationUnit = time.Hour
case "d":
durationUnit = 24 * time.Hour
case "w":
durationUnit = 24 * 7 * time.Hour
default:
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
return
}
d = time.Duration(durationFactor) * durationUnit
return
}
cmd/config/config_test.go
-29
View File
@@ -1,29 +0,0 @@
package config
import (
"github.com/kr/pretty"
"path/filepath"
"testing"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
paths, err := filepath.Glob("./samples/*")
if err != nil {
t.Errorf("glob failed: %+v", err)
}
for _, p := range paths {
t.Run(p, func(t *testing.T) {
c, err := ParseConfig(p)
if err != nil {
t.Errorf("error parsing %s:\n%+v", p, err)
}
t.Logf("file: %s", p)
t.Log(pretty.Sprint(c))
})
}
}
cmd/config/retentiongrid.go
-123
View File
@@ -1,123 +0,0 @@
package config
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
type RetentionIntervalList []RetentionInterval
type PruneGrid struct {
Type string `yaml:"type"`
Grid RetentionIntervalList `yaml:"grid"`
KeepBookmarks string `yaml:"keep_bookmarks"`
}
type RetentionInterval struct {
length time.Duration
keepCount int
}
func (i *RetentionInterval) Length() time.Duration {
return i.length
}
func (i *RetentionInterval) KeepCount() int {
return i.keepCount
}
const RetentionGridKeepCountAll int = -1
type RetentionGrid struct {
intervals []RetentionInterval
}
func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var in string
if err := u(&in, true); err != nil {
return err
}
intervals, err := parseRetentionGridIntervalsString(in)
if err != nil {
return err
}
*t = intervals
return nil
}
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
if comps == nil {
err = fmt.Errorf("retention string does not match expected format")
return
}
times, err := strconv.Atoi(comps[1])
if err != nil {
return nil, err
} else if times <= 0 {
return nil, fmt.Errorf("contains factor <= 0")
}
duration, err := parsePostitiveDuration(comps[2])
if err != nil {
return nil, err
}
keepCount := 1
if comps[3] != "" {
// Decompose key=value, comma separated
// For now, only keep_count is supported
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
res := re.FindStringSubmatch(comps[4])
if res == nil || len(res) != 2 {
err = fmt.Errorf("interval parameter contains unknown parameters")
return
}
if res[1] == "all" {
keepCount = RetentionGridKeepCountAll
} else {
keepCount, err = strconv.Atoi(res[1])
if err != nil {
err = fmt.Errorf("cannot parse keep_count value")
return
}
}
}
intervals = make([]RetentionInterval, times)
for i := range intervals {
intervals[i] = RetentionInterval{
length: duration,
keepCount: keepCount,
}
}
return
}
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
ges := strings.Split(s, "|")
intervals = make([]RetentionInterval, 0, 7*len(ges))
for intervalIdx, e := range ges {
parsed, err := parseRetentionGridIntervalString(e)
if err != nil {
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
}
intervals = append(intervals, parsed...)
}
return
}
cmd/config/samples/local.yml
-42
View File
@@ -1,42 +0,0 @@
jobs:
- name: mirror_local
type: local
# snapshot the filesystems matched by the left-hand-side of the mapping
# every 10m with zrepl_ as prefix
replication:
filesystems: {
"pool1/var/db<": true,
"pool1/usr/home<": true,
"pool1/usr/home/paranoid": false, #don't backup paranoid user
"pool1/poudriere/ports<": false #don't backup the ports trees
}
# TODO FIXME enforce that the tree under root_dataset and the trees allowed (true) by filesystems are non-overlapping
root_dataset: "pool2/backups/pool1"
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/pull.yml
-40
View File
@@ -1,40 +0,0 @@
jobs:
- name: pull_servers
type: pull
replication:
connect:
type: tls
address: "server1.foo.bar:8888"
ca: "/certs/ca.crt"
cert: "/certs/cert.crt"
key: "/certs/key.pem"
server_cn: "server1"
root_dataset: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/pull_ssh.yml
-44
View File
@@ -1,44 +0,0 @@
jobs:
- name: pull_servers
type: pull
replication:
connect:
type: ssh+stdinserver
host: app-srv.example.com
user: root
port: 22
identity_file: /etc/zrepl/ssh/identity
options: # optional, default [], `-o` arguments passed to ssh
- "Compression=on"
root_dataset: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: prefix
prefix: keep_
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/push.yml
-41
View File
@@ -1,41 +0,0 @@
jobs:
- type: push
name: "push"
replication:
connect:
type: tcp
address: "backup-server.foo.bar:8888"
filesystems: {
"<": true,
"tmp": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep_sender:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
keep_receiver:
- type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/sink.yml
-28
View File
@@ -1,28 +0,0 @@
jobs:
- type: sink
name: "laptop_sink"
replication:
root_dataset: "pool2/backup_laptops"
serve:
type: tls
listen: "192.168.122.189:8888"
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
client_cn: "laptop1"
global:
logging:
- type: "tcp"
address: "123.123.123.123:1234"
retry_interval: 10s
tls:
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
level: "warn"
format: "human"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/source.yml
-40
View File
@@ -1,40 +0,0 @@
jobs:
- name: pull_source
type: source
replication:
serve:
type: tcp
listen: "0.0.0.0:8888"
clients: {
"192.168.122.123" : "client1"
}
filesystems: {
"<": true,
"secret": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config/samples/source_ssh.yml
-37
View File
@@ -1,37 +0,0 @@
jobs:
- name: pull_source
type: source
replication:
serve:
type: stdinserver
client_identity: "client1"
filesystems: {
"<": true,
"secret": false
}
snapshotting:
snapshot_prefix: zrepl_
interval: 10m
pruning:
keep:
- type: not_replicated
- type: last_n
count: 10
- type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_bookmarks: all
global:
logging:
- type: "stdout"
time: true
level: "warn"
format: "human"
monitoring:
- type: "prometheus"
listen: ":9091"
control:
sockpath: /var/run/zrepl/control
serve:
stdinserver:
sockdir: /var/run/zrepl/stdinserver
cmd/config_connect.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"github.com/pkg/errors"
"github.com/problame/go-netssh"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/tlsconf"
"time"
)
cmd/config_job_local.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"context"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/zfs"
"sync"
cmd/config_job_pull.go
+2 -2
View File
@@ -11,8 +11,8 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/replication"
)
cmd/config_job_source.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/endpoint"
"net"
)
cmd/config_logging.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"crypto/x509"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/tlsconf"
"github.com/zrepl/zrepl/logger"
"os"
cmd/config_mapfilter.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/zfs"
)
cmd/config_parse.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/pruning/retentiongrid"
"os"
)
cmd/config_serve_stdinserver.go
+1 -1
View File
@@ -2,7 +2,7 @@ package cmd
import (
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/helpers"
"net"
"path"
cmd/config_serve_tcp.go
+2 -3
View File
@@ -1,10 +1,9 @@
package cmd
import (
"net"
"github.com/zrepl/zrepl/config"
"time"
"github.com/zrepl/zrepl/cmd/config"
"net"
)
type TCPListenerFactory struct {
cmd/config_serve_tls.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/tlsconf"
)
cmd/daemon/control.go → cmd/daemon.deact/control.go
View File
cmd/daemon/daemon.go → cmd/daemon.deact/daemon.go
+13
View File
@@ -1,5 +1,18 @@
package daemon
import (
"context"
"github.com/zrepl/zrepl/logger"
"os"
"os/signal"
"syscall"
"time"
"github.com/zrepl/zrepl/version"
"fmt"
)
.daesdfadsfsafjlsjfda
import (
"context"
"fmt"
cmd/daemon/job/job.go → cmd/daemon.deact/job/job.go
View File
cmd/daemon/pprof.go → cmd/daemon.deact/pprof.go
View File
cmd/daemon/prometheus.go → cmd/daemon.deact/prometheus.go
View File
cmd/daemon.go
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"context"
"fmt"
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/cmd/daemon"
"github.com/zrepl/zrepl/cmd/daemon/job"
"github.com/zrepl/zrepl/logger"
cmd/endpoint/context.go
-26
View File
@@ -1,26 +0,0 @@
package endpoint
import (
"context"
"github.com/zrepl/zrepl/logger"
)
type contextKey int
const (
contextKeyLogger contextKey = iota
)
type Logger = logger.Logger
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLogger, log)
}
func getLogger(ctx context.Context) Logger {
l, ok := ctx.Value(contextKeyLogger).(Logger)
if !ok {
l = logger.NewNullLogger()
}
return l
}
cmd/endpoint/endpoint.go
-417
View File
@@ -1,417 +0,0 @@
// Package endpoint implements replication endpoints for use with package replication.
package endpoint
import (
"bytes"
"context"
"github.com/golang/protobuf/proto"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/replication/pdu"
"github.com/zrepl/zrepl/zfs"
"io"
)
// Sender implements replication.ReplicationEndpoint for a sending side
type Sender struct {
FSFilter zfs.DatasetFilter
FilesystemVersionFilter zfs.FilesystemVersionFilter
}
func NewSender(fsf zfs.DatasetFilter, fsvf zfs.FilesystemVersionFilter) *Sender {
return &Sender{fsf, fsvf}
}
func (p *Sender) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
fss, err := zfs.ZFSListMapping(p.FSFilter)
if err != nil {
return nil, err
}
rfss := make([]*pdu.Filesystem, len(fss))
for i := range fss {
rfss[i] = &pdu.Filesystem{
Path: fss[i].ToString(),
// FIXME: not supporting ResumeToken yet
}
}
return rfss, nil
}
func (p *Sender) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
dp, err := zfs.NewDatasetPath(fs)
if err != nil {
return nil, err
}
pass, err := p.FSFilter.Filter(dp)
if err != nil {
return nil, err
}
if !pass {
return nil, replication.NewFilteredError(fs)
}
fsvs, err := zfs.ZFSListFilesystemVersions(dp, p.FilesystemVersionFilter)
if err != nil {
return nil, err
}
rfsvs := make([]*pdu.FilesystemVersion, len(fsvs))
for i := range fsvs {
rfsvs[i] = pdu.FilesystemVersionFromZFS(fsvs[i])
}
return rfsvs, nil
}
func (p *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
dp, err := zfs.NewDatasetPath(r.Filesystem)
if err != nil {
return nil, nil, err
}
pass, err := p.FSFilter.Filter(dp)
if err != nil {
return nil, nil, err
}
if !pass {
return nil, nil, replication.NewFilteredError(r.Filesystem)
}
stream, err := zfs.ZFSSend(r.Filesystem, r.From, r.To)
if err != nil {
return nil, nil, err
}
return &pdu.SendRes{}, stream, nil
}
type FSFilter interface {
Filter(path *zfs.DatasetPath) (pass bool, err error)
}
// FIXME: can we get away without error types here?
type FSMap interface {
FSFilter
Map(path *zfs.DatasetPath) (*zfs.DatasetPath, error)
Invert() (FSMap, error)
AsFilter() FSFilter
}
// Receiver implements replication.ReplicationEndpoint for a receiving side
type Receiver struct {
fsmapInv FSMap
fsmap FSMap
fsvf zfs.FilesystemVersionFilter
}
func NewReceiver(fsmap FSMap, fsvf zfs.FilesystemVersionFilter) (*Receiver, error) {
fsmapInv, err := fsmap.Invert()
if err != nil {
return nil, err
}
return &Receiver{fsmapInv, fsmap, fsvf}, nil
}
func (e *Receiver) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
filtered, err := zfs.ZFSListMapping(e.fsmapInv.AsFilter())
if err != nil {
return nil, errors.Wrap(err, "error checking client permission")
}
fss := make([]*pdu.Filesystem, len(filtered))
for i, a := range filtered {
mapped, err := e.fsmapInv.Map(a)
if err != nil {
return nil, err
}
fss[i] = &pdu.Filesystem{Path: mapped.ToString()}
}
return fss, nil
}
func (e *Receiver) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
p, err := zfs.NewDatasetPath(fs)
if err != nil {
return nil, err
}
lp, err := e.fsmap.Map(p)
if err != nil {
return nil, err
}
if lp == nil {
return nil, errors.New("access to filesystem denied")
}
fsvs, err := zfs.ZFSListFilesystemVersions(lp, e.fsvf)
if err != nil {
return nil, err
}
rfsvs := make([]*pdu.FilesystemVersion, len(fsvs))
for i := range fsvs {
rfsvs[i] = pdu.FilesystemVersionFromZFS(fsvs[i])
}
return rfsvs, nil
}
func (e *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, sendStream io.ReadCloser) error {
defer sendStream.Close()
p, err := zfs.NewDatasetPath(req.Filesystem)
if err != nil {
return err
}
lp, err := e.fsmap.Map(p)
if err != nil {
return err
}
if lp == nil {
return errors.New("receive to filesystem denied")
}
// create placeholder parent filesystems as appropriate
var visitErr error
f := zfs.NewDatasetPathForest()
f.Add(lp)
getLogger(ctx).Debug("begin tree-walk")
f.WalkTopDown(func(v zfs.DatasetPathVisit) (visitChildTree bool) {
if v.Path.Equal(lp) {
return false
}
_, err := zfs.ZFSGet(v.Path, []string{zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME})
if err != nil {
// interpret this as an early exit of the zfs binary due to the fs not existing
if err := zfs.ZFSCreatePlaceholderFilesystem(v.Path); err != nil {
getLogger(ctx).
WithError(err).
WithField("placeholder_fs", v.Path).
Error("cannot create placeholder filesystem")
visitErr = err
return false
}
}
getLogger(ctx).WithField("filesystem", v.Path.ToString()).Debug("exists")
return true // leave this fs as is
})
getLogger(ctx).WithField("visitErr", visitErr).Debug("complete tree-walk")
if visitErr != nil {
return visitErr
}
needForceRecv := false
props, err := zfs.ZFSGet(lp, []string{zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME})
if err == nil {
if isPlaceholder, _ := zfs.IsPlaceholder(lp, props.Get(zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME)); isPlaceholder {
needForceRecv = true
}
}
args := make([]string, 0, 1)
if needForceRecv {
args = append(args, "-F")
}
getLogger(ctx).Debug("start receive command")
if err := zfs.ZFSRecv(lp.ToString(), sendStream, args...); err != nil {
return err
}
return nil
}
// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
// RPC STUBS
// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
const (
RPCListFilesystems = "ListFilesystems"
RPCListFilesystemVersions = "ListFilesystemVersions"
RPCReceive = "Receive"
RPCSend = "Send"
)
// Remote implements an endpoint stub that uses streamrpc as a transport.
type Remote struct {
c *streamrpc.Client
}
func NewRemote(c *streamrpc.Client) Remote {
return Remote{c}
}
func (s Remote) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) {
req := pdu.ListFilesystemReq{}
b, err := proto.Marshal(&req)
if err != nil {
return nil, err
}
rb, rs, err := s.c.RequestReply(ctx, RPCListFilesystems, bytes.NewBuffer(b), nil)
if err != nil {
return nil, err
}
if rs != nil {
rs.Close()
return nil, errors.New("response contains unexpected stream")
}
var res pdu.ListFilesystemRes
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
return nil, err
}
return res.Filesystems, nil
}
func (s Remote) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) {
req := pdu.ListFilesystemVersionsReq{
Filesystem: fs,
}
b, err := proto.Marshal(&req)
if err != nil {
return nil, err
}
rb, rs, err := s.c.RequestReply(ctx, RPCListFilesystemVersions, bytes.NewBuffer(b), nil)
if err != nil {
return nil, err
}
if rs != nil {
rs.Close()
return nil, errors.New("response contains unexpected stream")
}
var res pdu.ListFilesystemVersionsRes
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
return nil, err
}
return res.Versions, nil
}
func (s Remote) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
b, err := proto.Marshal(r)
if err != nil {
return nil, nil, err
}
rb, rs, err := s.c.RequestReply(ctx, RPCSend, bytes.NewBuffer(b), nil)
if err != nil {
return nil, nil, err
}
if rs == nil {
return nil, nil, errors.New("response does not contain a stream")
}
var res pdu.SendRes
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
rs.Close()
return nil, nil, err
}
return &res, rs, nil
}
func (s Remote) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error {
defer sendStream.Close()
b, err := proto.Marshal(r)
if err != nil {
return err
}
rb, rs, err := s.c.RequestReply(ctx, RPCReceive, bytes.NewBuffer(b), sendStream)
if err != nil {
return err
}
if rs != nil {
rs.Close()
return errors.New("response contains unexpected stream")
}
var res pdu.ReceiveRes
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
return err
}
return nil
}
// Handler implements the server-side streamrpc.HandlerFunc for a Remote endpoint stub.
type Handler struct {
ep replication.Endpoint
}
func NewHandler(ep replication.Endpoint) Handler {
return Handler{ep}
}
func (a *Handler) Handle(ctx context.Context, endpoint string, reqStructured *bytes.Buffer, reqStream io.ReadCloser) (resStructured *bytes.Buffer, resStream io.ReadCloser, err error) {
switch endpoint {
case RPCListFilesystems:
var req pdu.ListFilesystemReq
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
return nil, nil, err
}
fsses, err := a.ep.ListFilesystems(ctx)
if err != nil {
return nil, nil, err
}
res := &pdu.ListFilesystemRes{
Filesystems: fsses,
}
b, err := proto.Marshal(res)
if err != nil {
return nil, nil, err
}
return bytes.NewBuffer(b), nil, nil
case RPCListFilesystemVersions:
var req pdu.ListFilesystemVersionsReq
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
return nil, nil, err
}
fsvs, err := a.ep.ListFilesystemVersions(ctx, req.Filesystem)
if err != nil {
return nil, nil, err
}
res := &pdu.ListFilesystemVersionsRes{
Versions: fsvs,
}
b, err := proto.Marshal(res)
if err != nil {
return nil, nil, err
}
return bytes.NewBuffer(b), nil, nil
case RPCSend:
sender, ok := a.ep.(replication.Sender)
if !ok {
goto Err
}
var req pdu.SendReq
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
return nil, nil, err
}
res, sendStream, err := sender.Send(ctx, &req)
if err != nil {
return nil, nil, err
}
b, err := proto.Marshal(res)
if err != nil {
return nil, nil, err
}
return bytes.NewBuffer(b), sendStream, err
case RPCReceive:
receiver, ok := a.ep.(replication.Receiver)
if !ok {
goto Err
}
var req pdu.ReceiveReq
if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil {
return nil, nil, err
}
err := receiver.Receive(ctx, &req, reqStream)
if err != nil {
return nil, nil, err
}
b, err := proto.Marshal(&pdu.ReceiveRes{})
if err != nil {
return nil, nil, err
}
return bytes.NewBuffer(b), nil, err
}
Err:
return nil, nil, errors.New("no handler for given endpoint")
}
cmd/helpers/helpers.go
-49
View File
@@ -1,49 +0,0 @@
package helpers
import (
"github.com/pkg/errors"
"net"
"os"
"path/filepath"
)
func PreparePrivateSockpath(sockpath string) error {
sockdir := filepath.Dir(sockpath)
sdstat, err := os.Stat(sockdir)
if err != nil {
return errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
}
if !sdstat.IsDir() {
return errors.Errorf("not a directory: %s", sockdir)
}
p := sdstat.Mode().Perm()
if p&0007 != 0 {
return errors.Errorf("socket directory must not be world-accessible: %s (permissions are %#o)", sockdir, p)
}
// Maybe things have not been cleaned up before
s, err := os.Stat(sockpath)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return errors.Wrapf(err, "cannot stat(2) '%s'", sockpath)
}
if s.Mode()&os.ModeSocket == 0 {
return errors.Errorf("unexpected file type at path '%s'", sockpath)
}
err = os.Remove(sockpath)
if err != nil {
return errors.Wrapf(err, "cannot remove presumably stale socket '%s'", sockpath)
}
return nil
}
func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
if err := PreparePrivateSockpath(sockaddr.Name); err != nil {
return nil, err
}
return net.ListenUnix("unix", sockaddr)
}
cmd/pruning/retentiongrid/config_prune_grid.go
+1 -1
View File
@@ -2,7 +2,7 @@ package retentiongrid
import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/zfs"
"math"
"sort"
cmd/tlsconf/tlsconf.go
-121
View File
@@ -1,121 +0,0 @@
package tlsconf
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"net"
"time"
)
func ParseCAFile(certfile string) (*x509.CertPool, error) {
pool := x509.NewCertPool()
pem, err := ioutil.ReadFile(certfile)
if err != nil {
return nil, err
}
if !pool.AppendCertsFromPEM(pem) {
return nil, errors.New("PEM parsing error")
}
return pool, nil
}
type ClientAuthListener struct {
l net.Listener
clientCommonName string
handshakeTimeout time.Duration
}
func NewClientAuthListener(
l net.Listener, ca *x509.CertPool, serverCert tls.Certificate,
clientCommonName string, handshakeTimeout time.Duration) *ClientAuthListener {
if ca == nil {
panic(ca)
}
if serverCert.Certificate == nil || serverCert.PrivateKey == nil {
panic(serverCert)
}
if clientCommonName == "" {
panic(clientCommonName)
}
tlsConf := tls.Config{
Certificates: []tls.Certificate{serverCert},
ClientCAs: ca,
ClientAuth: tls.RequireAndVerifyClientCert,
PreferServerCipherSuites: true,
}
l = tls.NewListener(l, &tlsConf)
return &ClientAuthListener{
l,
clientCommonName,
handshakeTimeout,
}
}
func (l *ClientAuthListener) Accept() (c net.Conn, err error) {
c, err = l.l.Accept()
if err != nil {
return nil, err
}
tlsConn, ok := c.(*tls.Conn)
if !ok {
return c, err
}
var (
cn string
peerCerts []*x509.Certificate
)
if err = tlsConn.SetDeadline(time.Now().Add(l.handshakeTimeout)); err != nil {
goto CloseAndErr
}
if err = tlsConn.Handshake(); err != nil {
goto CloseAndErr
}
peerCerts = tlsConn.ConnectionState().PeerCertificates
if len(peerCerts) != 1 {
err = errors.New("unexpected number of certificates presented by TLS client")
goto CloseAndErr
}
cn = peerCerts[0].Subject.CommonName
if cn != l.clientCommonName {
err = fmt.Errorf("client cert common name does not match client_identity: %q != %q", cn, l.clientCommonName)
goto CloseAndErr
}
return c, nil
CloseAndErr:
c.Close()
return nil, err
}
func (l *ClientAuthListener) Addr() net.Addr {
return l.l.Addr()
}
func (l *ClientAuthListener) Close() error {
return l.l.Close()
}
func ClientAuthClient(serverName string, rootCA *x509.CertPool, clientCert tls.Certificate) (*tls.Config, error) {
if serverName == "" {
panic(serverName)
}
if rootCA == nil {
panic(rootCA)
}
if clientCert.Certificate == nil || clientCert.PrivateKey == nil {
panic(clientCert)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{clientCert},
RootCAs: rootCA,
ServerName: serverName,
}
tlsConfig.BuildNameToCertificate()
return tlsConfig, nil
}