From f3689563b59d44e1a43ac198d899e38a90068ef5 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 11 Sep 2017 13:43:18 +0200 Subject: [PATCH] config: restructure in 'jobs' and 'global' section --- cmd/config.go | 11 +- cmd/config_parse.go | 111 +++++++++++-------- cmd/sampleconf/localbackup/host1.yml | 4 +- cmd/sampleconf/pullbackup/backuphost.yml | 6 +- cmd/sampleconf/pullbackup/productionhost.yml | 30 +++-- cmd/sampleconf/pushbackup/backuphost.yml | 3 +- cmd/sampleconf/pushbackup/productionhost.yml | 3 +- 7 files changed, 108 insertions(+), 60 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index bf0f918..1684623 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -8,7 +8,16 @@ import ( ) type Config struct { - Jobs map[string]Job + Global Global + Jobs map[string]Job +} + +type Global struct { + Serve struct { + Stdinserver struct { + SockDir string + } + } } type RPCConnecter interface { diff --git a/cmd/config_parse.go b/cmd/config_parse.go index 3c11c02..0aa1930 100644 --- a/cmd/config_parse.go +++ b/cmd/config_parse.go @@ -3,10 +3,10 @@ package cmd import ( "io/ioutil" + "fmt" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" - "fmt" ) func ParseConfig(path string) (config *Config, err error) { @@ -30,43 +30,76 @@ func ParseConfig(path string) (config *Config, err error) { func parseConfig(i interface{}) (c *Config, err error) { - var jm map[string]map[string]interface{} - if err := mapstructure.Decode(i, &jm); err != nil { - return nil, errors.Wrap(err, "config must be a dict with job name as key and jobs as values") + var asMap struct { + Global map[string]interface{} + Jobs []map[string]interface{} + } + if err := mapstructure.Decode(i, &asMap); err != nil { + return nil, errors.Wrap(err, "config root must be a dict") } - c = &Config{ - Jobs: make(map[string]Job, len(jm)), + c = &Config{} + + // Parse global with defaults + c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver" + err = mapstructure.Decode(asMap.Global, &c.Global) + if err != nil { + err = errors.Wrap(err, "cannot parse global section: %s") + return } - for name := range jm { - - c.Jobs[name], err = parseJob(name, jm[name]) + // Parse Jobs + c.Jobs = make(map[string]Job, len(asMap.Jobs)) + for i := range asMap.Jobs { + job, err := parseJob(asMap.Jobs[i]) if err != nil { - err = errors.Wrapf(err, "cannot parse job '%s'", name) + // Try to find its name + namei, ok := asMap.Jobs[i]["name"] + if !ok { + namei = fmt.Sprintf("", i) + } + err = errors.Wrapf(err, "cannot parse job '%v'", namei) return nil, err } - + c.Jobs[job.JobName()] = job } return c, nil } -func parseJob(name string, i map[string]interface{}) (j Job, err error) { - - jobtype_i, ok := i["type"] +func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) { + vi, ok := i[key] if !ok { - err = errors.New("must have field 'type'") - return nil, err + err = errors.Errorf("must have field '%s'", key) + return "", err } - jobtype_str, ok := jobtype_i.(string) + field, ok = vi.(string) if !ok { - err = errors.New("'type' field must have type string") - return nil, err + err = errors.Errorf("'%s' field must have type string", key) + return "", err + } + if notempty && len(field) <= 0 { + err = errors.Errorf("'%s' field must not be empty", key) + return "", err + } + return +} + +func parseJob(i map[string]interface{}) (j Job, err error) { + + name, err := extractStringField(i, "name", true) + if err != nil { + return } - switch jobtype_str { + jobtype, err := extractStringField(i, "type", true) + if err != nil { + return + + } + + switch jobtype { case "pull": return parsePullJob(name, i) case "source": @@ -74,7 +107,7 @@ func parseJob(name string, i map[string]interface{}) (j Job, err error) { case "local": return parseLocalJob(name, i) default: - return nil, errors.Errorf("unknown job type '%s'", jobtype_str) + return nil, errors.Errorf("unknown job type '%s'", jobtype) } panic("implementation error") @@ -83,22 +116,17 @@ func parseJob(name string, i map[string]interface{}) (j Job, err error) { } func parseConnect(i map[string]interface{}) (c RPCConnecter, err error) { - type_i, ok := i["type"] - if !ok { - err = errors.New("must have field 'type'") - return - } - type_str, ok := type_i.(string) - if !ok { - err = errors.New("'type' field must have type string") + + t, err := extractStringField(i, "type", true) + if err != nil { return nil, err } - switch type_str { + switch t { case "ssh+stdinserver": return parseSSHStdinserverConnecter(i) default: - return nil, errors.Errorf("unknown connection type '%s'", type_str) + return nil, errors.Errorf("unknown connection type '%s'", t) } panic("implementation error") @@ -131,9 +159,8 @@ err: func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) { - policyName, ok := v["policy"] - if !ok { - err = errors.Errorf("policy name not specified") + policyName, err := extractStringField(v, "policy", true) + if err != nil { return } @@ -152,22 +179,16 @@ func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) { func parseAuthenticatedChannelListenerFactory(v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) { - t, ok := v["type"] - if !ok { - err = errors.Errorf("must specify 'type' field") - return - } - s, ok := t.(string) - if !ok { - err = errors.Errorf("'type' must be a string") - return + t, err := extractStringField(v, "type", true) + if err != nil { + return nil, err } - switch s{ + switch t { case "stdinserver": return parseStdinserverListenerFactory(v) default: - err = errors.Errorf("unknown type '%s'", s) + err = errors.Errorf("unknown type '%s'", t) return } diff --git a/cmd/sampleconf/localbackup/host1.yml b/cmd/sampleconf/localbackup/host1.yml index 3adad0b..f88aaea 100644 --- a/cmd/sampleconf/localbackup/host1.yml +++ b/cmd/sampleconf/localbackup/host1.yml @@ -1,5 +1,5 @@ -mirror_local: - +jobs: +- name: mirror_local type: local # snapshot the filesystems matched by the left-hand-side of the mapping diff --git a/cmd/sampleconf/pullbackup/backuphost.yml b/cmd/sampleconf/pullbackup/backuphost.yml index a2b2814..a6eca1d 100644 --- a/cmd/sampleconf/pullbackup/backuphost.yml +++ b/cmd/sampleconf/pullbackup/backuphost.yml @@ -1,7 +1,7 @@ -fullbackup_prod1: - - # connect to remote using ssh / stdinserver command +jobs: +- name: fullbackup_prod1 type: pull + # connect to remote using ssh / stdinserver command connect: type: ssh+stdinserver host: prod1.example.com diff --git a/cmd/sampleconf/pullbackup/productionhost.yml b/cmd/sampleconf/pullbackup/productionhost.yml index b96a9ad..1d080a0 100644 --- a/cmd/sampleconf/pullbackup/productionhost.yml +++ b/cmd/sampleconf/pullbackup/productionhost.yml @@ -1,14 +1,30 @@ -fullbackup_prod1: +global: + serve: + stdinserver: + # Directory where AF_UNIX sockets for stdinserver command are placed. + # + # `zrepl stdinserver CLIENT_IDENTITY` + # * connects to the socket in $sockdir/CLIENT_IDENTITY + # * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3)) + # * does nothing more + # + # This enables a setup where `zrepl daemon` is not directly exposed to the internet + # but instead all traffic is tunnelled through SSH. + # The server with the source job has an authorized_keys file entry for the public key + # used by the corresponding pull job + # + # command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver + # + # Below is the default value. + sockdir: /var/run/zrepl/stdinserver +jobs: + +- name: fullbackup_prod1 # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity type: source serve: - # Creates an AF_UNIX socket with name client_identity in a well-known directory - # private to the zrepl user (which runs both the zrepld and the stdinserver command via authorized_keys) - # The stdinserver command connects to that socket and sends its stdin and stdout - # file descriptors over that UNIX socket to the zrepld. - # This avoids additional memory-to-memory copies and is more portable than splice(2) on Linux. - type: stdinserver + type: stdinserver # see global.serve.stdinserver for explanation client_identity: fullbackup_prod1 # snapshot these filesystems every 10m with zrepl_ as prefix diff --git a/cmd/sampleconf/pushbackup/backuphost.yml b/cmd/sampleconf/pushbackup/backuphost.yml index 68aec02..25431a6 100644 --- a/cmd/sampleconf/pushbackup/backuphost.yml +++ b/cmd/sampleconf/pushbackup/backuphost.yml @@ -1,4 +1,5 @@ -fullbackup_prod1: +jobs: +- name: fullbackup_prod1 # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity type: push-sink diff --git a/cmd/sampleconf/pushbackup/productionhost.yml b/cmd/sampleconf/pushbackup/productionhost.yml index 7722d58..58e962e 100644 --- a/cmd/sampleconf/pushbackup/productionhost.yml +++ b/cmd/sampleconf/pushbackup/productionhost.yml @@ -1,4 +1,5 @@ -fullbackup_prod1: +jobs: +- name: fullbackup_prod1 # connect to remote using ssh / stdinserver command type: push