diff --git a/cmd/autosnap.go b/cmd/autosnap.go index 52dbc5b..74fa70f 100644 --- a/cmd/autosnap.go +++ b/cmd/autosnap.go @@ -5,14 +5,11 @@ import ( "github.com/spf13/cobra" "github.com/zrepl/zrepl/jobrun" "github.com/zrepl/zrepl/zfs" + "os" "sync" "time" ) -var autosnapArgs struct { - job string -} - var AutosnapCmd = &cobra.Command{ Use: "autosnap", Short: "perform automatic snapshotting", @@ -20,7 +17,6 @@ var AutosnapCmd = &cobra.Command{ } func init() { - AutosnapCmd.Flags().StringVar(&autosnapArgs.job, "job", "", "job to run") RootCmd.AddCommand(AutosnapCmd) } @@ -33,34 +29,34 @@ func cmdAutosnap(cmd *cobra.Command, args []string) { runner.Start() }() - log.Printf("autosnap...") - - for i := range conf.Autosnaps { - - snap := conf.Autosnaps[i] - - if autosnapArgs.job == "" || autosnapArgs.job == snap.Name { - - job := jobrun.Job{ - Name: fmt.Sprintf("autosnap.%s", snap.Name), - RepeatStrategy: snap.Interval, - RunFunc: func(log jobrun.Logger) error { - log.Printf("doing autosnap: %v", snap) - ctx := AutosnapContext{snap} - return doAutosnap(ctx, log) - }, - } - runner.AddJob(job) - - } + if len(args) < 1 { + log.Printf("must specify exactly one job as positional argument") + os.Exit(1) } + snap, ok := conf.Autosnaps[args[0]] + if !ok { + log.Printf("could not find autosnap job: %s", args[0]) + os.Exit(1) + } + + job := jobrun.Job{ + Name: snap.JobName, + RepeatStrategy: snap.Interval, + RunFunc: func(log jobrun.Logger) error { + log.Printf("doing autosnap: %v", snap) + ctx := AutosnapContext{snap} + return doAutosnap(ctx, log) + }, + } + runner.AddJob(job) + wg.Wait() } type AutosnapContext struct { - Autosnap Autosnap + Autosnap *Autosnap } func doAutosnap(ctx AutosnapContext, log Logger) (err error) { diff --git a/cmd/config.go b/cmd/config.go index 7dd1833..37a7805 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -19,6 +19,13 @@ import ( "time" ) +var ( + JobSectionPush string = "push" + JobSectionPull string = "pull" + JobSectionPrune string = "prune" + JobSectionAutosnap string = "autosnap" +) + type Pool struct { Name string Transport Transport @@ -43,12 +50,14 @@ type SSHTransport struct { } type Push struct { + JobName string // for use with jobrun package To *Pool Filter zfs.DatasetMapping InitialReplPolicy rpc.InitialReplPolicy RepeatStrategy jobrun.RepeatStrategy } type Pull struct { + JobName string // for use with jobrun package From *Pool Mapping zfs.DatasetMapping InitialReplPolicy rpc.InitialReplPolicy @@ -61,27 +70,27 @@ type ClientMapping struct { } type Prune struct { - Name string + JobName string // for use with jobrun package DatasetFilter zfs.DatasetMapping SnapshotFilter zfs.FilesystemVersionFilter RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies? } type Autosnap struct { - Name string + JobName string // for use with jobrun package Prefix string Interval jobrun.RepeatStrategy DatasetFilter zfs.DatasetMapping } type Config struct { - Pools []Pool - Pushs []Push - Pulls []Pull - Sinks []ClientMapping - PullACLs []ClientMapping - Prunes []Prune - Autosnaps []Autosnap + Pools map[string]*Pool + Pushs map[string]*Push // job name -> job + Pulls map[string]*Pull // job name -> job + Sinks map[string]*ClientMapping // client identity -> mapping + PullACLs map[string]*ClientMapping // client identity -> mapping + Prunes map[string]*Prune // job name -> job + Autosnaps map[string]*Autosnap // job name -> job } func ParseConfig(path string) (config Config, err error) { @@ -106,13 +115,12 @@ func parseMain(root map[string]interface{}) (c Config, err error) { return } - poolLookup := func(name string) (*Pool, error) { - for _, pool := range c.Pools { - if pool.Name == name { - return &pool, nil - } + poolLookup := func(name string) (pool *Pool, err error) { + pool = c.Pools[name] + if pool == nil { + err = fmt.Errorf("pool '%s' not defined", name) } - return nil, errors.New(fmt.Sprintf("pool '%s' not defined", name)) + return } if c.Pushs, err = parsePushs(root["pushs"], poolLookup); err != nil { @@ -133,23 +141,32 @@ func parseMain(root map[string]interface{}) (c Config, err error) { if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil { return } + return } -func parsePools(v interface{}) (pools []Pool, err error) { +func fullJobName(section, name string) (full string, err error) { + if len(name) < 1 { + err = fmt.Errorf("job name not set") + return + } + full = fmt.Sprintf("%s.%s", section, name) + return +} - asList := make([]struct { - Name string +func parsePools(v interface{}) (pools map[string]*Pool, err error) { + + asMap := make(map[string]struct { Transport map[string]interface{} }, 0) - if err = mapstructure.Decode(v, &asList); err != nil { + if err = mapstructure.Decode(v, &asMap); err != nil { return } - pools = make([]Pool, len(asList)) - for i, p := range asList { + pools = make(map[string]*Pool, len(asMap)) + for name, p := range asMap { - if p.Name == rpc.LOCAL_TRANSPORT_IDENTITY { + if name == rpc.LOCAL_TRANSPORT_IDENTITY { err = errors.New(fmt.Sprintf("pool name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY)) return } @@ -158,8 +175,8 @@ func parsePools(v interface{}) (pools []Pool, err error) { if transport, err = parseTransport(p.Transport); err != nil { return } - pools[i] = Pool{ - Name: p.Name, + pools[name] = &Pool{ + Name: name, Transport: transport, } } @@ -194,31 +211,34 @@ func parseTransport(it map[string]interface{}) (t Transport, err error) { type poolLookup func(name string) (*Pool, error) -func parsePushs(v interface{}, pl poolLookup) (p []Push, err error) { +func parsePushs(v interface{}, pl poolLookup) (p map[string]*Push, err error) { - asList := make([]struct { + asMap := make(map[string]struct { To string Filter map[string]string InitialReplPolicy string Repeat map[string]string }, 0) - if err = mapstructure.Decode(v, &asList); err != nil { + if err = mapstructure.Decode(v, &asMap); err != nil { return } - p = make([]Push, len(asList)) + p = make(map[string]*Push, len(asMap)) - for i, e := range asList { + for name, e := range asMap { var toPool *Pool if toPool, err = pl(e.To); err != nil { return } - push := Push{ + push := &Push{ To: toPool, } + if push.JobName, err = fullJobName(JobSectionPush, name); err != nil { + return + } if push.Filter, err = parseComboMapping(e.Filter); err != nil { return } @@ -231,34 +251,39 @@ func parsePushs(v interface{}, pl poolLookup) (p []Push, err error) { return } - p[i] = push + p[name] = push } return } -func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) { +func parsePulls(v interface{}, pl poolLookup) (p map[string]*Pull, err error) { - asList := make([]struct { + asMap := make(map[string]struct { From string Mapping map[string]string InitialReplPolicy string Repeat map[string]string }, 0) - if err = mapstructure.Decode(v, &asList); err != nil { + if err = mapstructure.Decode(v, &asMap); err != nil { return } - p = make([]Pull, len(asList)) + p = make(map[string]*Pull, len(asMap)) - for i, e := range asList { + for name, e := range asMap { + + if len(e.From) < 1 { + err = fmt.Errorf("source pool not set (from attribute is empty)") + return + } var fromPool *Pool if e.From == rpc.LOCAL_TRANSPORT_IDENTITY { fromPool = &Pool{ - Name: "local", + Name: rpc.LOCAL_TRANSPORT_IDENTITY, Transport: LocalTransport{}, } } else { @@ -267,9 +292,12 @@ func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) { } } - pull := Pull{ + pull := &Pull{ From: fromPool, } + if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil { + return + } if pull.Mapping, err = parseComboMapping(e.Mapping); err != nil { return } @@ -280,7 +308,7 @@ func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) { return } - p[i] = pull + p[name] = pull } return @@ -337,35 +365,35 @@ func expectList(v interface{}) (asList []interface{}, err error) { return } -func parseClientMappings(v interface{}) (cm []ClientMapping, err error) { +func parseClientMappings(v interface{}) (cm map[string]*ClientMapping, err error) { - var asList []interface{} - if asList, err = expectList(v); err != nil { + var asMap map[string]interface{} + if err = mapstructure.Decode(v, &asMap); err != nil { return } - cm = make([]ClientMapping, len(asList)) + cm = make(map[string]*ClientMapping, len(asMap)) - for i, e := range asList { - var m ClientMapping - if m, err = parseClientMapping(e); err != nil { + for identity, e := range asMap { + var m *ClientMapping + if m, err = parseClientMapping(e, identity); err != nil { return } - cm[i] = m + cm[identity] = m } return } -func parseClientMapping(v interface{}) (s ClientMapping, err error) { +func parseClientMapping(v interface{}, identity string) (s *ClientMapping, err error) { t := struct { - From string Mapping map[string]string }{} if err = mapstructure.Decode(v, &t); err != nil { return } - - s.From = t.From + s = &ClientMapping{ + From: identity, + } s.Mapping, err = parseComboMapping(t.Mapping) return } @@ -457,26 +485,29 @@ func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) { t.Handler = handler } -func parsePrunes(m interface{}) (rets []Prune, err error) { +func parsePrunes(m interface{}) (rets map[string]*Prune, err error) { - asList := make([]map[string]interface{}, 0) + asList := make(map[string]map[string]interface{}, 0) if err = mapstructure.Decode(m, &asList); err != nil { return } - rets = make([]Prune, len(asList)) + rets = make(map[string]*Prune, len(asList)) - for i, e := range asList { - if rets[i], err = parsePrune(e); err != nil { - err = fmt.Errorf("cannot parse prune job #%d: %s", i+1, err) + for name, e := range asList { + var prune *Prune + if prune, err = parsePrune(e, name); err != nil { + err = fmt.Errorf("cannot parse prune job %s: %s", name, err) return } + rets[name] = prune } return } -func parsePrune(e map[string]interface{}) (prune Prune, err error) { +func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) { + // Only support grid policy for now policyName, ok := e["policy"] if !ok || policyName != "grid" { @@ -485,7 +516,6 @@ func parsePrune(e map[string]interface{}) (prune Prune, err error) { } var i struct { - Name string Grid string DatasetFilter map[string]string `mapstructure:"dataset_filter"` SnapshotFilter map[string]string `mapstructure:"snapshot_filter"` @@ -495,7 +525,11 @@ func parsePrune(e map[string]interface{}) (prune Prune, err error) { return } - prune.Name = i.Name + prune = &Prune{} + + if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil { + return + } // Parse grid policy intervals, err := parseRetentionGridIntervalsString(i.Grid) @@ -636,30 +670,31 @@ func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersion return } -func parseAutosnaps(m interface{}) (snaps []Autosnap, err error) { +func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) { - asList := make([]map[string]interface{}, 0) - if err = mapstructure.Decode(m, &asList); err != nil { + asMap := make(map[string]interface{}, 0) + if err = mapstructure.Decode(m, &asMap); err != nil { return } - snaps = make([]Autosnap, len(asList)) + snaps = make(map[string]*Autosnap, len(asMap)) - for i, e := range asList { - if snaps[i], err = parseAutosnap(e); err != nil { - err = fmt.Errorf("cannot parse autonsap job #%d: %s", i+1, err) + for name, e := range asMap { + var snap *Autosnap + if snap, err = parseAutosnap(e, name); err != nil { + err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err) return } + snaps[name] = snap } return } -func parseAutosnap(m map[string]interface{}) (a Autosnap, err error) { +func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) { var i struct { - Name string Prefix string Interval string DatasetFilter map[string]string `mapstructure:"dataset_filter"` @@ -670,7 +705,11 @@ func parseAutosnap(m map[string]interface{}) (a Autosnap, err error) { return } - a.Name = i.Name + a = &Autosnap{} + + if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil { + return + } if len(i.Prefix) < 1 { err = fmt.Errorf("prefix must not be empty") diff --git a/cmd/prune.go b/cmd/prune.go index 864b46f..b305a84 100644 --- a/cmd/prune.go +++ b/cmd/prune.go @@ -29,35 +29,29 @@ func init() { func cmdPrune(cmd *cobra.Command, args []string) { - jobFailed := false - - log.Printf("retending...") - - for _, prune := range conf.Prunes { - - if pruneArgs.job == "" || pruneArgs.job == prune.Name { - log.Printf("Beginning prune job:\n%s", prune) - ctx := PruneContext{prune, time.Now(), pruneArgs.dryRun} - err := doPrune(ctx, log) - if err != nil { - jobFailed = true - log.Printf("Prune job failed with error: %s", err) - } - log.Printf("\n") - - } - + if len(args) < 1 { + log.Printf("must specify exactly one job as positional argument") + os.Exit(1) } - if jobFailed { - log.Printf("At least one job failed with an error. Check log for details.") + job, ok := conf.Prunes[args[0]] + if !ok { + log.Printf("could not find prune job: %s", args[0]) + os.Exit(1) + } + + log.Printf("Beginning prune job:\n%s", job) + ctx := PruneContext{job, time.Now(), pruneArgs.dryRun} + err := doPrune(ctx, log) + if err != nil { + log.Printf("Prune job failed with error: %s", err) os.Exit(1) } } type PruneContext struct { - Prune Prune + Prune *Prune Now time.Time DryRun bool } diff --git a/cmd/replication.go b/cmd/replication.go index afc9729..37dcd04 100644 --- a/cmd/replication.go +++ b/cmd/replication.go @@ -7,6 +7,7 @@ import ( "github.com/zrepl/zrepl/rpc" "github.com/zrepl/zrepl/zfs" "io" + "os" "sync" "time" ) @@ -22,10 +23,61 @@ var RunCmd = &cobra.Command{ Run: cmdRun, } +var PushCmd = &cobra.Command{ + Use: "push", + Short: "run push job (first positional argument)", + Run: cmdPush, +} + +var PullCmd = &cobra.Command{ + Use: "pull", + Short: "run pull job (first positional argument)", + Run: cmdPull, +} + func init() { RootCmd.AddCommand(RunCmd) RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior") RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job") + + RootCmd.AddCommand(PushCmd) + RootCmd.AddCommand(PullCmd) +} + +func cmdPush(cmd *cobra.Command, args []string) { + + if len(args) != 1 { + log.Printf("must specify exactly one job as positional argument") + os.Exit(1) + } + job, ok := conf.Pushs[args[0]] + if !ok { + log.Printf("could not find push job %s", args[0]) + os.Exit(1) + } + if err := jobPush(job, log); err != nil { + log.Printf("error doing push: %s", err) + os.Exit(1) + } + +} + +func cmdPull(cmd *cobra.Command, args []string) { + + if len(args) != 1 { + log.Printf("must specify exactly one job as positional argument") + os.Exit(1) + } + job, ok := conf.Pulls[args[0]] + if !ok { + log.Printf("could not find pull job %s", args[0]) + os.Exit(1) + } + if err := jobPull(job, log); err != nil { + log.Printf("error doing pull: %s", err) + os.Exit(1) + } + } func cmdRun(cmd *cobra.Command, args []string) { @@ -87,7 +139,7 @@ func cmdRun(cmd *cobra.Command, args []string) { } -func jobPull(pull Pull, log jobrun.Logger) (err error) { +func jobPull(pull *Pull, log jobrun.Logger) (err error) { if lt, ok := pull.From.Transport.(LocalTransport); ok { lt.SetHandler(Handler{ @@ -109,7 +161,7 @@ func jobPull(pull Pull, log jobrun.Logger) (err error) { return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy}) } -func jobPush(push Push, log jobrun.Logger) (err error) { +func jobPush(push *Push, log jobrun.Logger) (err error) { if _, ok := push.To.Transport.(LocalTransport); ok { panic("no support for local pushs") diff --git a/cmd/sampleconf/zrepl.yml b/cmd/sampleconf/zrepl.yml index 0588182..b8033ea 100644 --- a/cmd/sampleconf/zrepl.yml +++ b/cmd/sampleconf/zrepl.yml @@ -1,5 +1,5 @@ pools: - - name: offsite_backups + offsite_backups: transport: ssh: host: 192.168.122.6 @@ -8,21 +8,26 @@ pools: identity_file: /etc/zrepl/identities/offsite_backups pushs: - - to: offsite_backups + + offsite: + to: offsite_backups filter: { "tank/var/db*":"ok", "tank/usr/home*":"ok" } pulls: - - from: offsite_backups + + offsite: + from: offsite_backups mapping: { # like in sinks } # local replication, only allowed in pull mode # the from name 'local' is reserved for this purpose - - from: local + homemirror: + from: local repeat: interval: 15m mapping: { @@ -35,21 +40,21 @@ sinks: # 1:1 mapping of remote dataset to local dataset # We will reject a push request which contains > 0 datasets that do not # match a mapping - - from: db1 + db1: mapping: { "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data" } # "|" non-recursive wildcard # the remote must present excatly one dataset, mapped to the rhs - - from: cdn_master + cdn_master: mapping: { "|":"tank/srv/cdn" # NOTE: | is currently an invalid character for a ZFS dataset } # "*" recursive wildcard # the remote may present an arbitrary set of marks a recursive wildcard, i.e. map all remotes to a tree under rhs - - from: mirror1 + mirror1: mapping: { "tank/foo/bar*":"zroot/backups/mirror1" # NOTE: * is currently an invalid character for a ZFS dataset } @@ -60,7 +65,7 @@ sinks: # local dataset (same order) or '!optional reason' on stdout # If the acceptor scripts exits with non-zero status code, the remote's # request will be rejected - - from: complex_host + complex_host: mapping: { # "*":"!/path/to/acceptor" # we could just wire the path to the acceptor directly to the mapping # but let's stick with the same type for the mapping field for now' @@ -70,7 +75,7 @@ sinks: # Mixing the rules # Mixing should be possible if there is a defined precedence (direct before *) # and non-recursive wildcards are not allowed in multi-entry mapping objects - - from: special_snowflake + special_snowflake: mapping: { # an explicit mapping mixed with a recursive wildcard "sun/usr/home": backups/special_snowflake/homedirs, "sun/var/db": backups/special_snowflake/database, @@ -79,8 +84,9 @@ sinks: } pull_acls: + # same synatx as in sinks, but the returned mapping does not matter - - from: office_backup + office_backup: mapping: { "tank/usr/home":"notnull" } @@ -88,7 +94,7 @@ pull_acls: prune: - - name: clean_backups + clean_backups: policy: grid grid: 6x10min | 24x1h | 7x1d | 32 x 1d | 4 x 3mon dataset_filter: { @@ -98,7 +104,7 @@ prune: prefix: zrepl_ } - - name: hfbak_prune # cleans up after hfbak autosnap job + hfbak_prune: # cleans up after hfbak autosnap job policy: grid grid: 1x1min(keep=all) dataset_filter: { @@ -110,7 +116,7 @@ prune: autosnap: - - name: hfbak + hfbak: prefix: zrepl_hfbak_ interval: 1s dataset_filter: { diff --git a/cmd/stdinserver.go b/cmd/stdinserver.go index 5834b9a..e8c67d4 100644 --- a/cmd/stdinserver.go +++ b/cmd/stdinserver.go @@ -11,18 +11,13 @@ import ( "os" ) -var stdinserver struct { - identity string -} - var StdinserverCmd = &cobra.Command{ - Use: "stdinserver", + Use: "stdinserver CLIENT_IDENTITY", Short: "start in stdin server mode (from authorized_keys file)", Run: cmdStdinServer, } func init() { - StdinserverCmd.Flags().StringVar(&stdinserver.identity, "identity", "", "") RootCmd.AddCommand(StdinserverCmd) } @@ -36,37 +31,36 @@ func cmdStdinServer(cmd *cobra.Command, args []string) { } }() - if stdinserver.identity == "" { - err = fmt.Errorf("identity flag not set") + if len(args) != 1 || args[0] == "" { + err = fmt.Errorf("must specify client identity as positional argument") + return + } + identity := args[0] + + pullACL := conf.PullACLs[identity] + if pullACL == nil { + err = fmt.Errorf("could not find PullACL for identity '%s'", identity) return } - identity := stdinserver.identity var sshByteStream io.ReadWriteCloser if sshByteStream, err = sshbytestream.Incoming(); err != nil { return } - findMapping := func(cm []ClientMapping, identity string) zfs.DatasetMapping { - for i := range cm { - if cm[i].From == identity { - return cm[i].Mapping - } - } - return nil - } - sinkMapping := func(identity string) (sink zfs.DatasetMapping, err error) { - if sink = findMapping(conf.Sinks, identity); sink == nil { + sinkMapping := func(identity string) (m zfs.DatasetMapping, err error) { + sink := conf.Sinks[identity] + if sink == nil { return nil, fmt.Errorf("could not find sink for dataset") } - return + return sink.Mapping, nil } sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags) handler := Handler{ Logger: sinkLogger, SinkMappingFunc: sinkMapping, - PullACL: findMapping(conf.PullACLs, identity), + PullACL: pullACL.Mapping, } if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {