diff --git a/cmd/config.go b/cmd/config.go index e7bca2b..dee7f6a 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -52,26 +52,21 @@ type SSHTransport struct { type Push struct { JobName string // for use with jobrun package To *Remote - Filter zfs.DatasetMapping + Filter zfs.DatasetFilter InitialReplPolicy rpc.InitialReplPolicy RepeatStrategy jobrun.RepeatStrategy } type Pull struct { JobName string // for use with jobrun package From *Remote - Mapping zfs.DatasetMapping + Mapping DatasetMapFilter InitialReplPolicy rpc.InitialReplPolicy RepeatStrategy jobrun.RepeatStrategy } -type ClientMapping struct { - From string - Mapping zfs.DatasetMapping -} - type Prune struct { JobName string // for use with jobrun package - DatasetFilter zfs.DatasetMapping + DatasetFilter zfs.DatasetFilter SnapshotFilter zfs.FilesystemVersionFilter RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies? } @@ -80,17 +75,17 @@ type Autosnap struct { JobName string // for use with jobrun package Prefix string Interval jobrun.RepeatStrategy - DatasetFilter zfs.DatasetMapping + DatasetFilter zfs.DatasetFilter } type Config struct { Remotes map[string]*Remote - Pushs map[string]*Push // job name -> job - Pulls map[string]*Pull // job name -> job - Sinks map[string]*ClientMapping // client identity -> mapping - PullACLs map[string]*ClientMapping // client identity -> mapping - Prunes map[string]*Prune // job name -> job - Autosnaps map[string]*Autosnap // job name -> job + Pushs map[string]*Push // job name -> job + Pulls map[string]*Pull // job name -> job + Sinks map[string]DatasetMapFilter // client identity -> mapping + PullACLs map[string]DatasetMapFilter // client identity -> filter + Prunes map[string]*Prune // job name -> job + Autosnaps map[string]*Autosnap // job name -> job } func ParseConfig(path string) (config Config, err error) { @@ -129,10 +124,10 @@ func parseMain(root map[string]interface{}) (c Config, err error) { if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil { return } - if c.Sinks, err = parseClientMappings(root["sinks"]); err != nil { + if c.Sinks, err = parseSinks(root["sinks"]); err != nil { return } - if c.PullACLs, err = parseClientMappings(root["pull_acls"]); err != nil { + if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil { return } if c.Prunes, err = parsePrunes(root["prune"]); err != nil { @@ -239,7 +234,7 @@ func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) if push.JobName, err = fullJobName(JobSectionPush, name); err != nil { return } - if push.Filter, err = parseComboMapping(e.Filter); err != nil { + if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil { return } @@ -298,7 +293,7 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil { return } - if pull.Mapping, err = parseComboMapping(e.Mapping); err != nil { + if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil { return } if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil { @@ -365,97 +360,78 @@ func expectList(v interface{}) (asList []interface{}, err error) { return } -func parseClientMappings(v interface{}) (cm map[string]*ClientMapping, err error) { - - var asMap map[string]interface{} +func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) { + var asMap map[string]map[string]interface{} if err = mapstructure.Decode(v, &asMap); err != nil { return } - cm = make(map[string]*ClientMapping, len(asMap)) + m = make(map[string]DatasetMapFilter, len(asMap)) - for identity, e := range asMap { - var m *ClientMapping - if m, err = parseClientMapping(e, identity); err != nil { + for identity, entry := range asMap { + parseSink := func() (mapping DatasetMapFilter, err error) { + mappingMap, ok := entry["mapping"] + if !ok { + err = fmt.Errorf("no mapping specified") + return + } + mapping, err = parseDatasetMapFilter(mappingMap, false) return } - cm[identity] = m + mapping, sinkErr := parseSink() + if sinkErr != nil { + err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr) + return + } + m[identity] = mapping } return } -func parseClientMapping(v interface{}, identity string) (s *ClientMapping, err error) { - t := struct { - Mapping map[string]string - }{} - if err = mapstructure.Decode(v, &t); err != nil { +func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) { + var asMap map[string]map[string]interface{} + if err = mapstructure.Decode(v, &asMap); err != nil { return } - s = &ClientMapping{ - From: identity, + + m = make(map[string]DatasetMapFilter, len(asMap)) + + for identity, entry := range asMap { + parsePullACL := func() (filter DatasetMapFilter, err error) { + filterMap, ok := entry["filter"] + if !ok { + err = fmt.Errorf("no filter specified") + return + } + filter, err = parseDatasetMapFilter(filterMap, true) + return + } + filter, filterErr := parsePullACL() + if filterErr != nil { + err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr) + return + } + m[identity] = filter } - s.Mapping, err = parseComboMapping(t.Mapping) return } -func parseComboMapping(m map[string]string) (c zfs.ComboMapping, err error) { - - c.Mappings = make([]zfs.DatasetMapping, 0, len(m)) - - for lhs, rhs := range m { - - if lhs == "*" && strings.HasPrefix(rhs, "!") { - - m := zfs.ExecMapping{} - fields := strings.Fields(strings.TrimPrefix(rhs, "!")) - if len(fields) < 1 { - err = errors.New("ExecMapping without acceptor path") - return - } - m.Name = fields[0] - m.Args = fields[1:] - - c.Mappings = append(c.Mappings, m) - - } else if strings.HasSuffix(lhs, "*") { - - m := zfs.GlobMapping{} - - m.PrefixPath, err = zfs.NewDatasetPath(strings.TrimSuffix(lhs, "*")) - if err != nil { - return - } - - if m.TargetRoot, err = zfs.NewDatasetPath(rhs); err != nil { - return - } - - c.Mappings = append(c.Mappings, m) - - } else { - - m := zfs.DirectMapping{} - - if lhs == "|" { - m.Source = nil - } else { - if m.Source, err = zfs.NewDatasetPath(lhs); err != nil { - return - } - } - - if m.Target, err = zfs.NewDatasetPath(rhs); err != nil { - return - } - - c.Mappings = append(c.Mappings, m) - - } +func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) { + var m map[string]string + if err = mapstructure.Decode(mi, &m); err != nil { + err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err) + return } + f = NewDatasetMapFilter(len(m), filterOnly) + for pathPattern, mapping := range m { + if err = f.Add(pathPattern, mapping); err != nil { + err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err) + return + } + } return - } func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) { @@ -551,7 +527,7 @@ func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) prune.RetentionPolicy = NewRetentionGrid(intervals) // Parse filters - if prune.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil { + if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil { err = fmt.Errorf("cannot parse dataset filter: %s", err) return } @@ -746,7 +722,7 @@ func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) { err = fmt.Errorf("dataset_filter not specified") return } - if a.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil { + if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil { err = fmt.Errorf("cannot parse dataset filter: %s", err) } diff --git a/cmd/config_mapfilter.go b/cmd/config_mapfilter.go new file mode 100644 index 0000000..65546b7 --- /dev/null +++ b/cmd/config_mapfilter.go @@ -0,0 +1,164 @@ +package cmd + +import ( + "errors" + "fmt" + "strings" + + "github.com/zrepl/zrepl/zfs" +) + +type DatasetMapFilter struct { + entries []datasetMapFilterEntry + + // if set, only valid filter entries can be added using Add() + // and Map() will always return an error + filterOnly bool +} + +type datasetMapFilterEntry struct { + path zfs.DatasetPath + // the mapping. since this datastructure acts as both mapping and filter + // we have to convert it to the desired rep dynamically + mapping string + subtreeMatch bool +} + +var NoMatchError error = errors.New("no match found in mapping") + +func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter { + return DatasetMapFilter{ + entries: make([]datasetMapFilterEntry, 0, capacity), + } +} + +func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) { + + if m.filterOnly { + if _, err = parseDatasetFilterResult(mapping); err != nil { + return + } + } + + // assert path glob adheres to spec + const SUBTREE_PATTERN string = "<" + patternCount := strings.Count(pathPattern, SUBTREE_PATTERN) + switch { + case patternCount > 1: + case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN): + err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed") + return + } + + var path zfs.DatasetPath + pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN) + path, err = zfs.NewDatasetPath(pathStr) + if err != nil { + err = fmt.Errorf("pattern is not a dataset path: %s", err) + return + } + + entry := datasetMapFilterEntry{ + path: path, + mapping: mapping, + subtreeMatch: patternCount > 0, + } + m.entries = append(m.entries, entry) + return + +} + +// find the most specific prefix mapping we have +// +// longer prefix wins over shorter prefix, direct wins over glob +func (m DatasetMapFilter) mostSpecificPrefixMapping(path zfs.DatasetPath) (idx int, found bool) { + lcp, lcp_entry_idx := -1, -1 + direct_idx := -1 + for e := range m.entries { + entry := m.entries[e] + ep := m.entries[e].path + lep := ep.Length() + + switch { + case !entry.subtreeMatch && ep.Equal(path): + direct_idx = e + continue + case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp: + lcp = lep + lcp_entry_idx = e + default: + continue + } + } + + if lcp_entry_idx >= 0 || direct_idx >= 0 { + found = true + switch { + case direct_idx >= 0: + idx = direct_idx + case lcp_entry_idx >= 0: + idx = lcp_entry_idx + } + } + return +} + +func (m DatasetMapFilter) Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error) { + + if m.filterOnly { + err = fmt.Errorf("using a filter for mapping simply does not work") + return + } + + mi, hasMapping := m.mostSpecificPrefixMapping(source) + if !hasMapping { + err = NoMatchError + return + } + me := m.entries[mi] + + target, err = zfs.NewDatasetPath(me.mapping) + if err != nil { + err = fmt.Errorf("mapping target is not a dataset path: %s", err) + return + } + if m.entries[mi].subtreeMatch { + // strip common prefix + extendComps := source.Copy() + if me.path.Empty() { + // special case: trying to map the root => strip first component + extendComps.TrimNPrefixComps(1) + } else { + extendComps.TrimPrefix(me.path) + } + target.Extend(extendComps) + } + return +} + +func (m DatasetMapFilter) Filter(p zfs.DatasetPath) (pass bool, err error) { + mi, hasMapping := m.mostSpecificPrefixMapping(p) + if !hasMapping { + pass = false + return + } + me := m.entries[mi] + pass, err = parseDatasetFilterResult(me.mapping) + return +} + +// Parse a dataset filter result +func parseDatasetFilterResult(result string) (pass bool, err error) { + l := strings.ToLower(result) + switch strings.ToLower(l) { + case "ok": + pass = true + return + case "omit": + return + default: + err = fmt.Errorf("'%s' is not a valid filter result", result) + return + } + return +} diff --git a/cmd/config_test.go b/cmd/config_test.go index 59c894b..a3d1f77 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -1,10 +1,12 @@ package cmd import ( - "github.com/stretchr/testify/assert" - "github.com/zrepl/zrepl/util" "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/zrepl/zrepl/util" + "github.com/zrepl/zrepl/zfs" ) func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) { @@ -45,3 +47,77 @@ func TestParseRetentionGridStringParsing(t *testing.T) { assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount) } + +func TestDatasetMapFilter(t *testing.T) { + + expectMapping := func(m map[string]string, from, to string) { + dmf, err := parseDatasetMapFilter(m, false) + if err != nil { + t.Logf("expect test map to be valid: %s", err) + t.FailNow() + } + fromPath, err := zfs.NewDatasetPath(from) + if err != nil { + t.Logf("expect test from path to be valid: %s", err) + t.FailNow() + } + toPath, err := zfs.NewDatasetPath(to) + if err != nil { + t.Logf("expect test to path to be valid: %s", err) + t.FailNow() + } + + res, err := dmf.Map(fromPath) + t.Logf("%s => %s", fromPath.ToString(), res.ToString()) + assert.Nil(t, err) + assert.True(t, res.Equal(toPath)) + } + + expectFilter := func(m map[string]string, path string, pass bool) { + dmf, err := parseDatasetMapFilter(m, true) + if err != nil { + t.Logf("expect test filter to be valid: %s", err) + t.FailNow() + } + p, err := zfs.NewDatasetPath(path) + if err != nil { + t.Logf("expect test path to be valid: %s", err) + t.FailNow() + } + res, err := dmf.Filter(p) + assert.Nil(t, err) + assert.Equal(t, pass, res) + } + + map1 := map[string]string{ + "a/b/c<": "root1", + "a/b<": "root2", + "<": "root3/b/c", + "q<": "root4/1/2", + } + + expectMapping(map1, "a/b/c", "root1") + expectMapping(map1, "a/b/c/d", "root1/d") + expectMapping(map1, "a/b/e", "root2/e") + expectMapping(map1, "a/b", "root2") + expectMapping(map1, "x", "root3/b/c") + expectMapping(map1, "x/y", "root3/b/c/y") + expectMapping(map1, "q", "root4/1/2") + expectMapping(map1, "q/r", "root4/1/2/r") + + filter1 := map[string]string{ + "<": "omit", + "a<": "ok", + "a/b<": "omit", + } + + expectFilter(filter1, "b", false) + expectFilter(filter1, "a", true) + expectFilter(filter1, "a/d", true) + expectFilter(filter1, "a/b", false) + expectFilter(filter1, "a/b/c", false) + + filter2 := map[string]string{} + expectFilter(filter2, "foo", false) // default to omit + +} diff --git a/cmd/handler.go b/cmd/handler.go index 18274f5..b4b39b1 100644 --- a/cmd/handler.go +++ b/cmd/handler.go @@ -7,10 +7,14 @@ import ( "io" ) +type DatasetMapping interface { + Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error) +} + type Handler struct { Logger Logger - PullACL zfs.DatasetMapping - SinkMappingFunc func(clientIdentity string) (mapping zfs.DatasetMapping, err error) + PullACL zfs.DatasetFilter + SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error) } func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []zfs.DatasetPath, err error) { @@ -90,7 +94,7 @@ func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, h.Logger.Printf("handling PullMeRequest: %#v", r) - var sinkMapping zfs.DatasetMapping + var sinkMapping DatasetMapping sinkMapping, err = h.SinkMappingFunc(clientIdentity) if err != nil { h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity) diff --git a/cmd/replication.go b/cmd/replication.go index 3a45f89..a7cb054 100644 --- a/cmd/replication.go +++ b/cmd/replication.go @@ -141,12 +141,24 @@ func cmdRun(cmd *cobra.Command, args []string) { } +type localPullACL struct{} + +func (a localPullACL) Filter(p zfs.DatasetPath) (pass bool, err error) { + return true, nil +} + func jobPull(pull *Pull, log jobrun.Logger) (err error) { if lt, ok := pull.From.Transport.(LocalTransport); ok { + lt.SetHandler(Handler{ - Logger: log, - PullACL: pull.Mapping, + Logger: log, + // Allow access to any dataset since we control what mapping + // is passed to the pull routine. + // All local datasets will be passed to its Map() function, + // but only those for which a mapping exists will actually be pulled. + // We can pay this small performance penalty for now. + PullACL: localPullACL{}, }) pull.From.Transport = lt log.Printf("fixing up local transport: %#v", pull.From.Transport) @@ -228,7 +240,7 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Durat type PullContext struct { Remote rpc.RPCRequester Log Logger - Mapping zfs.DatasetMapping + Mapping DatasetMapping InitialReplPolicy rpc.InitialReplPolicy } @@ -258,7 +270,7 @@ func doPull(pull PullContext) (err error) { var localFs zfs.DatasetPath localFs, err = pull.Mapping.Map(remoteFilesystems[fs]) if err != nil { - if err != zfs.NoMatchError { + if err != NoMatchError { log.Printf("error mapping %s: %#v\n", remoteFilesystems[fs], err) return err } diff --git a/cmd/sampleconf/zrepl.yml b/cmd/sampleconf/zrepl.yml index b9984dd..e4b8321 100644 --- a/cmd/sampleconf/zrepl.yml +++ b/cmd/sampleconf/zrepl.yml @@ -12,8 +12,9 @@ pushs: offsite: to: offsite_backups filter: { - "tank/var/db*":"ok", - "tank/usr/home*":"ok" + # like in pull_acls + "tank/var/db<": ok, + "tank/usr/home<": ok, } pulls: @@ -36,59 +37,39 @@ pulls: sinks: - # direct mapping - # 1:1 mapping of remote dataset to local dataset - # We will reject a push request which contains > 0 datasets that do not - # match a mapping db1: mapping: { + # direct mapping "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data" } - # "|" non-recursive wildcard - # the remote must present excatly one dataset, mapped to the rhs - cdn_master: - mapping: { - "|":"tank/srv/cdn" # NOTE: | is currently an invalid character for a ZFS dataset - } - - # "*" recursive wildcard - # the remote may present an arbitrary set of marks a recursive wildcard, i.e. map all remotes to a tree under rhs mirror1: mapping: { - "tank/foo/bar*":"zroot/backups/mirror1" # NOTE: * is currently an invalid character for a ZFS dataset + # "<" subtree wildcard matches the dataset left of < and all its children + "tank/foo/bar<":"zroot/backups/mirror1" } - # "*":"!..." acceptor script - # shell out to an accceptor that receives the remote's offered datasets - # on stdin and, foreach line of this input, returns the corresponding - # local dataset (same order) or '!optional reason' on stdout - # If the acceptor scripts exits with non-zero status code, the remote's - # request will be rejected - complex_host: - mapping: { # - "*":"!/path/to/acceptor" # we could just wire the path to the acceptor directly to the mapping - # but let's stick with the same type for the mapping field for now' - # NOTE: * and ! are currently invalid characters for a ZFS dataset + mirror2: + # more specific path patterns win over less specific ones + # direct mappings win over subtree wildcards + # detailed rule precedence: check unit tests & docs for exact behavior + # TODO subcommand to test a mapping & filter + mapping: { + "tank<": "zroot/backups/mirror1/tank1", + "tank/cdn/root<": "storage/cdn/root", + "tank/legacydb": "legacypool/backups/legacydb", } - # Mixing the rules - # Mixing should be possible if there is a defined precedence (direct before *) - # and non-recursive wildcards are not allowed in multi-entry mapping objects - special_snowflake: - mapping: { # an explicit mapping mixed with a recursive wildcard - "sun/usr/home": backups/special_snowflake/homedirs, - "sun/var/db": backups/special_snowflake/database, - "*": backups/special_snowflake/remainingbackup - # NOTE: ^ alignment, should be possible, looks nicer - } pull_acls: - # same synatx as in sinks, but the returned mapping does not matter office_backup: - mapping: { - "tank/usr/home":"notnull" + filter: { + # valid filter results (right hand side): ok, omit + # default is to omit + # rule precedence is same as for mappings + "tank<": omit, + "tank/usr/home": ok, } @@ -98,7 +79,8 @@ prune: policy: grid grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w dataset_filter: { - "tank/backups/*": ok + "tank/backups/legacyscript<": omit, + "tank/backups<": ok, } snapshot_filter: { prefix: zrepl_ @@ -108,7 +90,7 @@ prune: policy: grid grid: 1x1m(keep=all) dataset_filter: { - "pool1*": ok + "pool1<": ok } snapshot_filter: { prefix: zrepl_hfbak_ @@ -120,7 +102,7 @@ autosnap: prefix: zrepl_hfbak_ interval: 1s dataset_filter: { - "pool1*": ok + "pool1<": ok } # prune: hfbak_prune # future versions may inline the retention policy here, but for now, diff --git a/cmd/stdinserver.go b/cmd/stdinserver.go index e8c67d4..a3f12dd 100644 --- a/cmd/stdinserver.go +++ b/cmd/stdinserver.go @@ -5,7 +5,6 @@ import ( "github.com/spf13/cobra" "github.com/zrepl/zrepl/rpc" "github.com/zrepl/zrepl/sshbytestream" - "github.com/zrepl/zrepl/zfs" "io" golog "log" "os" @@ -37,8 +36,8 @@ func cmdStdinServer(cmd *cobra.Command, args []string) { } identity := args[0] - pullACL := conf.PullACLs[identity] - if pullACL == nil { + pullACL, ok := conf.PullACLs[identity] + if !ok { err = fmt.Errorf("could not find PullACL for identity '%s'", identity) return } @@ -48,19 +47,19 @@ func cmdStdinServer(cmd *cobra.Command, args []string) { return } - sinkMapping := func(identity string) (m zfs.DatasetMapping, err error) { - sink := conf.Sinks[identity] - if sink == nil { - return nil, fmt.Errorf("could not find sink for dataset") + sinkMapping := func(identity string) (m DatasetMapping, err error) { + sink, ok := conf.Sinks[identity] + if !ok { + return nil, fmt.Errorf("could not find sink for identity '%s'", identity) } - return sink.Mapping, nil + return sink, nil } sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags) handler := Handler{ Logger: sinkLogger, SinkMappingFunc: sinkMapping, - PullACL: pullACL.Mapping, + PullACL: pullACL, } if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil { diff --git a/zfs/datasetpath_visitor.go b/zfs/datasetpath_visitor.go index 02abd88..dd1857d 100644 --- a/zfs/datasetpath_visitor.go +++ b/zfs/datasetpath_visitor.go @@ -11,20 +11,20 @@ func NewDatasetPathForest() *DatasetPathForest { } func (f *DatasetPathForest) Add(p DatasetPath) { - if len(p) <= 0 { + if len(p.comps) <= 0 { panic("dataset path too short. must have length > 0") } // Find its root var root *datasetPathTree for _, r := range f.roots { - if r.Add(p) { + if r.Add(p.comps) { root = r break } } if root == nil { - root = newDatasetPathTree(p) + root = newDatasetPathTree(p.comps) f.roots = append(f.roots, root) } } @@ -57,7 +57,7 @@ type datasetPathTree struct { Children []*datasetPathTree } -func (t *datasetPathTree) Add(p DatasetPath) bool { +func (t *datasetPathTree) Add(p []string) bool { if len(p) == 0 { return true @@ -88,11 +88,15 @@ func (t *datasetPathTree) Add(p DatasetPath) bool { } -func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVisitor) { +func (t *datasetPathTree) WalkTopDown(parent []string, visitor DatasetPathsVisitor) { this := append(parent, t.Component) - visitChildTree := visitor(DatasetPathVisit{this, t.FilledIn}) + thisVisit := DatasetPathVisit{ + DatasetPath{this}, + t.FilledIn, + } + visitChildTree := visitor(thisVisit) if visitChildTree { for _, c := range t.Children { @@ -102,15 +106,15 @@ func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVi } -func newDatasetPathTree(initial DatasetPath) (t *datasetPathTree) { +func newDatasetPathTree(initialComps []string) (t *datasetPathTree) { t = &datasetPathTree{} var cur *datasetPathTree cur = t - for i, comp := range initial { + for i, comp := range initialComps { cur.Component = comp cur.FilledIn = true cur.Children = make([]*datasetPathTree, 0, 1) - if i == len(initial)-1 { + if i == len(initialComps)-1 { cur.FilledIn = false // last component is not filled in break } diff --git a/zfs/datasetpath_visitor_test.go b/zfs/datasetpath_visitor_test.go index 61d8e1b..4b4d5d1 100644 --- a/zfs/datasetpath_visitor_test.go +++ b/zfs/datasetpath_visitor_test.go @@ -7,7 +7,7 @@ import ( func TestNewDatasetPathTree(t *testing.T) { - r := newDatasetPathTree(toDatasetPath("pool1/foo/bar")) + r := newDatasetPathTree(toDatasetPath("pool1/foo/bar").comps) assert.Equal(t, "pool1", r.Component) assert.True(t, len(r.Children) == 1) diff --git a/zfs/mapping.go b/zfs/mapping.go index f248043..889166e 100644 --- a/zfs/mapping.go +++ b/zfs/mapping.go @@ -1,21 +1,15 @@ package zfs -import ( - "bufio" - "errors" - "fmt" - "io" - "os/exec" -) +import "fmt" -type DatasetMapping interface { - Map(source DatasetPath) (target DatasetPath, err error) +type DatasetFilter interface { + Filter(p DatasetPath) (pass bool, err error) } -func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) { +func ZFSListMapping(filter DatasetFilter) (datasets []DatasetPath, err error) { - if mapping == nil { - panic("mapping must not be nil") + if filter == nil { + panic("filter must not be nil") } var lines [][]string @@ -30,12 +24,11 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) return } - _, mapErr := mapping.Map(path) - if mapErr != nil && mapErr != NoMatchError { - return nil, mapErr + pass, filterErr := filter.Filter(path) + if filterErr != nil { + return nil, fmt.Errorf("error calling filter: %s", filterErr) } - - if mapErr == nil { + if pass { datasets = append(datasets, path) } @@ -43,139 +36,3 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) return } - -type GlobMapping struct { - PrefixPath DatasetPath - TargetRoot DatasetPath -} - -var NoMatchError error = errors.New("no match found in mapping") - -func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) { - - if len(source) < len(m.PrefixPath) { - err = NoMatchError - return - } - - target = make([]string, 0, len(source)+len(m.TargetRoot)) - target = append(target, m.TargetRoot...) - - for si, sc := range source { - target = append(target, sc) - if si < len(m.PrefixPath) { - - compsMatch := sc == m.PrefixPath[si] - endOfPrefixPath := si == len(m.PrefixPath)-1 && m.PrefixPath[si] == "" - - if !(compsMatch || endOfPrefixPath) { - err = NoMatchError - return - } - continue - } - } - - return -} - -type ComboMapping struct { - Mappings []DatasetMapping -} - -func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) { - for _, sm := range m.Mappings { - target, err = sm.Map(source) - if err == nil { - return target, err - } - } - return nil, NoMatchError -} - -type DirectMapping struct { - Source DatasetPath - Target DatasetPath -} - -func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) { - - if m.Source == nil { - return m.Target, nil - } - - if len(m.Source) != len(source) { - return nil, NoMatchError - } - - for i, c := range source { - if c != m.Source[i] { - return nil, NoMatchError - } - } - - return m.Target, nil -} - -type ExecMapping struct { - Name string - Args []string -} - -func NewExecMapping(name string, args ...string) (m *ExecMapping) { - m = &ExecMapping{ - Name: name, - Args: args, - } - return -} - -func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) { - - var stdin io.Writer - var stdout io.Reader - - cmd := exec.Command(m.Name, m.Args...) - - if stdin, err = cmd.StdinPipe(); err != nil { - return - } - - if stdout, err = cmd.StdoutPipe(); err != nil { - return - } - - resp := bufio.NewScanner(stdout) - - if err = cmd.Start(); err != nil { - return - } - - go func() { - err := cmd.Wait() - if err != nil { - panic(err) - // fmt.Printf("error: %v\n", err) // TODO - } - }() - - if _, err = io.WriteString(stdin, source.ToString()+"\n"); err != nil { - return - } - - if !resp.Scan() { - err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err())) - return - } - - t := resp.Text() - - switch { - case t == "NOMAP": - return nil, NoMatchError - } - - target = toDatasetPath(t) // TODO discover garbage? - - return -} diff --git a/zfs/mapping_test.go b/zfs/mapping_test.go deleted file mode 100644 index 93640c3..0000000 --- a/zfs/mapping_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package zfs - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestGlobMappingPrefixWildcard(t *testing.T) { - - m := GlobMapping{ - PrefixPath: toDatasetPath("a/b/c/"), // TRAILING empty component! - TargetRoot: toDatasetPath("x/y"), - } - - t.Logf("PrefixPath: %#v", m.PrefixPath) - - var r DatasetPath - var err error - - r, err = m.Map(toDatasetPath("a/b/c")) - assert.NotNil(t, err) - - r, err = m.Map(toDatasetPath("a/b/c/d")) - assert.Nil(t, err) - assert.Equal(t, toDatasetPath("x/y/a/b/c/d"), r) - -} - -func TestGlobMapping(t *testing.T) { - - m := GlobMapping{ - PrefixPath: toDatasetPath("tank/usr/home"), - TargetRoot: toDatasetPath("backups/share1"), - } - - var r DatasetPath - var err error - - r, err = m.Map(toDatasetPath("tank/usr/home")) - assert.Nil(t, err) - assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r) - - r, err = m.Map(toDatasetPath("zroot")) - assert.Equal(t, NoMatchError, err, "prefix-only match is an error") - - r, err = m.Map(toDatasetPath("zroot/notmapped")) - assert.Equal(t, NoMatchError, err, "non-prefix is an error") - -} - -func TestGlobMappingWildcard(t *testing.T) { - m := GlobMapping{ - PrefixPath: EmptyDatasetPath, - TargetRoot: toDatasetPath("backups/share1"), - } - - var r DatasetPath - var err error - r, err = m.Map(toDatasetPath("tank/usr/home")) - - assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r) - assert.NoError(t, err) -} - -func TestComboMapping(t *testing.T) { - - m1 := GlobMapping{ - PrefixPath: toDatasetPath("a/b"), - TargetRoot: toDatasetPath("c/d"), - } - - m2 := GlobMapping{ - PrefixPath: toDatasetPath("a/x"), - TargetRoot: toDatasetPath("c/y"), - } - - c := ComboMapping{ - Mappings: []DatasetMapping{m1, m2}, - } - - var r DatasetPath - var err error - - p := toDatasetPath("a/b/q") - - r, err = m2.Map(p) - assert.Equal(t, NoMatchError, err) - - r, err = c.Map(p) - assert.Nil(t, err) - assert.Equal(t, toDatasetPath("c/d/a/b/q"), r) - -} - -func TestDirectMapping(t *testing.T) { - - m := DirectMapping{ - Source: toDatasetPath("a/b/c"), - Target: toDatasetPath("x/y/z"), - } - - var r DatasetPath - var err error - - r, err = m.Map(toDatasetPath("a/b/c")) - assert.Nil(t, err) - assert.Equal(t, m.Target, r) - - r, err = m.Map(toDatasetPath("not/matching")) - assert.Equal(t, NoMatchError, err) - - r, err = m.Map(toDatasetPath("a/b")) - assert.Equal(t, NoMatchError, err) - -} - -func TestExecMapping(t *testing.T) { - - var err error - - var m DatasetMapping - m = NewExecMapping("test_helpers/exec_mapping_good.sh", "nostop") - assert.NoError(t, err) - - var p DatasetPath - p, err = m.Map(toDatasetPath("nomap/foobar")) - - assert.Equal(t, NoMatchError, err) - - p, err = m.Map(toDatasetPath("willmap/something")) - assert.Nil(t, err) - assert.Equal(t, toDatasetPath("didmap/willmap/something"), p) - -} diff --git a/zfs/zfs.go b/zfs/zfs.go index c8bc369..55047e9 100644 --- a/zfs/zfs.go +++ b/zfs/zfs.go @@ -11,27 +11,97 @@ import ( "strings" ) -type DatasetPath []string +type DatasetPath struct { + comps []string +} func (p DatasetPath) ToString() string { - return strings.Join(p, "/") + return strings.Join(p.comps, "/") } func (p DatasetPath) Empty() bool { - return len(p) == 0 + return len(p.comps) == 0 } -var EmptyDatasetPath DatasetPath = []string{} +func (p *DatasetPath) Extend(extend DatasetPath) { + p.comps = append(p.comps, extend.comps...) +} + +func (p DatasetPath) HasPrefix(prefix DatasetPath) bool { + if len(prefix.comps) > len(p.comps) { + return false + } + for i := range prefix.comps { + if prefix.comps[i] != p.comps[i] { + return false + } + } + return true +} + +func (p *DatasetPath) TrimPrefix(prefix DatasetPath) { + if !p.HasPrefix(prefix) { + return + } + prelen := len(prefix.comps) + newlen := len(p.comps) - prelen + oldcomps := p.comps + p.comps = make([]string, newlen) + for i := 0; i < newlen; i++ { + p.comps[i] = oldcomps[prelen+i] + } + return +} + +func (p *DatasetPath) TrimNPrefixComps(n int) { + if len(p.comps) < n { + n = len(p.comps) + } + if n == 0 { + return + } + p.comps = p.comps[n:] + +} + +func (p DatasetPath) Equal(q DatasetPath) bool { + if len(p.comps) != len(q.comps) { + return false + } + for i := range p.comps { + if p.comps[i] != q.comps[i] { + return false + } + } + return true +} + +func (p DatasetPath) Length() int { + return len(p.comps) +} + +func (p DatasetPath) Copy() (c DatasetPath) { + c.comps = make([]string, len(p.comps)) + copy(c.comps, p.comps) + return +} func NewDatasetPath(s string) (p DatasetPath, err error) { if s == "" { - return EmptyDatasetPath, nil // the empty dataset path + p.comps = make([]string, 0) + return p, nil // the empty dataset path } - const FORBIDDEN = "@#|\t " + const FORBIDDEN = "@#|\t <>*" if strings.ContainsAny(s, FORBIDDEN) { // TODO space may be a bit too restrictive... - return nil, errors.New(fmt.Sprintf("path '%s' contains forbidden characters (any of '%s')", s, FORBIDDEN)) + err = fmt.Errorf("contains forbidden characters (any of '%s')", FORBIDDEN) + return } - return strings.Split(s, "/"), nil + p.comps = strings.Split(s, "/") + if p.comps[len(p.comps)-1] == "" { + err = fmt.Errorf("must not end with a '/'") + return + } + return } func toDatasetPath(s string) DatasetPath { @@ -42,8 +112,6 @@ func toDatasetPath(s string) DatasetPath { return p } -type DatasetFilter func(path DatasetPath) bool - type ZFSError struct { Stderr []byte WaitErr error diff --git a/zfs/zfs_test.go b/zfs/zfs_test.go index 784045f..cf89b40 100644 --- a/zfs/zfs_test.go +++ b/zfs/zfs_test.go @@ -17,3 +17,14 @@ func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) { assert.True(t, ok) assert.Equal(t, "error: this is a mock\n", string(zfsError.Stderr)) } + +func TestDatasetPathTrimNPrefixComps(t *testing.T) { + p, err := NewDatasetPath("foo/bar/a/b") + assert.Nil(t, err) + p.TrimNPrefixComps(2) + assert.True(t, p.Equal(toDatasetPath("a/b"))) + p.TrimNPrefixComps((2)) + assert.True(t, p.Empty()) + p.TrimNPrefixComps((1)) + assert.True(t, p.Empty(), "empty trimming shouldn't do harm") +}