adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework filters & mappings

Browse Source 
config defines a single datastructure that can act both as a Map and as a Filter
(DatasetMapFilter)

Cleanup wildcard syntax along the way (also changes semantics).
This commit is contained in:
Christian Schwarz
2017-08-05 21:15:37 +02:00
parent 3fac6a67df
commit 2ce07c9342
13 changed files with 478 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/config.go
+67 -91
View File
@@ -52,26 +52,21 @@ type SSHTransport struct {
type Push struct {
JobName string // for use with jobrun package
To *Remote
Filter zfs.DatasetMapping
Filter zfs.DatasetFilter
InitialReplPolicy rpc.InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type Pull struct {
JobName string // for use with jobrun package
From *Remote
Mapping zfs.DatasetMapping
Mapping DatasetMapFilter
InitialReplPolicy rpc.InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type ClientMapping struct {
From string
Mapping zfs.DatasetMapping
}
type Prune struct {
JobName string // for use with jobrun package
DatasetFilter zfs.DatasetMapping
DatasetFilter zfs.DatasetFilter
SnapshotFilter zfs.FilesystemVersionFilter
RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
}
@@ -80,17 +75,17 @@ type Autosnap struct {
JobName string // for use with jobrun package
Prefix string
Interval jobrun.RepeatStrategy
DatasetFilter zfs.DatasetMapping
DatasetFilter zfs.DatasetFilter
}
type Config struct {
Remotes map[string]*Remote
Pushs map[string]*Push // job name -> job
Pulls map[string]*Pull // job name -> job
Sinks map[string]*ClientMapping // client identity -> mapping
PullACLs map[string]*ClientMapping // client identity -> mapping
Prunes map[string]*Prune // job name -> job
Autosnaps map[string]*Autosnap // job name -> job
Pushs map[string]*Push // job name -> job
Pulls map[string]*Pull // job name -> job
Sinks map[string]DatasetMapFilter // client identity -> mapping
PullACLs map[string]DatasetMapFilter // client identity -> filter
Prunes map[string]*Prune // job name -> job
Autosnaps map[string]*Autosnap // job name -> job
}
func ParseConfig(path string) (config Config, err error) {
@@ -129,10 +124,10 @@ func parseMain(root map[string]interface{}) (c Config, err error) {
if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
return
}
if c.Sinks, err = parseClientMappings(root["sinks"]); err != nil {
if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
return
}
if c.PullACLs, err = parseClientMappings(root["pull_acls"]); err != nil {
if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
return
}
if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
@@ -239,7 +234,7 @@ func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error)
if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
return
}
if push.Filter, err = parseComboMapping(e.Filter); err != nil {
if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
return
}
@@ -298,7 +293,7 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error)
if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
return
}
if pull.Mapping, err = parseComboMapping(e.Mapping); err != nil {
if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
return
}
if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
@@ -365,97 +360,78 @@ func expectList(v interface{}) (asList []interface{}, err error) {
return
}
func parseClientMappings(v interface{}) (cm map[string]*ClientMapping, err error) {
var asMap map[string]interface{}
func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
var asMap map[string]map[string]interface{}
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
cm = make(map[string]*ClientMapping, len(asMap))
m = make(map[string]DatasetMapFilter, len(asMap))
for identity, e := range asMap {
var m *ClientMapping
if m, err = parseClientMapping(e, identity); err != nil {
for identity, entry := range asMap {
parseSink := func() (mapping DatasetMapFilter, err error) {
mappingMap, ok := entry["mapping"]
if !ok {
err = fmt.Errorf("no mapping specified")
return
}
mapping, err = parseDatasetMapFilter(mappingMap, false)
return
}
cm[identity] = m
mapping, sinkErr := parseSink()
if sinkErr != nil {
err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
return
}
m[identity] = mapping
}
return
}
func parseClientMapping(v interface{}, identity string) (s *ClientMapping, err error) {
t := struct {
Mapping map[string]string
}{}
if err = mapstructure.Decode(v, &t); err != nil {
func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
var asMap map[string]map[string]interface{}
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
s = &ClientMapping{
From: identity,
m = make(map[string]DatasetMapFilter, len(asMap))
for identity, entry := range asMap {
parsePullACL := func() (filter DatasetMapFilter, err error) {
filterMap, ok := entry["filter"]
if !ok {
err = fmt.Errorf("no filter specified")
return
}
filter, err = parseDatasetMapFilter(filterMap, true)
return
}
filter, filterErr := parsePullACL()
if filterErr != nil {
err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
return
}
m[identity] = filter
}
s.Mapping, err = parseComboMapping(t.Mapping)
return
}
func parseComboMapping(m map[string]string) (c zfs.ComboMapping, err error) {
c.Mappings = make([]zfs.DatasetMapping, 0, len(m))
for lhs, rhs := range m {
if lhs == "*" && strings.HasPrefix(rhs, "!") {
m := zfs.ExecMapping{}
fields := strings.Fields(strings.TrimPrefix(rhs, "!"))
if len(fields) < 1 {
err = errors.New("ExecMapping without acceptor path")
return
}
m.Name = fields[0]
m.Args = fields[1:]
c.Mappings = append(c.Mappings, m)
} else if strings.HasSuffix(lhs, "*") {
m := zfs.GlobMapping{}
m.PrefixPath, err = zfs.NewDatasetPath(strings.TrimSuffix(lhs, "*"))
if err != nil {
return
}
if m.TargetRoot, err = zfs.NewDatasetPath(rhs); err != nil {
return
}
c.Mappings = append(c.Mappings, m)
} else {
m := zfs.DirectMapping{}
if lhs == "|" {
m.Source = nil
} else {
if m.Source, err = zfs.NewDatasetPath(lhs); err != nil {
return
}
}
if m.Target, err = zfs.NewDatasetPath(rhs); err != nil {
return
}
c.Mappings = append(c.Mappings, m)
}
func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
var m map[string]string
if err = mapstructure.Decode(mi, &m); err != nil {
err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
return
}
f = NewDatasetMapFilter(len(m), filterOnly)
for pathPattern, mapping := range m {
if err = f.Add(pathPattern, mapping); err != nil {
err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
return
}
}
return
}
func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
@@ -551,7 +527,7 @@ func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error)
prune.RetentionPolicy = NewRetentionGrid(intervals)
// Parse filters
if prune.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil {
if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
err = fmt.Errorf("cannot parse dataset filter: %s", err)
return
}
@@ -746,7 +722,7 @@ func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
err = fmt.Errorf("dataset_filter not specified")
return
}
if a.DatasetFilter, err = parseComboMapping(i.DatasetFilter); err != nil {
if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
err = fmt.Errorf("cannot parse dataset filter: %s", err)
}
cmd/config_mapfilter.go
+164
View File
@@ -0,0 +1,164 @@
package cmd
import (
"errors"
"fmt"
"strings"
"github.com/zrepl/zrepl/zfs"
)
type DatasetMapFilter struct {
entries []datasetMapFilterEntry
// if set, only valid filter entries can be added using Add()
// and Map() will always return an error
filterOnly bool
}
type datasetMapFilterEntry struct {
path zfs.DatasetPath
// the mapping. since this datastructure acts as both mapping and filter
// we have to convert it to the desired rep dynamically
mapping string
subtreeMatch bool
}
var NoMatchError error = errors.New("no match found in mapping")
func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
return DatasetMapFilter{
entries: make([]datasetMapFilterEntry, 0, capacity),
}
}
func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
if m.filterOnly {
if _, err = parseDatasetFilterResult(mapping); err != nil {
return
}
}
// assert path glob adheres to spec
const SUBTREE_PATTERN string = "<"
patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
switch {
case patternCount > 1:
case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
return
}
var path zfs.DatasetPath
pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
path, err = zfs.NewDatasetPath(pathStr)
if err != nil {
err = fmt.Errorf("pattern is not a dataset path: %s", err)
return
}
entry := datasetMapFilterEntry{
path: path,
mapping: mapping,
subtreeMatch: patternCount > 0,
}
m.entries = append(m.entries, entry)
return
}
// find the most specific prefix mapping we have
//
// longer prefix wins over shorter prefix, direct wins over glob
func (m DatasetMapFilter) mostSpecificPrefixMapping(path zfs.DatasetPath) (idx int, found bool) {
lcp, lcp_entry_idx := -1, -1
direct_idx := -1
for e := range m.entries {
entry := m.entries[e]
ep := m.entries[e].path
lep := ep.Length()
switch {
case !entry.subtreeMatch && ep.Equal(path):
direct_idx = e
continue
case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
lcp = lep
lcp_entry_idx = e
default:
continue
}
}
if lcp_entry_idx >= 0 || direct_idx >= 0 {
found = true
switch {
case direct_idx >= 0:
idx = direct_idx
case lcp_entry_idx >= 0:
idx = lcp_entry_idx
}
}
return
}
func (m DatasetMapFilter) Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error) {
if m.filterOnly {
err = fmt.Errorf("using a filter for mapping simply does not work")
return
}
mi, hasMapping := m.mostSpecificPrefixMapping(source)
if !hasMapping {
err = NoMatchError
return
}
me := m.entries[mi]
target, err = zfs.NewDatasetPath(me.mapping)
if err != nil {
err = fmt.Errorf("mapping target is not a dataset path: %s", err)
return
}
if m.entries[mi].subtreeMatch {
// strip common prefix
extendComps := source.Copy()
if me.path.Empty() {
// special case: trying to map the root => strip first component
extendComps.TrimNPrefixComps(1)
} else {
extendComps.TrimPrefix(me.path)
}
target.Extend(extendComps)
}
return
}
func (m DatasetMapFilter) Filter(p zfs.DatasetPath) (pass bool, err error) {
mi, hasMapping := m.mostSpecificPrefixMapping(p)
if !hasMapping {
pass = false
return
}
me := m.entries[mi]
pass, err = parseDatasetFilterResult(me.mapping)
return
}
// Parse a dataset filter result
func parseDatasetFilterResult(result string) (pass bool, err error) {
l := strings.ToLower(result)
switch strings.ToLower(l) {
case "ok":
pass = true
return
case "omit":
return
default:
err = fmt.Errorf("'%s' is not a valid filter result", result)
return
}
return
}
cmd/config_test.go
+78 -2
View File
@@ -1,10 +1,12 @@
package cmd
import (
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/util"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/zrepl/zrepl/util"
"github.com/zrepl/zrepl/zfs"
)
func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
@@ -45,3 +47,77 @@ func TestParseRetentionGridStringParsing(t *testing.T) {
assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
}
func TestDatasetMapFilter(t *testing.T) {
expectMapping := func(m map[string]string, from, to string) {
dmf, err := parseDatasetMapFilter(m, false)
if err != nil {
t.Logf("expect test map to be valid: %s", err)
t.FailNow()
}
fromPath, err := zfs.NewDatasetPath(from)
if err != nil {
t.Logf("expect test from path to be valid: %s", err)
t.FailNow()
}
toPath, err := zfs.NewDatasetPath(to)
if err != nil {
t.Logf("expect test to path to be valid: %s", err)
t.FailNow()
}
res, err := dmf.Map(fromPath)
t.Logf("%s => %s", fromPath.ToString(), res.ToString())
assert.Nil(t, err)
assert.True(t, res.Equal(toPath))
}
expectFilter := func(m map[string]string, path string, pass bool) {
dmf, err := parseDatasetMapFilter(m, true)
if err != nil {
t.Logf("expect test filter to be valid: %s", err)
t.FailNow()
}
p, err := zfs.NewDatasetPath(path)
if err != nil {
t.Logf("expect test path to be valid: %s", err)
t.FailNow()
}
res, err := dmf.Filter(p)
assert.Nil(t, err)
assert.Equal(t, pass, res)
}
map1 := map[string]string{
"a/b/c<": "root1",
"a/b<": "root2",
"<": "root3/b/c",
"q<": "root4/1/2",
}
expectMapping(map1, "a/b/c", "root1")
expectMapping(map1, "a/b/c/d", "root1/d")
expectMapping(map1, "a/b/e", "root2/e")
expectMapping(map1, "a/b", "root2")
expectMapping(map1, "x", "root3/b/c")
expectMapping(map1, "x/y", "root3/b/c/y")
expectMapping(map1, "q", "root4/1/2")
expectMapping(map1, "q/r", "root4/1/2/r")
filter1 := map[string]string{
"<": "omit",
"a<": "ok",
"a/b<": "omit",
}
expectFilter(filter1, "b", false)
expectFilter(filter1, "a", true)
expectFilter(filter1, "a/d", true)
expectFilter(filter1, "a/b", false)
expectFilter(filter1, "a/b/c", false)
filter2 := map[string]string{}
expectFilter(filter2, "foo", false) // default to omit
}
cmd/handler.go
+7 -3
View File
@@ -7,10 +7,14 @@ import (
"io"
)
type DatasetMapping interface {
Map(source zfs.DatasetPath) (target zfs.DatasetPath, err error)
}
type Handler struct {
Logger Logger
PullACL zfs.DatasetMapping
SinkMappingFunc func(clientIdentity string) (mapping zfs.DatasetMapping, err error)
PullACL zfs.DatasetFilter
SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
}
func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []zfs.DatasetPath, err error) {
@@ -90,7 +94,7 @@ func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string,
h.Logger.Printf("handling PullMeRequest: %#v", r)
var sinkMapping zfs.DatasetMapping
var sinkMapping DatasetMapping
sinkMapping, err = h.SinkMappingFunc(clientIdentity)
if err != nil {
h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
cmd/replication.go
+16 -4
View File
@@ -141,12 +141,24 @@ func cmdRun(cmd *cobra.Command, args []string) {
}
type localPullACL struct{}
func (a localPullACL) Filter(p zfs.DatasetPath) (pass bool, err error) {
return true, nil
}
func jobPull(pull *Pull, log jobrun.Logger) (err error) {
if lt, ok := pull.From.Transport.(LocalTransport); ok {
lt.SetHandler(Handler{
Logger: log,
PullACL: pull.Mapping,
Logger: log,
// Allow access to any dataset since we control what mapping
// is passed to the pull routine.
// All local datasets will be passed to its Map() function,
// but only those for which a mapping exists will actually be pulled.
// We can pay this small performance penalty for now.
PullACL: localPullACL{},
})
pull.From.Transport = lt
log.Printf("fixing up local transport: %#v", pull.From.Transport)
@@ -228,7 +240,7 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Durat
type PullContext struct {
Remote rpc.RPCRequester
Log Logger
Mapping zfs.DatasetMapping
Mapping DatasetMapping
InitialReplPolicy rpc.InitialReplPolicy
}
@@ -258,7 +270,7 @@ func doPull(pull PullContext) (err error) {
var localFs zfs.DatasetPath
localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
if err != nil {
if err != zfs.NoMatchError {
if err != NoMatchError {
log.Printf("error mapping %s: %#v\n", remoteFilesystems[fs], err)
return err
}
cmd/sampleconf/zrepl.yml
+25 -43
View File
@@ -12,8 +12,9 @@ pushs:
offsite:
to: offsite_backups
filter: {
"tank/var/db*":"ok",
"tank/usr/home*":"ok"
# like in pull_acls
"tank/var/db<": ok,
"tank/usr/home<": ok,
}
pulls:
@@ -36,59 +37,39 @@ pulls:
sinks:
# direct mapping
# 1:1 mapping of remote dataset to local dataset
# We will reject a push request which contains > 0 datasets that do not
# match a mapping
db1:
mapping: {
# direct mapping
"ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
}
# "|" non-recursive wildcard
# the remote must present excatly one dataset, mapped to the rhs
cdn_master:
mapping: {
"|":"tank/srv/cdn" # NOTE: | is currently an invalid character for a ZFS dataset
}
# "*" recursive wildcard
# the remote may present an arbitrary set of marks a recursive wildcard, i.e. map all remotes to a tree under rhs
mirror1:
mapping: {
"tank/foo/bar*":"zroot/backups/mirror1" # NOTE: * is currently an invalid character for a ZFS dataset
# "<" subtree wildcard matches the dataset left of < and all its children
"tank/foo/bar<":"zroot/backups/mirror1"
}
# "*":"!..." acceptor script
# shell out to an accceptor that receives the remote's offered datasets
# on stdin and, foreach line of this input, returns the corresponding
# local dataset (same order) or '!<space>optional reason' on stdout
# If the acceptor scripts exits with non-zero status code, the remote's
# request will be rejected
complex_host:
mapping: { #
"*":"!/path/to/acceptor" # we could just wire the path to the acceptor directly to the mapping
# but let's stick with the same type for the mapping field for now'
# NOTE: * and ! are currently invalid characters for a ZFS dataset
mirror2:
# more specific path patterns win over less specific ones
# direct mappings win over subtree wildcards
# detailed rule precedence: check unit tests & docs for exact behavior
# TODO subcommand to test a mapping & filter
mapping: {
"tank<": "zroot/backups/mirror1/tank1",
"tank/cdn/root<": "storage/cdn/root",
"tank/legacydb": "legacypool/backups/legacydb",
}
# Mixing the rules
# Mixing should be possible if there is a defined precedence (direct before *)
# and non-recursive wildcards are not allowed in multi-entry mapping objects
special_snowflake:
mapping: { # an explicit mapping mixed with a recursive wildcard
"sun/usr/home": backups/special_snowflake/homedirs,
"sun/var/db": backups/special_snowflake/database,
"*": backups/special_snowflake/remainingbackup
# NOTE: ^ alignment, should be possible, looks nicer
}
pull_acls:
# same synatx as in sinks, but the returned mapping does not matter
office_backup:
mapping: {
"tank/usr/home":"notnull"
filter: {
# valid filter results (right hand side): ok, omit
# default is to omit
# rule precedence is same as for mappings
"tank<": omit,
"tank/usr/home": ok,
}
@@ -98,7 +79,8 @@ prune:
policy: grid
grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
dataset_filter: {
"tank/backups/*": ok
"tank/backups/legacyscript<": omit,
"tank/backups<": ok,
}
snapshot_filter: {
prefix: zrepl_
@@ -108,7 +90,7 @@ prune:
policy: grid
grid: 1x1m(keep=all)
dataset_filter: {
"pool1*": ok
"pool1<": ok
}
snapshot_filter: {
prefix: zrepl_hfbak_
@@ -120,7 +102,7 @@ autosnap:
prefix: zrepl_hfbak_
interval: 1s
dataset_filter: {
"pool1*": ok
"pool1<": ok
}
# prune: hfbak_prune
# future versions may inline the retention policy here, but for now,
cmd/stdinserver.go
+8 -9
View File
@@ -5,7 +5,6 @@ import (
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/sshbytestream"
"github.com/zrepl/zrepl/zfs"
"io"
golog "log"
"os"
@@ -37,8 +36,8 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
}
identity := args[0]
pullACL := conf.PullACLs[identity]
if pullACL == nil {
pullACL, ok := conf.PullACLs[identity]
if !ok {
err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
return
}
@@ -48,19 +47,19 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
return
}
sinkMapping := func(identity string) (m zfs.DatasetMapping, err error) {
sink := conf.Sinks[identity]
if sink == nil {
return nil, fmt.Errorf("could not find sink for dataset")
sinkMapping := func(identity string) (m DatasetMapping, err error) {
sink, ok := conf.Sinks[identity]
if !ok {
return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
}
return sink.Mapping, nil
return sink, nil
}
sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
handler := Handler{
Logger: sinkLogger,
SinkMappingFunc: sinkMapping,
PullACL: pullACL.Mapping,
PullACL: pullACL,
}
if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {