adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework filters & mappings

Browse Source 
config defines a single datastructure that can act both as a Map and as a Filter
(DatasetMapFilter)

Cleanup wildcard syntax along the way (also changes semantics).
This commit is contained in:
Christian Schwarz
2017-08-05 21:15:37 +02:00
parent 3fac6a67df
commit 2ce07c9342
13 changed files with 478 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zfs/datasetpath_visitor.go
+13 -9
View File
@@ -11,20 +11,20 @@ func NewDatasetPathForest() *DatasetPathForest {
}
func (f *DatasetPathForest) Add(p DatasetPath) {
if len(p) <= 0 {
if len(p.comps) <= 0 {
panic("dataset path too short. must have length > 0")
}
// Find its root
var root *datasetPathTree
for _, r := range f.roots {
if r.Add(p) {
if r.Add(p.comps) {
root = r
break
}
}
if root == nil {
root = newDatasetPathTree(p)
root = newDatasetPathTree(p.comps)
f.roots = append(f.roots, root)
}
}
@@ -57,7 +57,7 @@ type datasetPathTree struct {
Children []*datasetPathTree
}
func (t *datasetPathTree) Add(p DatasetPath) bool {
func (t *datasetPathTree) Add(p []string) bool {
if len(p) == 0 {
return true
@@ -88,11 +88,15 @@ func (t *datasetPathTree) Add(p DatasetPath) bool {
}
func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVisitor) {
func (t *datasetPathTree) WalkTopDown(parent []string, visitor DatasetPathsVisitor) {
this := append(parent, t.Component)
visitChildTree := visitor(DatasetPathVisit{this, t.FilledIn})
thisVisit := DatasetPathVisit{
DatasetPath{this},
t.FilledIn,
}
visitChildTree := visitor(thisVisit)
if visitChildTree {
for _, c := range t.Children {
@@ -102,15 +106,15 @@ func (t *datasetPathTree) WalkTopDown(parent DatasetPath, visitor DatasetPathsVi
}
func newDatasetPathTree(initial DatasetPath) (t *datasetPathTree) {
func newDatasetPathTree(initialComps []string) (t *datasetPathTree) {
t = &datasetPathTree{}
var cur *datasetPathTree
cur = t
for i, comp := range initial {
for i, comp := range initialComps {
cur.Component = comp
cur.FilledIn = true
cur.Children = make([]*datasetPathTree, 0, 1)
if i == len(initial)-1 {
if i == len(initialComps)-1 {
cur.FilledIn = false // last component is not filled in
break
}
zfs/datasetpath_visitor_test.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
func TestNewDatasetPathTree(t *testing.T) {
r := newDatasetPathTree(toDatasetPath("pool1/foo/bar"))
r := newDatasetPathTree(toDatasetPath("pool1/foo/bar").comps)
assert.Equal(t, "pool1", r.Component)
assert.True(t, len(r.Children) == 1)
zfs/mapping.go
+10 -153
View File
@@ -1,21 +1,15 @@
package zfs
import (
"bufio"
"errors"
"fmt"
"io"
"os/exec"
)
import "fmt"
type DatasetMapping interface {
Map(source DatasetPath) (target DatasetPath, err error)
type DatasetFilter interface {
Filter(p DatasetPath) (pass bool, err error)
}
func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) {
func ZFSListMapping(filter DatasetFilter) (datasets []DatasetPath, err error) {
if mapping == nil {
panic("mapping must not be nil")
if filter == nil {
panic("filter must not be nil")
}
var lines [][]string
@@ -30,12 +24,11 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error)
return
}
_, mapErr := mapping.Map(path)
if mapErr != nil && mapErr != NoMatchError {
return nil, mapErr
pass, filterErr := filter.Filter(path)
if filterErr != nil {
return nil, fmt.Errorf("error calling filter: %s", filterErr)
}
if mapErr == nil {
if pass {
datasets = append(datasets, path)
}
@@ -43,139 +36,3 @@ func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error)
return
}
type GlobMapping struct {
PrefixPath DatasetPath
TargetRoot DatasetPath
}
var NoMatchError error = errors.New("no match found in mapping")
func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) {
if len(source) < len(m.PrefixPath) {
err = NoMatchError
return
}
target = make([]string, 0, len(source)+len(m.TargetRoot))
target = append(target, m.TargetRoot...)
for si, sc := range source {
target = append(target, sc)
if si < len(m.PrefixPath) {
compsMatch := sc == m.PrefixPath[si]
endOfPrefixPath := si == len(m.PrefixPath)-1 && m.PrefixPath[si] == ""
if !(compsMatch || endOfPrefixPath) {
err = NoMatchError
return
}
continue
}
}
return
}
type ComboMapping struct {
Mappings []DatasetMapping
}
func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) {
for _, sm := range m.Mappings {
target, err = sm.Map(source)
if err == nil {
return target, err
}
}
return nil, NoMatchError
}
type DirectMapping struct {
Source DatasetPath
Target DatasetPath
}
func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) {
if m.Source == nil {
return m.Target, nil
}
if len(m.Source) != len(source) {
return nil, NoMatchError
}
for i, c := range source {
if c != m.Source[i] {
return nil, NoMatchError
}
}
return m.Target, nil
}
type ExecMapping struct {
Name string
Args []string
}
func NewExecMapping(name string, args ...string) (m *ExecMapping) {
m = &ExecMapping{
Name: name,
Args: args,
}
return
}
func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) {
var stdin io.Writer
var stdout io.Reader
cmd := exec.Command(m.Name, m.Args...)
if stdin, err = cmd.StdinPipe(); err != nil {
return
}
if stdout, err = cmd.StdoutPipe(); err != nil {
return
}
resp := bufio.NewScanner(stdout)
if err = cmd.Start(); err != nil {
return
}
go func() {
err := cmd.Wait()
if err != nil {
panic(err)
// fmt.Printf("error: %v\n", err) // TODO
}
}()
if _, err = io.WriteString(stdin, source.ToString()+"\n"); err != nil {
return
}
if !resp.Scan() {
err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err()))
return
}
t := resp.Text()
switch {
case t == "NOMAP":
return nil, NoMatchError
}
target = toDatasetPath(t) // TODO discover garbage?
return
}
zfs/mapping_test.go
-134
View File
@@ -1,134 +0,0 @@
package zfs
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestGlobMappingPrefixWildcard(t *testing.T) {
m := GlobMapping{
PrefixPath: toDatasetPath("a/b/c/"), // TRAILING empty component!
TargetRoot: toDatasetPath("x/y"),
}
t.Logf("PrefixPath: %#v", m.PrefixPath)
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("a/b/c"))
assert.NotNil(t, err)
r, err = m.Map(toDatasetPath("a/b/c/d"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("x/y/a/b/c/d"), r)
}
func TestGlobMapping(t *testing.T) {
m := GlobMapping{
PrefixPath: toDatasetPath("tank/usr/home"),
TargetRoot: toDatasetPath("backups/share1"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("tank/usr/home"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r)
r, err = m.Map(toDatasetPath("zroot"))
assert.Equal(t, NoMatchError, err, "prefix-only match is an error")
r, err = m.Map(toDatasetPath("zroot/notmapped"))
assert.Equal(t, NoMatchError, err, "non-prefix is an error")
}
func TestGlobMappingWildcard(t *testing.T) {
m := GlobMapping{
PrefixPath: EmptyDatasetPath,
TargetRoot: toDatasetPath("backups/share1"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("tank/usr/home"))
assert.Equal(t, toDatasetPath("backups/share1/tank/usr/home"), r)
assert.NoError(t, err)
}
func TestComboMapping(t *testing.T) {
m1 := GlobMapping{
PrefixPath: toDatasetPath("a/b"),
TargetRoot: toDatasetPath("c/d"),
}
m2 := GlobMapping{
PrefixPath: toDatasetPath("a/x"),
TargetRoot: toDatasetPath("c/y"),
}
c := ComboMapping{
Mappings: []DatasetMapping{m1, m2},
}
var r DatasetPath
var err error
p := toDatasetPath("a/b/q")
r, err = m2.Map(p)
assert.Equal(t, NoMatchError, err)
r, err = c.Map(p)
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("c/d/a/b/q"), r)
}
func TestDirectMapping(t *testing.T) {
m := DirectMapping{
Source: toDatasetPath("a/b/c"),
Target: toDatasetPath("x/y/z"),
}
var r DatasetPath
var err error
r, err = m.Map(toDatasetPath("a/b/c"))
assert.Nil(t, err)
assert.Equal(t, m.Target, r)
r, err = m.Map(toDatasetPath("not/matching"))
assert.Equal(t, NoMatchError, err)
r, err = m.Map(toDatasetPath("a/b"))
assert.Equal(t, NoMatchError, err)
}
func TestExecMapping(t *testing.T) {
var err error
var m DatasetMapping
m = NewExecMapping("test_helpers/exec_mapping_good.sh", "nostop")
assert.NoError(t, err)
var p DatasetPath
p, err = m.Map(toDatasetPath("nomap/foobar"))
assert.Equal(t, NoMatchError, err)
p, err = m.Map(toDatasetPath("willmap/something"))
assert.Nil(t, err)
assert.Equal(t, toDatasetPath("didmap/willmap/something"), p)
}
zfs/zfs.go
+78 -10
View File
@@ -11,27 +11,97 @@ import (
"strings"
)
type DatasetPath []string
type DatasetPath struct {
comps []string
}
func (p DatasetPath) ToString() string {
return strings.Join(p, "/")
return strings.Join(p.comps, "/")
}
func (p DatasetPath) Empty() bool {
return len(p) == 0
return len(p.comps) == 0
}
var EmptyDatasetPath DatasetPath = []string{}
func (p *DatasetPath) Extend(extend DatasetPath) {
p.comps = append(p.comps, extend.comps...)
}
func (p DatasetPath) HasPrefix(prefix DatasetPath) bool {
if len(prefix.comps) > len(p.comps) {
return false
}
for i := range prefix.comps {
if prefix.comps[i] != p.comps[i] {
return false
}
}
return true
}
func (p *DatasetPath) TrimPrefix(prefix DatasetPath) {
if !p.HasPrefix(prefix) {
return
}
prelen := len(prefix.comps)
newlen := len(p.comps) - prelen
oldcomps := p.comps
p.comps = make([]string, newlen)
for i := 0; i < newlen; i++ {
p.comps[i] = oldcomps[prelen+i]
}
return
}
func (p *DatasetPath) TrimNPrefixComps(n int) {
if len(p.comps) < n {
n = len(p.comps)
}
if n == 0 {
return
}
p.comps = p.comps[n:]
}
func (p DatasetPath) Equal(q DatasetPath) bool {
if len(p.comps) != len(q.comps) {
return false
}
for i := range p.comps {
if p.comps[i] != q.comps[i] {
return false
}
}
return true
}
func (p DatasetPath) Length() int {
return len(p.comps)
}
func (p DatasetPath) Copy() (c DatasetPath) {
c.comps = make([]string, len(p.comps))
copy(c.comps, p.comps)
return
}
func NewDatasetPath(s string) (p DatasetPath, err error) {
if s == "" {
return EmptyDatasetPath, nil // the empty dataset path
p.comps = make([]string, 0)
return p, nil // the empty dataset path
}
const FORBIDDEN = "@#|\t "
const FORBIDDEN = "@#|\t <>*"
if strings.ContainsAny(s, FORBIDDEN) { // TODO space may be a bit too restrictive...
return nil, errors.New(fmt.Sprintf("path '%s' contains forbidden characters (any of '%s')", s, FORBIDDEN))
err = fmt.Errorf("contains forbidden characters (any of '%s')", FORBIDDEN)
return
}
return strings.Split(s, "/"), nil
p.comps = strings.Split(s, "/")
if p.comps[len(p.comps)-1] == "" {
err = fmt.Errorf("must not end with a '/'")
return
}
return
}
func toDatasetPath(s string) DatasetPath {
@@ -42,8 +112,6 @@ func toDatasetPath(s string) DatasetPath {
return p
}
type DatasetFilter func(path DatasetPath) bool
type ZFSError struct {
Stderr []byte
WaitErr error
zfs/zfs_test.go
+11
View File
@@ -17,3 +17,14 @@ func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, "error: this is a mock\n", string(zfsError.Stderr))
}
func TestDatasetPathTrimNPrefixComps(t *testing.T) {
p, err := NewDatasetPath("foo/bar/a/b")
assert.Nil(t, err)
p.TrimNPrefixComps(2)
assert.True(t, p.Equal(toDatasetPath("a/b")))
p.TrimNPrefixComps((2))
assert.True(t, p.Empty())
p.TrimNPrefixComps((1))
assert.True(t, p.Empty(), "empty trimming shouldn't do harm")
}