move retentiongrid to own package

2018-08-26 19:20:08 +02:00
parent cd9a428841
commit add1b69809
7 changed files with 355 additions and 310 deletions
@@ -0,0 +1,156 @@
+package retentiongrid
+
+import (
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/cmd/config"
+	"github.com/zrepl/zrepl/zfs"
+	"math"
+	"sort"
+	"strconv"
+	"time"
+)
+
+type GridPrunePolicy struct {
+	retentionGrid *retentionGrid
+	keepBookmarks int
+}
+
+const GridPrunePolicyMaxBookmarksKeepAll = -1
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+// Prune filters snapshots with the retention grid.
+// Bookmarks are deleted such that keepBookmarks are kept in the end.
+// The oldest bookmarks are removed first.
+func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+	skeep, sremove := p.pruneSnapshots(versions)
+	keep, remove = p.pruneBookmarks(skeep)
+	remove = append(remove, sremove...)
+	return keep, remove, nil
+}
+
+func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
+
+	// Build adaptors for retention grid
+	keep = []zfs.FilesystemVersion{}
+	adaptors := make([]RetentionGridEntry, 0)
+	for fsv := range versions {
+		if versions[fsv].Type != zfs.Snapshot {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
+	}
+
+	sort.SliceStable(adaptors, func(i, j int) bool {
+		return adaptors[i].LessThan(adaptors[j])
+	})
+	now := adaptors[len(adaptors)-1].Date()
+
+	// Evaluate retention grid
+	keepa, removea := p.retentionGrid.FitEntries(now, adaptors)
+
+	// Revert adaptors
+	for i := range keepa {
+		keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
+	}
+	remove = make([]zfs.FilesystemVersion, len(removea))
+	for i := range removea {
+		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
+	}
+	return
+
+}
+
+func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
+
+	if p.keepBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
+		return versions, []zfs.FilesystemVersion{}
+	}
+
+	keep = []zfs.FilesystemVersion{}
+	bookmarks := make([]zfs.FilesystemVersion, 0)
+	for fsv := range versions {
+		if versions[fsv].Type != zfs.Bookmark {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		bookmarks = append(bookmarks, versions[fsv])
+	}
+
+	if len(bookmarks) == 0 {
+		return keep, []zfs.FilesystemVersion{}
+	}
+	if len(bookmarks) < p.keepBookmarks {
+		keep = append(keep, bookmarks...)
+		return keep, []zfs.FilesystemVersion{}
+	}
+
+	// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
+	sort.SliceStable(bookmarks, func(i, j int) bool {
+		return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
+	})
+
+	keep = append(keep, bookmarks[:p.keepBookmarks]...)
+	remove = bookmarks[p.keepBookmarks:]
+
+	return keep, remove
+}
+
+func ParseGridPrunePolicy(in config.PruneGrid, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
+
+	const KeepBookmarksAllString = "all"
+
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range in.Grid {
+
+		if in.Grid[i].Length() < lastDuration {
+			// If all intervals before were keep=all, this is ok
+			allPrevKeepCountAll := true
+			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
+				allPrevKeepCountAll = in.Grid[j].KeepCount() == config.RetentionGridKeepCountAll
+			}
+			if allPrevKeepCountAll {
+				goto isMonotonicIncrease
+			}
+			err = errors.New("retention grid interval length must be monotonically increasing")
+			return
+		}
+	isMonotonicIncrease:
+		lastDuration = in.Grid[i].Length()
+
+	}
+
+	// Parse keepBookmarks
+	keepBookmarks := 0
+	if in.KeepBookmarks == KeepBookmarksAllString || (in.KeepBookmarks == "" && !willSeeBookmarks) {
+		keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
+	} else {
+		i, err := strconv.ParseInt(in.KeepBookmarks, 10, 32)
+		if err != nil || i <= 0 || i > math.MaxInt32 {
+			return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
+		}
+		keepBookmarks = int(i)
+	}
+
+	retentionIntervals := make([]RetentionInterval, len(in.Grid))
+	for i := range in.Grid {
+		retentionIntervals[i] = &in.Grid[i]
+	}
+
+	return &GridPrunePolicy{
+		newRetentionGrid(retentionIntervals),
+		keepBookmarks,
+	}, nil
+}
@@ -0,0 +1,100 @@
+package retentiongrid
+
+import (
+	"sort"
+	"time"
+)
+
+type RetentionInterval interface {
+	Length() time.Duration
+	KeepCount() int
+}
+
+const RetentionGridKeepCountAll int = -1
+
+type retentionGrid struct {
+	intervals []RetentionInterval
+}
+
+//A point inside the grid, i.e. a thing the grid can decide to remove
+type RetentionGridEntry interface {
+	Date() time.Time
+	LessThan(b RetentionGridEntry) bool
+}
+
+func dateInInterval(date, startDateInterval time.Time, i RetentionInterval) bool {
+	return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length()))
+}
+
+func newRetentionGrid(l []RetentionInterval) *retentionGrid {
+	// TODO Maybe check for ascending interval lengths here, although the algorithm
+	// 		itself doesn't care about that.
+	return &retentionGrid{l}
+}
+
+// Partition a list of RetentionGridEntries into the retentionGrid,
+// relative to a given start date `now`.
+//
+// The `keepCount` oldest entries per `RetentionInterval` are kept (`keep`),
+// the others are removed (`remove`).
+//
+// Entries that are younger than `now` are always kept.
+// Those that are older than the earliest beginning of an interval are removed.
+func (g retentionGrid) FitEntries(now time.Time, entries []RetentionGridEntry) (keep, remove []RetentionGridEntry) {
+
+	type bucket struct {
+		entries []RetentionGridEntry
+	}
+	buckets := make([]bucket, len(g.intervals))
+
+	keep = make([]RetentionGridEntry, 0)
+	remove = make([]RetentionGridEntry, 0)
+
+	oldestIntervalStart := now
+	for i := range g.intervals {
+		oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length())
+	}
+
+	for ei := 0; ei < len(entries); ei++ {
+		e := entries[ei]
+
+		date := e.Date()
+
+		if date == now || date.After(now) {
+			keep = append(keep, e)
+			continue
+		} else if date.Before(oldestIntervalStart) {
+			remove = append(remove, e)
+			continue
+		}
+
+		iStartTime := now
+		for i := 0; i < len(g.intervals); i++ {
+			iStartTime = iStartTime.Add(-g.intervals[i].Length())
+			if date == iStartTime || dateInInterval(date, iStartTime, g.intervals[i]) {
+				buckets[i].entries = append(buckets[i].entries, e)
+			}
+		}
+	}
+
+	for bi, b := range buckets {
+
+		interval := g.intervals[bi]
+
+		sort.SliceStable(b.entries, func(i, j int) bool {
+			return b.entries[i].LessThan((b.entries[j]))
+		})
+
+		i := 0
+		for ; (interval.KeepCount() == RetentionGridKeepCountAll || i < interval.KeepCount()) && i < len(b.entries); i++ {
+			keep = append(keep, b.entries[i])
+		}
+		for ; i < len(b.entries); i++ {
+			remove = append(remove, b.entries[i])
+		}
+
+	}
+
+	return
+
+}
@@ -0,0 +1,160 @@
+package retentiongrid
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+type retentionIntervalStub struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *retentionIntervalStub) Length() time.Duration {
+	return i.length
+}
+
+func (i *retentionIntervalStub) KeepCount() int {
+	return i.keepCount
+}
+
+func retentionGridFromString(gs string) (g *retentionGrid) {
+	intervals := strings.Split(gs, "|")
+	g = &retentionGrid{
+		intervals: make([]RetentionInterval, len(intervals)),
+	}
+	for idx, i := range intervals {
+		comps := strings.SplitN(i, ",", 2)
+		var durationStr, numSnapsStr string
+		durationStr = comps[0]
+		if len(comps) == 1 {
+			numSnapsStr = "1"
+		} else {
+			numSnapsStr = comps[1]
+		}
+
+		var err error
+		var interval retentionIntervalStub
+
+		if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil {
+			panic(err)
+		}
+		if interval.length, err = time.ParseDuration(durationStr); err != nil {
+			panic(err)
+		}
+
+		g.intervals[idx] = &interval
+	}
+	return
+}
+
+type dummySnap struct {
+	Name       string
+	ShouldKeep bool
+	date       time.Time
+}
+
+func (ds dummySnap) Date() time.Time {
+	return ds.date
+}
+
+func (ds dummySnap) LessThan(b RetentionGridEntry) bool {
+	return ds.date.Before(b.(dummySnap).date) // don't have a txg here
+}
+
+func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []RetentionGridEntry) {
+
+	snapDescr := func(d dummySnap) string {
+		return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now))
+	}
+
+	t.Logf("keep list:\n")
+	for k := range keep {
+		t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
+	}
+	t.Logf("remove list:\n")
+	for k := range remove {
+		t.Logf("\t%s\n", snapDescr(remove[k].(dummySnap)))
+	}
+
+	t.Logf("\n\n")
+
+	for _, s := range input {
+		d := s.(dummySnap)
+		descr := snapDescr(d)
+		t.Logf("testing %s\n", descr)
+		if d.ShouldKeep {
+			assert.Contains(t, keep, d, "expecting %s to be kept", descr)
+		} else {
+			assert.Contains(t, remove, d, "expecting %s to be removed", descr)
+		}
+	}
+
+	t.Logf("resulting list:\n")
+	for k := range keep {
+		t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
+	}
+}
+
+func TestRetentionGridFitEntriesEmptyInput(t *testing.T) {
+	g := retentionGridFromString("10m|10m|10m|1h")
+	keep, remove := g.FitEntries(time.Now(), []RetentionGridEntry{})
+	assert.Empty(t, keep)
+	assert.Empty(t, remove)
+}
+
+func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) {
+
+	// Intervals are (duration], i.e. 10min is in the first interval, not in the second
+
+	g := retentionGridFromString("10m|10m|10m")
+
+	t.Logf("%#v\n", g)
+
+	now := time.Unix(0, 0)
+
+	snaps := []RetentionGridEntry{
+		dummySnap{"0", true, now.Add(1 * time.Minute)},    // before now
+		dummySnap{"1", true, now},                         // before now
+		dummySnap{"2", true, now.Add(-10 * time.Minute)},  // 1st interval
+		dummySnap{"3", true, now.Add(-20 * time.Minute)},  // 2nd interval
+		dummySnap{"4", true, now.Add(-30 * time.Minute)},  // 3rd interval
+		dummySnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval
+	}
+
+	keep, remove := g.FitEntries(now, snaps)
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+
+}
+
+func TestRetentionGridFitEntries(t *testing.T) {
+
+	g := retentionGridFromString("10m,-1|10m|10m,2|1h")
+
+	t.Logf("%#v\n", g)
+
+	now := time.Unix(0, 0)
+
+	snaps := []RetentionGridEntry{
+		dummySnap{"1", true, now.Add(3 * time.Minute)},   // pre-now must always be kept
+		dummySnap{"b1", true, now.Add(-6 * time.Minute)}, // 1st interval allows unlimited entries
+		dummySnap{"b3", true, now.Add(-8 * time.Minute)}, // 1st interval allows unlimited entries
+		dummySnap{"b2", true, now.Add(-9 * time.Minute)}, // 1st interval allows unlimited entries
+		dummySnap{"a", false, now.Add(-11 * time.Minute)},
+		dummySnap{"c", true, now.Add(-19 * time.Minute)}, // 2nd interval allows 1 entry
+		dummySnap{"foo", false, now.Add(-25 * time.Minute)},
+		dummySnap{"bar", true, now.Add(-26 * time.Minute)}, // 3rd interval allows 2 entries
+		dummySnap{"border", true, now.Add(-30 * time.Minute)},
+		dummySnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
+		dummySnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)}, // before earliest interval must always be deleted
+		dummySnap{"f", false, now.Add(-2 * time.Hour)},
+	}
+	keep, remove := g.FitEntries(now, snaps)
+
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+
+}