Pruner: parallel pruning

refs #62

cmd/autosnap.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/zrepl/zrepl/zfs"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -165,33 +166,46 @@ func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
 
 	// don't cache the result from previous run in case the user added
 	// a new dataset in the meantime
-	ds, stop := a.filterFilesystems()
+	fss, stop := a.filterFilesystems()
 	if stop {
 		return
 	}
 
+	a.task.Log().Info("beginning parallel snapshots")
+
 	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
+	var wg sync.WaitGroup
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+	for fsi := range fss {
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
+		wg.Add(1)
+		go func(fs *zfs.DatasetPath) {
+			defer wg.Done()
 
-		l := a.task.Log().WithField(logFSField, d.ToString()).
+			suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-			WithField("snapname", snapname)
+			snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
 
-		l.Info("create snapshot")
+			l := a.task.Log().WithField(logFSField, fs.ToString()).
-		err := zfs.ZFSSnapshot(d, snapname, false)
+				WithField("snapname", snapname)
-		if err != nil {
-			a.task.Log().WithError(err).Error("cannot create snapshot")
-		}
 
-		l.Info("create corresponding bookmark")
+			l.Info("create snapshot")
-		err = zfs.ZFSBookmark(d, snapname, snapname)
+			err := zfs.ZFSSnapshot(fs, snapname, false)
-		if err != nil {
+			if err != nil {
-			a.task.Log().WithError(err).Error("cannot create bookmark")
+				l.WithError(err).Error("cannot create snapshot")
-		}
+				return
+			}
 
+			l.Info("create corresponding bookmark")
+			err = zfs.ZFSBookmark(fs, snapname, snapname)
+			if err != nil {
+				l.WithError(err).Error("cannot create bookmark")
+			}
+
+		}(fss[fsi])
 	}
 
+	a.task.Log().Info("waiting for parallel snapshots to finish")
+	wg.Wait()
+	a.task.Log().Info("snapshots finished")
+
 	select {
 	case didSnaps <- struct{}{}:
 	default:

cmd/prune.go

@@ -3,7 +3,9 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
+	"sync"
 	"time"
 )
 
@@ -23,9 +25,8 @@ type PruneResult struct {
 	Remove     []zfs.FilesystemVersion
 }
 
+// FIXME must not call p.task.Enter because it runs in parallel
 func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
-	p.task.Enter("filter_fs")
-	defer p.task.Finish()
 	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
 	if err != nil {
 		p.task.Log().WithError(err).Error("error applying filesystem filter")
@@ -38,9 +39,8 @@ func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool)
 	return filesystems, false
 }
 
+// FIXME must not call p.task.Enter because it runs in parallel
 func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
-	p.task.Enter("filter_versions")
-	defer p.task.Finish()
 	log := p.task.Log().WithField(logFSField, fs.ToString())
 
 	filter := NewPrefixFilter(p.SnapshotPrefix)
@@ -56,9 +56,8 @@ func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.Filesyste
 	return fsversions, false
 }
 
-func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool) {
+// FIXME must not call p.task.Enter because it runs in parallel
-	p.task.Enter("prune_fs")
+func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath, destroySemaphore util.Semaphore) (r PruneResult, valid bool) {
-	defer p.task.Finish()
 	log := p.task.Log().WithField(logFSField, fs.ToString())
 
 	fsversions, stop := p.filterVersions(fs)
@@ -66,9 +65,7 @@ func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool
 		return
 	}
 
-	p.task.Enter("prune_policy")
 	keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-	p.task.Finish()
 	if err != nil {
 		log.WithError(err).Error("error evaluating prune policy")
 		return
@@ -81,33 +78,37 @@ func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool
 
 	r = PruneResult{fs, fsversions, keep, remove}
 
-	makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
+	var wg sync.WaitGroup
-		fields = make(map[string]interface{})
+	for v := range remove {
-		fields["version"] = v.ToAbsPath(fs)
+		wg.Add(1)
-		timeSince := v.Creation.Sub(p.Now)
+		go func(v zfs.FilesystemVersion) {
-		fields["age_ns"] = timeSince
+			defer wg.Done()
-		const day time.Duration = 24 * time.Hour
+			// log fields
-		days := timeSince / day
+			fields := make(map[string]interface{})
-		remainder := timeSince % day
+			fields["version"] = v.ToAbsPath(fs)
-		fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
+			timeSince := v.Creation.Sub(p.Now)
-		return
+			fields["age_ns"] = timeSince
-	}
+			const day time.Duration = 24 * time.Hour
+			days := timeSince / day
+			remainder := timeSince % day
+			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
 
-	for _, v := range remove {
+			log.WithFields(fields).Info("destroying version")
-		fields := makeFields(v)
+			// echo what we'll do and exec zfs destroy if not dry run
-		log.WithFields(fields).Info("destroying version")
+			// TODO special handling for EBUSY (zfs hold)
-		// echo what we'll do and exec zfs destroy if not dry run
+			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-		// TODO special handling for EBUSY (zfs hold)
+			if !p.DryRun {
-		// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+				destroySemaphore.Down()
-		if !p.DryRun {
+				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
-			p.task.Enter("destroy")
+				destroySemaphore.Up()
-			err := zfs.ZFSDestroyFilesystemVersion(fs, v)
+				if err != nil {
-			p.task.Finish()
+					log.WithFields(fields).WithError(err).Error("error destroying version")
-			if err != nil {
+				}
-				log.WithFields(fields).WithError(err).Error("error destroying version")
 			}
-		}
+		}(remove[v])
 	}
+	wg.Wait()
+
 	return r, true
 }
 
@@ -124,13 +125,31 @@ func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
 		return
 	}
 
-	r = make([]PruneResult, 0, len(filesystems))
+	maxConcurrentDestroy := len(filesystems)
+	p.task.Log().WithField("max_concurrent_destroy", maxConcurrentDestroy).Info("begin concurrent destroy")
+	destroySem := util.NewSemaphore(maxConcurrentDestroy)
 
+	resChan := make(chan PruneResult, len(filesystems))
+	var wg sync.WaitGroup
 	for _, fs := range filesystems {
-		res, ok := p.pruneFilesystem(fs)
+		wg.Add(1)
-		if ok {
+		go func(fs *zfs.DatasetPath) {
-			r = append(r, res)
+			defer wg.Done()
-		}
+			res, ok := p.pruneFilesystem(fs, destroySem)
+			if ok {
+				resChan <- res
+			}
+
+		}(fs)
+	}
+	wg.Wait()
+	close(resChan)
+
+	p.task.Log().Info("destroys done")
+
+	r = make([]PruneResult, 0, len(filesystems))
+	for res := range resChan {
+		r = append(r, res)
 	}
 
 	return

util/semaphore.go

@@ -0,0 +1,17 @@
+package util
+
+type Semaphore struct {
+	c chan struct{}
+}
+
+func NewSemaphore(cap int) Semaphore {
+	return Semaphore{make(chan struct{}, cap)}
+}
+
+func (s Semaphore) Down() {
+	s.c <- struct{}{}
+}
+
+func (s Semaphore) Up() {
+	<-s.c
+}