endpoint: refactor, fix stale holds on initial replication, holds release subcmds

- endpoint abstractions now share an interface `Abstraction` - pkg endpoint now has a query facitilty (`ListAbstractions`) which is used to find on-disk - step holds and bookmarks - replication cursors (v1, v2) - last-received-holds - the `zrepl holds list` command consumes endpoint.ListAbstractions - the new `zrepl holds release-{all,stale}` commands can be used to remove abstractions of package endpoint Co-authored-by: InsanePrawn <insane.prawny@gmail.com> supersedes #282 fixes #280 fixes #278
2020-03-26 23:43:17 +01:00
parent 44bd354eae
commit f3734ed0d4
28 changed files with 1854 additions and 1168 deletions
@@ -1,13 +1,11 @@
 package client

 import (
-	"context"
-	"encoding/json"
 	"fmt"
-	"os"
+	"sort"
 	"strings"

-	"github.com/pkg/errors"
+	"github.com/spf13/pflag"

 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/daemon/filters"
@@ -20,68 +18,126 @@ var (
 		Use:   "holds",
 		Short: "manage holds & step bookmarks",
 		SetupSubcommands: func() []*cli.Subcommand {
-			return holdsList
+			return []*cli.Subcommand{
+				holdsCmdList,
+				holdsCmdReleaseAll,
+				holdsCmdReleaseStale,
+			}
 		},
 	}
 )

-var holdsList = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use:             "list [FSFILTER]",
-		Run:             doHoldsList,
-		NoRequireConfig: true,
-		Short: `
-FSFILTER SYNTAX:
-representation of a 'filesystems' filter statement on the command line
-		`,
-	},
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type holdsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
 }

-func fsfilterFromCliArg(arg string) (zfs.DatasetFilter, error) {
-	mappings := strings.Split(arg, ",")
+// produce a query from the CLI flags
+func (f holdsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:    f.Filesystems.FlagValue(),
+		What:  f.Types.FlagValue(),
+		JobID: f.Job.FlagValue(),
+		Until: nil, // TODO support this as a flag
+	}
+	return q, q.Validate()
+}
+
+func (f *holdsFilterFlags) registerHoldsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
 	f := filters.NewDatasetMapFilter(len(mappings), true)
 	for _, m := range mappings {
 		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
 		lhsrhs := strings.SplitN(m, ":", 2)
 		if len(lhsrhs) != 2 {
-			return nil, thisMappingErr
+			return thisMappingErr
 		}
 		err := f.Add(lhsrhs[0], lhsrhs[1])
 		if err != nil {
-			return nil, fmt.Errorf("%s: %s", thisMappingErr, err)
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
 		}
 	}
-	return f.AsFilter(), nil
-}
-
-func doHoldsList(sc *cli.Subcommand, args []string) error {
-	var err error
-	ctx := context.Background()
-
-	if len(args) > 1 {
-		return errors.New("this subcommand takes at most one argument")
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
 	}
-
-	var filter zfs.DatasetFilter
-	if len(args) == 0 {
-		filter = zfs.NoFilter()
-	} else {
-		filter, err = fsfilterFromCliArg(args[0])
-		if err != nil {
-			return errors.Wrap(err, "cannot parse filesystem filter args")
-		}
-	}
-
-	listing, err := endpoint.ListZFSHoldsAndBookmarks(ctx, filter)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("  ", "  ")
-	if err := enc.Encode(listing); err != nil {
-		panic(err)
-	}
-
 	return nil
 }
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}
@@ -0,0 +1,71 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+var holdsListFlags struct {
+	Filter holdsFilterFlags
+	Json   bool
+}
+
+var holdsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Run:             doHoldsList,
+	NoRequireConfig: true,
+	Short:           "list holds and bookmarks",
+	SetupFlags: func(f *pflag.FlagSet) {
+		holdsListFlags.Filter.registerHoldsFilterFlags(f, "list")
+		f.BoolVar(&holdsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doHoldsList(sc *cli.Subcommand, args []string) error {
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := holdsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	// always print what we got
+	if holdsListFlags.Json {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(abstractions); err != nil {
+			panic(err)
+		}
+		fmt.Println()
+	} else {
+		for _, a := range abstractions {
+			fmt.Println(a)
+		}
+	}
+
+	// then potential errors, so that users always see them
+	if len(errors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", errors)
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}
@@ -0,0 +1,144 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var holdsReleaseFlags struct {
+	Filter holdsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerHoldsReleaseFlags(s *pflag.FlagSet) {
+	holdsReleaseFlags.Filter.registerHoldsFilterFlags(s, "release")
+	s.BoolVar(&holdsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&holdsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var holdsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doHoldsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release all zrepl-managed holds and bookmarks, mostly useful for uninstalling zrepl`,
+	SetupFlags:      registerHoldsReleaseFlags,
+}
+
+var holdsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doHoldsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl-managed holds and boomkarks (useful if zrepl has a bug and doesn't do it by itself)`,
+	SetupFlags:      registerHoldsReleaseFlags,
+}
+
+func doHoldsReleaseAll(sc *cli.Subcommand, args []string) error {
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := holdsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doHoldsRelease_Common(ctx, abstractions)
+}
+
+func doHoldsReleaseStale(sc *cli.Subcommand, args []string) error {
+
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := holdsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doHoldsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doHoldsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if holdsReleaseFlags.DryRun {
+		if holdsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if holdsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}