diff --git a/Gopkg.lock b/Gopkg.lock index 5506819..229907e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,163 +3,293 @@ [[projects]] branch = "master" + digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb" name = "github.com/beorn7/perks" packages = ["quantile"] + pruneopts = "" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] + digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] - branch = "master" - name = "github.com/dustin/go-humanize" + digest = "1:e988ed0ca0d81f4d28772760c02ee95084961311291bdfefc1b04617c178b722" + name = "github.com/fatih/color" packages = ["."] - revision = "bb3d318650d48840a39aa21a027c6630e198e626" + pruneopts = "" + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" [[projects]] branch = "master" + digest = "1:5d0a2385edf4ba44f3b7b76bc0436ceb8f62bf55aa5d540a9eb9ec6c58d86809" name = "github.com/ftrvxmtrx/fd" packages = ["."] + pruneopts = "" revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad" [[projects]] + digest = "1:6a4a01d58b227c4b6b11111b9f172ec5c17682b82724e58e6daf3f19f4faccd8" name = "github.com/go-logfmt/logfmt" packages = ["."] + pruneopts = "" revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" [[projects]] branch = "v2" + digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61" name = "github.com/go-yaml/yaml" packages = ["."] + pruneopts = "" revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f" [[projects]] + digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" + pruneopts = "" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" [[projects]] branch = "master" + digest = "1:cb09475f771b9167fb9333629f5d6a7161572602ea040f1094602b0dc8709878" name = "github.com/jinzhu/copier" packages = ["."] + pruneopts = "" revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b" [[projects]] branch = "master" + digest = "1:1ed9eeebdf24aadfbca57eb50e6455bd1d2474525e0f0d4454de8c8e9bc7ee9a" name = "github.com/kr/logfmt" packages = ["."] + pruneopts = "" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" [[projects]] branch = "master" + digest = "1:591a2778aa6e896980757ea87e659b3aa13d8c0e790310614028463a31c0998b" name = "github.com/kr/pretty" packages = ["."] + pruneopts = "" revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4" [[projects]] branch = "master" + digest = "1:109e0a9b9e74c5c8adf94a2dc4dd4a9ca2a183d4d87ffecd4e62db69a5ede55a" name = "github.com/kr/text" packages = ["."] + pruneopts = "" revision = "7cafcd837844e784b526369c9bce262804aebc60" [[projects]] + digest = "1:9ea83adf8e96d6304f394d40436f2eb44c1dc3250d223b74088cc253a6cd0a1c" + name = "github.com/mattn/go-colorable" + packages = ["."] + pruneopts = "" + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" + +[[projects]] + digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c" name = "github.com/mattn/go-isatty" packages = ["."] + pruneopts = "" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] + digest = "1:82b912465c1da0668582a7d1117339c278e786c2536b3c3623029a0c7141c2d0" + name = "github.com/mattn/go-runewidth" + packages = ["."] + pruneopts = "" + revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb" + version = "v0.0.3" + +[[projects]] + digest = "1:4c23ced97a470b17d9ffd788310502a077b9c1f60221a85563e49696276b4147" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "" revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:c9ede10a9ded782d25d1f0be87c680e11409c23554828f19a19d691a95e76130" name = "github.com/mitchellh/mapstructure" packages = ["."] + pruneopts = "" revision = "d0303fe809921458f417bcf828397a65db30a7e4" [[projects]] + branch = "master" + digest = "1:20a553eff588d7abe1f05addf5f57cdbaef1d0f992427a0099b7eb51274b79cf" + name = "github.com/nsf/termbox-go" + packages = ["."] + pruneopts = "" + revision = "b66b20ab708e289ff1eb3e218478302e6aec28ce" + +[[projects]] + digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:1392748e290ca66ac8447ef24961f8ae9e1d846a53af0f58a5a0256982ce0577" name = "github.com/problame/go-netssh" packages = ["."] + pruneopts = "" revision = "c56ad38d2c91397ad3c8dd9443d7448e328a9e9e" [[projects]] branch = "master" + digest = "1:8c63c44f018bd52b03ebad65c9df26aabbc6793138e421df1c8c84c285a45bc6" name = "github.com/problame/go-rwccmd" packages = ["."] + pruneopts = "" revision = "391d2c78c8404a9683d79f75dd24ab53040f89f7" +[[projects]] + digest = "1:c2ba1c9dc003c15856e4529dac028cacba08ee8924300f058b3467cde9acf7a9" + name = "github.com/problame/go-streamrpc" + packages = [ + ".", + "internal/pdu", + ] + pruneopts = "" + revision = "de6f6a4041c77f700f02d8fe749e54efa50811f7" + version = "v0.4" + [[projects]] branch = "master" + digest = "1:ebf8ffdde9bdbf9c83e22121875c68c01d821776523546554b2a3ff6f72773ab" name = "github.com/prometheus/client_golang" - packages = ["prometheus","prometheus/promhttp"] + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "" revision = "e11c6ff8170beca9d5fd8b938e71165eeec53ac6" [[projects]] branch = "master" + digest = "1:60aca47f4eeeb972f1b9da7e7db51dee15ff6c59f7b401c1588b8e6771ba15ef" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "" revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" [[projects]] branch = "master" + digest = "1:af21ee3e0a8212f17bb317cd7237f9920bcb2641a291ac111f30f63b3cab817f" name = "github.com/prometheus/common" - packages = ["expfmt","internal/bitbucket.org/ww/goautoneg","model"] + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + ] + pruneopts = "" revision = "d0f7cd64bda49e08b22ae8a730aa57aa0db125d6" [[projects]] branch = "master" + digest = "1:61df0898746840afc7be5dc2c3eeec83022fab70df11ecee5b16c85e912cf5ed" name = "github.com/prometheus/procfs" - packages = [".","internal/util","nfs","xfs"] + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "" revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e" [[projects]] branch = "master" + digest = "1:146327ce93be37e68bd3ff8541090d96da8cb3adc9e35d57570e9170a29f6bf6" name = "github.com/spf13/cobra" packages = ["."] + pruneopts = "" revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3" [[projects]] + digest = "1:261bc565833ef4f02121450d74eb88d5ae4bd74bfe5d0e862cddb8550ec35000" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "" revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" version = "v1.0.0" [[projects]] + digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159" name = "github.com/stretchr/testify" - packages = ["assert"] + packages = [ + "assert", + "require", + ] + pruneopts = "" revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" version = "v1.1.4" +[[projects]] + branch = "v2" + digest = "1:9d92186f609a73744232323416ddafd56fae67cb552162cc190ab903e36900dd" + name = "github.com/zrepl/yaml-config" + packages = ["."] + pruneopts = "" + revision = "af27d27978ad95808723a62d87557d63c3ff0605" + [[projects]] branch = "master" + digest = "1:9c286cf11d0ca56368185bada5dd6d97b6be4648fc26c354fcba8df7293718f7" name = "golang.org/x/sys" packages = ["unix"] + pruneopts = "" revision = "bf42f188b9bc6f2cf5b8ee5a912ef1aedd0eba4c" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "36731e77252dcc851fdfa2b0b0778b980597e3c1a47b5e2af3bd0bcb802662ec" + input-imports = [ + "github.com/fatih/color", + "github.com/go-logfmt/logfmt", + "github.com/go-yaml/yaml", + "github.com/golang/protobuf/proto", + "github.com/jinzhu/copier", + "github.com/kr/pretty", + "github.com/mattn/go-isatty", + "github.com/mitchellh/mapstructure", + "github.com/nsf/termbox-go", + "github.com/pkg/errors", + "github.com/problame/go-netssh", + "github.com/problame/go-rwccmd", + "github.com/problame/go-streamrpc", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/spf13/cobra", + "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", + "github.com/zrepl/yaml-config", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index f526185..fe541e1 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -38,7 +38,7 @@ ignored = [ "github.com/inconshreveable/mousetrap" ] [[constraint]] branch = "v2" - name = "github.com/go-yaml/yaml" + name = "github.com/zrepl/yaml-config" [[constraint]] name = "github.com/go-logfmt/logfmt" @@ -55,3 +55,18 @@ ignored = [ "github.com/inconshreveable/mousetrap" ] [[constraint]] name = "github.com/prometheus/client_golang" branch = "master" + +[[constraint]] + name = "github.com/golang/protobuf" + version = "1.2.0" + +[[constraint]] + name = "github.com/nsf/termbox-go" + branch = "master" +[[constraint]] + name = "github.com/fatih/color" + version = "1.7.0" + +[[constraint]] + name = "github.com/problame/go-streamrpc" + version = "0.4.0" diff --git a/Makefile b/Makefile index 6d26c7d..bcaec3c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,32 @@ .DEFAULT_GOAL := build ROOT := github.com/zrepl/zrepl -SUBPKGS := cmd logger rpc util zfs +SUBPKGS += client +SUBPKGS += config +SUBPKGS += daemon +SUBPKGS += daemon/filters +SUBPKGS += daemon/job +SUBPKGS += daemon/logging +SUBPKGS += daemon/nethelpers +SUBPKGS += daemon/pruner +SUBPKGS += daemon/snapper +SUBPKGS += daemon/streamrpcconfig +SUBPKGS += daemon/transport +SUBPKGS += daemon/transport/connecter +SUBPKGS += daemon/transport/serve +SUBPKGS += endpoint +SUBPKGS += logger +SUBPKGS += pruning +SUBPKGS += pruning/retentiongrid +SUBPKGS += replication +SUBPKGS += replication/fsrep +SUBPKGS += replication/pdu +SUBPKGS += replication/internal/queue +SUBPKGS += replication/internal/diff +SUBPKGS += tlsconf +SUBPKGS += util +SUBPKGS += version +SUBPKGS += zfs _TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p)) @@ -14,7 +39,7 @@ ifndef ZREPL_VERSION $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation) endif endif -GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)" +GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(ZREPL_VERSION)" GO_BUILD := go build -ldflags $(GO_LDFLAGS) @@ -26,6 +51,7 @@ vendordeps: dep ensure -v -vendor-only generate: #not part of the build, must do that manually + protoc -I=replication/pdu --go_out=replication/pdu replication/pdu/pdu.proto @for pkg in $(_TESTPKGS); do\ go generate "$$pkg" || exit 1; \ done; diff --git a/README.md b/README.md index 770dfc4..e4d4027 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Make sure to develop an understanding how zrepl is typically used by studying th ``` ├── cmd +│   ├── endpoint # implementations of endpoints for package replication │   ├── sampleconf # example configuration ├── docs # sphinx-based documentation │   ├── **/*.rst # documentation in reStructuredText @@ -55,6 +56,7 @@ Make sure to develop an understanding how zrepl is typically used by studying th │   ├── publish.sh # shell script for automated rendering & deploy to zrepl.github.io repo │   ├── public_git # checkout of zrepl.github.io managed by above shell script ├── logger # logger package used by zrepl +├── replication # replication functionality ├── rpc # rpc protocol implementation ├── util └── zfs # ZFS wrappers, filesystemm diffing diff --git a/build.Dockerfile b/build.Dockerfile index 47cec34..d7729e5 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -1,7 +1,12 @@ FROM golang:latest RUN apt-get update && apt-get install -y \ - python3-pip + python3-pip \ + unzip + +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip +RUN echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c +RUN unzip -d /usr protoc-3.6.1-linux-x86_64.zip ADD lazy.sh /tmp/lazy.sh ADD docs/requirements.txt /tmp/requirements.txt diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 0000000..8904bdc --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,116 @@ +package cli + +import ( + "fmt" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/zrepl/zrepl/config" + "os" +) + +var rootArgs struct { + configPath string +} + +var rootCmd = &cobra.Command{ + Use: "zrepl", + Short: "One-stop ZFS replication solution", +} + +var bashcompCmd = &cobra.Command{ + Use: "bashcomp path/to/out/file", + Short: "generate bash completions", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n") + cmd.Usage() + os.Exit(1) + } + if err := rootCmd.GenBashCompletionFile(args[0]); err != nil { + fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err) + os.Exit(1) + } + }, + Hidden: true, +} + +func init() { + rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path") + rootCmd.AddCommand(bashcompCmd) +} + +type Subcommand struct { + Use string + Short string + NoRequireConfig bool + Run func(subcommand *Subcommand, args []string) error + SetupFlags func(f *pflag.FlagSet) + SetupSubcommands func() []*Subcommand + + config *config.Config + configErr error +} + +func (s *Subcommand) ConfigParsingError() error { + return s.configErr +} + +func (s *Subcommand) Config() *config.Config { + if !s.NoRequireConfig && s.config == nil { + panic("command that requires config is running and has no config set") + } + return s.config +} + +func (s *Subcommand) run(cmd *cobra.Command, args []string) { + s.tryParseConfig() + err := s.Run(s, args) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } +} + +func (s *Subcommand) tryParseConfig() { + config, err := config.ParseConfig(rootArgs.configPath) + s.configErr = err + if err != nil { + if s.NoRequireConfig { + // doesn't matter + return + } else { + fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err) + os.Exit(1) + } + } + s.config = config +} + +func AddSubcommand(s *Subcommand) { + addSubcommandToCobraCmd(rootCmd, s) +} + +func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) { + cmd := cobra.Command{ + Use: s.Use, + Short: s.Short, + } + if s.SetupSubcommands == nil { + cmd.Run = s.run + } else { + for _, sub := range s.SetupSubcommands() { + addSubcommandToCobraCmd(&cmd, sub) + } + } + if s.SetupFlags != nil { + s.SetupFlags(cmd.Flags()) + } + c.AddCommand(&cmd) +} + + +func Run() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} \ No newline at end of file diff --git a/client/configcheck.go b/client/configcheck.go new file mode 100644 index 0000000..34c1b13 --- /dev/null +++ b/client/configcheck.go @@ -0,0 +1,36 @@ +package client + +import ( + "encoding/json" + "github.com/kr/pretty" + "github.com/spf13/pflag" + "github.com/zrepl/yaml-config" + "github.com/zrepl/zrepl/cli" + "os" +) + +var configcheckArgs struct { + format string +} + +var ConfigcheckCmd = &cli.Subcommand{ + Use: "configcheck", + Short: "check if config can be parsed without errors", + SetupFlags: func(f *pflag.FlagSet) { + f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]") + }, + Run: func(subcommand *cli.Subcommand, args []string) error { + switch configcheckArgs.format { + case "pretty": + _, err := pretty.Println(subcommand.Config()) + return err + case "json": + return json.NewEncoder(os.Stdout).Encode(subcommand.Config()) + case "yaml": + return yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()) + default: // no output + } + return nil + }, +} + diff --git a/client/jsonclient.go b/client/jsonclient.go new file mode 100644 index 0000000..41c272d --- /dev/null +++ b/client/jsonclient.go @@ -0,0 +1,48 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "github.com/pkg/errors" + "io" + "net" + "net/http" +) + +func controlHttpClient(sockpath string) (client http.Client, err error) { + return http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", sockpath) + }, + }, + }, nil +} + +func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error { + var buf bytes.Buffer + encodeErr := json.NewEncoder(&buf).Encode(req) + if encodeErr != nil { + return encodeErr + } + + resp, err := c.Post("http://unix"+endpoint, "application/json", &buf) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var msg bytes.Buffer + io.CopyN(&msg, resp.Body, 4096) + return errors.Errorf("%s", msg.String()) + } + + decodeError := json.NewDecoder(resp.Body).Decode(&res) + if decodeError != nil { + return decodeError + } + + return nil +} diff --git a/client/pprof.go b/client/pprof.go new file mode 100644 index 0000000..29cd4e5 --- /dev/null +++ b/client/pprof.go @@ -0,0 +1,66 @@ +package client + +import ( + "errors" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon" + "log" + "os" +) + +var pprofArgs struct { + daemon.PprofServerControlMsg +} + +var PprofCmd = &cli.Subcommand{ + Use: "pprof off | [on TCP_LISTEN_ADDRESS]", + Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS", + Run: func(subcommand *cli.Subcommand, args []string) error { + if len(args) < 1 { + goto enargs + } + switch args[0] { + case "on": + pprofArgs.Run = true + if len(args) != 2 { + return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument") + } + pprofArgs.HttpListenAddress = args[1] + case "off": + if len(args) != 1 { + goto enargs + } + pprofArgs.Run = false + } + + RunPProf(subcommand.Config()) + return nil + enargs: + return errors.New("invalid number of positional arguments") + + }, +} + +func RunPProf(conf *config.Config) { + log := log.New(os.Stderr, "", 0) + + die := func() { + log.Printf("exiting after error") + os.Exit(1) + } + + log.Printf("connecting to zrepl daemon") + + httpc, err := controlHttpClient(conf.Global.Control.SockPath) + if err != nil { + log.Printf("error creating http client: %s", err) + die() + } + err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{}) + if err != nil { + log.Printf("error sending control message: %s", err) + die() + } + log.Printf("finished") +} diff --git a/client/signal.go b/client/signal.go new file mode 100644 index 0000000..701849e --- /dev/null +++ b/client/signal.go @@ -0,0 +1,39 @@ +package client + +import ( + "github.com/pkg/errors" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon" +) + +var SignalCmd = &cli.Subcommand{ + Use: "signal [wakeup|reset] JOB", + Short: "wake up a job from wait state or abort its current invocation", + Run: func(subcommand *cli.Subcommand, args []string) error { + return runSignalCmd(subcommand.Config(), args) + }, +} + +func runSignalCmd(config *config.Config, args []string) error { + if len(args) != 2 { + return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB") + } + + httpc, err := controlHttpClient(config.Global.Control.SockPath) + if err != nil { + return err + } + + err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal, + struct { + Name string + Op string + }{ + Name: args[1], + Op: args[0], + }, + struct{}{}, + ) + return err +} diff --git a/client/status.go b/client/status.go new file mode 100644 index 0000000..7e715e9 --- /dev/null +++ b/client/status.go @@ -0,0 +1,547 @@ +package client + +import ( + "fmt" + "github.com/nsf/termbox-go" + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/zrepl/yaml-config" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/daemon" + "github.com/zrepl/zrepl/daemon/job" + "github.com/zrepl/zrepl/daemon/pruner" + "github.com/zrepl/zrepl/replication" + "github.com/zrepl/zrepl/replication/fsrep" + "io" + "math" + "net/http" + "os" + "sort" + "strings" + "sync" + "time" +) + +type tui struct { + x, y int + indent int + + lock sync.Mutex //For report and error + report map[string]job.Status + err error +} + +func newTui() tui { + return tui{} +} + +func (t *tui) moveCursor(x, y int) { + t.x += x + t.y += y +} + +func (t *tui) moveLine(dl int, col int) { + t.y += dl + t.x = t.indent*4 + col +} + +func (t *tui) write(text string) { + for _, c := range text { + if c == '\n' { + t.newline() + continue + } + termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault) + t.x += 1 + } +} + +func (t *tui) printf(text string, a ...interface{}) { + t.write(fmt.Sprintf(text, a...)) +} + +func (t *tui) newline() { + t.moveLine(1, 0) +} + +func (t *tui) setIndent(indent int) { + t.indent = indent + t.moveLine(0, 0) +} + +func (t *tui) addIndent(indent int) { + t.indent += indent + t.moveLine(0, 0) +} + +var statusFlags struct { + Raw bool +} + +var StatusCmd = &cli.Subcommand{ + Use: "status", + Short: "show job activity or dump as JSON for monitoring", + SetupFlags: func(f *pflag.FlagSet) { + f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon") + }, + Run: runStatus, +} + +func runStatus(s *cli.Subcommand, args []string) error { + httpc, err := controlHttpClient(s.Config().Global.Control.SockPath) + if err != nil { + return err + } + + if statusFlags.Raw { + resp, err := httpc.Get("http://unix"+daemon.ControlJobEndpointStatus) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "Received error response:\n") + io.CopyN(os.Stderr, resp.Body, 4096) + return errors.Errorf("exit") + } + if _, err := io.Copy(os.Stdout, resp.Body); err != nil { + return err + } + return nil + } + + t := newTui() + t.lock.Lock() + t.err = errors.New("Got no report yet") + t.lock.Unlock() + + err = termbox.Init() + if err != nil { + return err + } + defer termbox.Close() + + update := func() { + m := make(map[string]job.Status) + + err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus, + struct{}{}, + &m, + ) + + t.lock.Lock() + t.err = err2 + t.report = m + t.lock.Unlock() + t.draw() + } + update() + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + go func() { + for _ = range ticker.C { + update() + } + }() + + termbox.HideCursor() + termbox.Clear(termbox.ColorDefault, termbox.ColorDefault) + +loop: + for { + switch ev := termbox.PollEvent(); ev.Type { + case termbox.EventKey: + switch ev.Key { + case termbox.KeyEsc: + break loop + case termbox.KeyCtrlC: + break loop + } + case termbox.EventResize: + t.draw() + } + } + + return nil + +} + +func (t *tui) draw() { + t.lock.Lock() + defer t.lock.Unlock() + + termbox.Clear(termbox.ColorDefault, termbox.ColorDefault) + t.x = 0 + t.y = 0 + t.indent = 0 + + if t.err != nil { + t.write(t.err.Error()) + } else { + //Iterate over map in alphabetical order + keys := make([]string, len(t.report)) + i := 0 + for k, _ := range t.report { + keys[i] = k + i++ + } + sort.Strings(keys) + + for _, k := range keys { + v := t.report[k] + if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job + continue + } + t.setIndent(0) + + t.printf("Job: %s", k) + t.setIndent(1) + t.newline() + t.printf("Type: %s", v.Type) + t.setIndent(1) + t.newline() + + if v.Type != job.TypePush && v.Type != job.TypePull { + t.printf("No status representation for job type '%s', dumping as YAML", v.Type) + t.newline() + asYaml, err := yaml.Marshal(v.JobSpecific) + if err != nil { + t.printf("Error marshaling status to YAML: %s", err) + t.newline() + continue + } + t.write(string(asYaml)) + t.newline() + continue + } + + pushStatus, ok := v.JobSpecific.(*job.ActiveSideStatus) + if !ok || pushStatus == nil { + t.printf("ActiveSideStatus is null") + t.newline() + continue + } + + t.printf("Replication:") + t.newline() + t.addIndent(1) + t.renderReplicationReport(pushStatus.Replication) + t.addIndent(-1) + + t.printf("Pruning Sender:") + t.newline() + t.addIndent(1) + t.renderPrunerReport(pushStatus.PruningSender) + t.addIndent(-1) + + t.printf("Pruning Receiver:") + t.newline() + t.addIndent(1) + t.renderPrunerReport(pushStatus.PruningReceiver) + t.addIndent(-1) + + } + } + termbox.Flush() +} + +func (t *tui) renderReplicationReport(rep *replication.Report) { + if rep == nil { + t.printf("...\n") + return + } + + all := make([]*fsrep.Report, 0, len(rep.Completed)+len(rep.Pending) + 1) + all = append(all, rep.Completed...) + all = append(all, rep.Pending...) + if rep.Active != nil { + all = append(all, rep.Active) + } + sort.Slice(all, func(i, j int) bool { + return all[i].Filesystem < all[j].Filesystem + }) + + state, err := replication.StateString(rep.Status) + if err != nil { + t.printf("Status: %q (parse error: %q)\n", rep.Status, err) + return + } + + t.printf("Status: %s", state) + t.newline() + if rep.Problem != "" { + t.printf("Problem: %s", rep.Problem) + t.newline() + } + if rep.SleepUntil.After(time.Now()) && + state & ^(replication.ContextDone|replication.Completed) != 0 { + t.printf("Sleeping until %s (%s left)\n", rep.SleepUntil, rep.SleepUntil.Sub(time.Now())) + } + + if state != replication.Planning && state != replication.PlanningError { + // Progress: [---------------] + sumUpFSRep := func(rep *fsrep.Report) (transferred, total int64) { + for _, s := range rep.Pending { + transferred += s.Bytes + total += s.ExpectedBytes + } + for _, s := range rep.Completed { + transferred += s.Bytes + total += s.ExpectedBytes + } + return + } + var transferred, total int64 + for _, fs := range all { + fstx, fstotal := sumUpFSRep(fs) + transferred += fstx + total += fstotal + } + t.write("Progress: ") + t.drawBar(80, transferred, total) + t.write(fmt.Sprintf(" %s / %s", ByteCountBinary(transferred), ByteCountBinary(total))) + t.newline() + } + + var maxFSLen int + for _, fs := range all { + if len(fs.Filesystem) > maxFSLen { + maxFSLen = len(fs.Filesystem) + } + } + for _, fs := range all { + printFilesystemStatus(fs, t, fs == rep.Active, maxFSLen) + } +} + +func (t *tui) renderPrunerReport(r *pruner.Report) { + if r == nil { + t.printf("...\n") + return + } + + state, err := pruner.StateString(r.State) + if err != nil { + t.printf("Status: %q (parse error: %q)\n", r.State, err) + return + } + + t.printf("Status: %s", state) + t.newline() + + if r.Error != "" { + t.printf("Error: %s\n", r.Error) + } + if r.SleepUntil.After(time.Now()) { + t.printf("Sleeping until %s (%s left)\n", r.SleepUntil, r.SleepUntil.Sub(time.Now())) + } + + type commonFS struct { + *pruner.FSReport + completed bool + } + all := make([]commonFS, 0, len(r.Pending) + len(r.Completed)) + for i := range r.Pending { + all = append(all, commonFS{&r.Pending[i], false}) + } + for i := range r.Completed { + all = append(all, commonFS{&r.Completed[i], true}) + } + + switch state { + case pruner.Plan: fallthrough + case pruner.PlanWait: fallthrough + case pruner.ErrPerm: + return + } + + if len(all) == 0 { + t.printf("nothing to do\n") + return + } + + var totalDestroyCount, completedDestroyCount int + var maxFSname int + for _, fs := range all { + totalDestroyCount += len(fs.DestroyList) + if fs.completed { + completedDestroyCount += len(fs.DestroyList) + } + if maxFSname < len(fs.Filesystem) { + maxFSname = len(fs.Filesystem) + } + } + + // global progress bar + progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount))) + t.write("Progress: ") + t.write("[") + t.write(times("=", progress)) + t.write(">") + t.write(times("-", 80 - progress)) + t.write("]") + t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount) + t.newline() + + sort.SliceStable(all, func(i, j int) bool { + return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1 + }) + + // Draw a table-like representation of 'all' + for _, fs := range all { + t.write(rightPad(fs.Filesystem, maxFSname, " ")) + t.write(" ") + if fs.Error != "" { + t.printf("ERROR: %s\n", fs.Error) // whitespace is padding + continue + } + + pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)", + len(fs.DestroyList), len(fs.SnapshotList)) + + if fs.completed { + t.printf( "Completed %s\n", pruneRuleActionStr) + continue + } + + t.write("Pending ") // whitespace is padding 10 + if len(fs.DestroyList) == 1 { + t.write(fs.DestroyList[0].Name) + } else { + t.write(pruneRuleActionStr) + } + t.newline() + } + +} + +const snapshotIndent = 1 +func calculateMaxFSLength(all []*fsrep.Report) (maxFS, maxStatus int) { + for _, e := range all { + if len(e.Filesystem) > maxFS { + maxFS = len(e.Filesystem) + } + all2 := make([]*fsrep.StepReport, 0, len(e.Pending) + len(e.Completed)) + all2 = append(all2, e.Pending...) + all2 = append(all2, e.Completed...) + for _, e2 := range all2 { + elen := len(e2.Problem) + len(e2.From) + len(e2.To) + 60 // random spacing, units, labels, etc + if elen > maxStatus { + maxStatus = elen + } + } + } + return +} + +func times(str string, n int) (out string) { + for i := 0; i < n; i++ { + out += str + } + return +} + +func rightPad(str string, length int, pad string) string { + if len(str) > length { + return str[:length] + } + return str + times(pad, length-len(str)) +} + + +func leftPad(str string, length int, pad string) string { + if len(str) > length { + return str[len(str)-length:] + } + return times(pad, length-len(str)) + str +} + +func (t *tui) drawBar(length int, bytes, totalBytes int64) { + var completedLength int + if totalBytes > 0 { + completedLength = int(int64(length) * bytes / totalBytes) + if completedLength > length { + completedLength = length + } + } else if totalBytes == bytes { + completedLength = length + } + + t.write("[") + t.write(times("=", completedLength)) + t.write(">") + t.write(times("-", length-completedLength)) + t.write("]") +} + +func StringStepState(s fsrep.StepState) string { + switch s { + case fsrep.StepReplicationReady: return "Ready" + case fsrep.StepReplicationRetry: return "Retry" + case fsrep.StepMarkReplicatedReady: return "MarkReady" + case fsrep.StepMarkReplicatedRetry: return "MarkRetry" + case fsrep.StepPermanentError: return "PermanentError" + case fsrep.StepCompleted: return "Completed" + default: + return fmt.Sprintf("UNKNOWN %d", s) + } +} + +func filesystemStatusString(rep *fsrep.Report, active bool, fsWidth int) (line string, bytes, totalBytes int64) { + bytes = int64(0) + totalBytes = int64(0) + for _, s := range rep.Pending { + bytes += s.Bytes + totalBytes += s.ExpectedBytes + } + for _, s := range rep.Completed { + bytes += s.Bytes + totalBytes += s.ExpectedBytes + } + + next := "" + if rep.Problem != "" { + next = " problem: " + rep.Problem + } else if len(rep.Pending) > 0 { + if rep.Pending[0].From != "" { + next = fmt.Sprintf(" next: %s => %s", rep.Pending[0].From, rep.Pending[0].To) + } else { + next = fmt.Sprintf(" next: %s (full)", rep.Pending[0].To) + } + } + status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s", + rep.Status, + len(rep.Completed), len(rep.Pending) + len(rep.Completed), + ByteCountBinary(bytes), ByteCountBinary(totalBytes), + next, + ) + activeIndicator := " " + if active { + activeIndicator = "*" + } + line = fmt.Sprintf("%s %s %s", + activeIndicator, + rightPad(rep.Filesystem, fsWidth, " "), + status) + return line, bytes, totalBytes +} + +func printFilesystemStatus(rep *fsrep.Report, t *tui, active bool, maxFS int) { + totalStatus, _, _ := filesystemStatusString(rep, active, maxFS) + t.write(totalStatus) + t.newline() +} + +func ByteCountBinary(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/cmd/stdinserver.go b/client/stdinserver.go similarity index 51% rename from cmd/stdinserver.go rename to client/stdinserver.go index 6d5d41e..5db5520 100644 --- a/cmd/stdinserver.go +++ b/client/stdinserver.go @@ -1,55 +1,49 @@ -package cmd +package client import ( + "github.com/zrepl/zrepl/cli" "os" "context" + "errors" "github.com/problame/go-netssh" - "github.com/spf13/cobra" + "github.com/zrepl/zrepl/config" "log" "path" ) -var StdinserverCmd = &cobra.Command{ +var StdinserverCmd = &cli.Subcommand{ Use: "stdinserver CLIENT_IDENTITY", - Short: "start in stdinserver mode (from authorized_keys file)", - Run: cmdStdinServer, + Short: "stdinserver transport mode (started from authorized_keys file as forced command)", + Run: func(subcommand *cli.Subcommand, args []string) error { + return runStdinserver(subcommand.Config(), args) + }, } -func init() { - RootCmd.AddCommand(StdinserverCmd) -} - -func cmdStdinServer(cmd *cobra.Command, args []string) { +func runStdinserver(config *config.Config, args []string) error { // NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong defer os.Exit(1) log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime) - conf, err := ParseConfig(rootArgs.configFile) - if err != nil { - log.Printf("error parsing config: %s", err) - return - } - if len(args) != 1 || args[0] == "" { - log.Print("must specify client_identity as positional argument") - return + err := errors.New("must specify client_identity as positional argument") + return err } identity := args[0] - unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity) + unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity) log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr) ctx := netssh.ContextWithLog(context.TODO(), log) - err = netssh.Proxy(ctx, unixaddr) + err := netssh.Proxy(ctx, unixaddr) if err == nil { log.Print("proxying finished successfully, exiting with status 0") os.Exit(0) } log.Printf("error proxying: %s", err) - + return nil } diff --git a/client/testcmd.go b/client/testcmd.go new file mode 100644 index 0000000..6635db5 --- /dev/null +++ b/client/testcmd.go @@ -0,0 +1,108 @@ +package client + +import ( + "fmt" + "github.com/spf13/pflag" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/filters" + "github.com/zrepl/zrepl/zfs" +) + +var TestCmd = &cli.Subcommand { + Use: "test", + SetupSubcommands: func() []*cli.Subcommand { + return []*cli.Subcommand{testFilter} + }, +} + +var testFilterArgs struct { + job string + all bool + input string +} + +var testFilter = &cli.Subcommand{ + Use: "filesystems --job JOB [--all | --input INPUT]", + Short: "test filesystems filter specified in push or source job", + SetupFlags: func(f *pflag.FlagSet) { + f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job") + f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters") + f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems") + }, + Run: runTestFilterCmd, +} + +func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error { + + if testFilterArgs.job == "" { + return fmt.Errorf("must specify --job flag") + } + if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor + return fmt.Errorf("must set one: --all or --input") + } + + conf := subcommand.Config() + + var confFilter config.FilesystemsFilter + job, err := conf.Job(testFilterArgs.job) + if err != nil { + return err + } + switch j := job.Ret.(type) { + case *config.SourceJob: confFilter = j.Filesystems + case *config.PushJob: confFilter = j.Filesystems + default: + return fmt.Errorf("job type %T does not have filesystems filter", j) + } + + f, err := filters.DatasetMapFilterFromConfig(confFilter) + if err != nil { + return fmt.Errorf("filter invalid: %s", err) + } + + var fsnames []string + if testFilterArgs.input != "" { + fsnames = []string{testFilterArgs.input} + } else { + out, err := zfs.ZFSList([]string{"name"}) + if err != nil { + return fmt.Errorf("could not list ZFS filesystems: %s", err) + } + for _, row := range out { + + fsnames = append(fsnames, row[0]) + } + } + + fspaths := make([]*zfs.DatasetPath, len(fsnames)) + for i, fsname := range fsnames { + path, err := zfs.NewDatasetPath(fsname) + if err != nil { + return err + } + fspaths[i] = path + } + + hadFilterErr := false + for _, in := range fspaths { + var res string + var errStr string + pass, err := f.Filter(in) + if err != nil { + res = "ERROR" + errStr = err.Error() + hadFilterErr = true + } else if pass { + res = "ACCEPT" + } else { + res = "REJECT" + } + fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr) + } + + if hadFilterErr { + return fmt.Errorf("filter errors occurred") + } + return nil +} \ No newline at end of file diff --git a/client/version.go b/client/version.go new file mode 100644 index 0000000..9dcec05 --- /dev/null +++ b/client/version.go @@ -0,0 +1,72 @@ +package client + +import ( + "fmt" + "github.com/spf13/pflag" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon" + "github.com/zrepl/zrepl/version" + "os" +) + +var versionArgs struct { + Show string + Config *config.Config + ConfigErr error +} + +var VersionCmd = &cli.Subcommand{ + Use: "version", + Short: "print version of zrepl binary and running daemon", + NoRequireConfig: true, + SetupFlags: func(f *pflag.FlagSet) { + f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)") + }, + Run: func(subcommand *cli.Subcommand, args []string) error { + versionArgs.Config = subcommand.Config() + versionArgs.ConfigErr = subcommand.ConfigParsingError() + return runVersionCmd() + }, +} + +func runVersionCmd() error { + args := versionArgs + + if args.Show != "daemon" && args.Show != "client" && args.Show != "" { + return fmt.Errorf("show flag must be 'client' or 'server' or be left empty") + } + + var clientVersion, daemonVersion *version.ZreplVersionInformation + if args.Show == "client" || args.Show == "" { + clientVersion = version.NewZreplVersionInformation() + fmt.Printf("client: %s\n", clientVersion.String()) + } + if args.Show == "daemon" || args.Show == "" { + + if args.ConfigErr != nil { + return fmt.Errorf("config parsing error: %s", args.ConfigErr) + } + + httpc, err := controlHttpClient(args.Config.Global.Control.SockPath) + if err != nil { + return fmt.Errorf("server: error: %s\n", err) + } + + var info version.ZreplVersionInformation + err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info) + if err != nil { + return fmt.Errorf("server: error: %s\n", err) + } + daemonVersion = &info + fmt.Printf("server: %s\n", daemonVersion.String()) + } + + if args.Show == "" { + if clientVersion.Version != daemonVersion.Version { + fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n") + } + } + + return nil +} diff --git a/cmd/autosnap.go b/cmd/autosnap.go deleted file mode 100644 index 776c387..0000000 --- a/cmd/autosnap.go +++ /dev/null @@ -1,201 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "github.com/zrepl/zrepl/zfs" - "sort" - "time" -) - -type IntervalAutosnap struct { - task *Task - DatasetFilter zfs.DatasetFilter - Prefix string - SnapshotInterval time.Duration -} - -func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) { - a.task.Enter("filter_filesystems") - defer a.task.Finish() - fss, err := zfs.ZFSListMapping(a.DatasetFilter) - stop = err != nil - if err != nil { - a.task.Log().WithError(err).Error("cannot list datasets") - } - if len(fss) == 0 { - a.task.Log().Warn("no filesystem matching filesystem filter") - } - return fss, stop -} - -func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) { - a.task.Enter("find_sync_point") - defer a.task.Finish() - type snapTime struct { - ds *zfs.DatasetPath - time time.Time - } - - if len(fss) == 0 { - return time.Now(), nil - } - - snaptimes := make([]snapTime, 0, len(fss)) - - now := time.Now() - - a.task.Log().Debug("examine filesystem state") - for _, d := range fss { - - l := a.task.Log().WithField(logFSField, d.ToString()) - - fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix)) - if err != nil { - l.WithError(err).Error("cannot list filesystem versions") - continue - } - if len(fsvs) <= 0 { - l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix") - continue - } - - // Sort versions by creation - sort.SliceStable(fsvs, func(i, j int) bool { - return fsvs[i].CreateTXG < fsvs[j].CreateTXG - }) - - latest := fsvs[len(fsvs)-1] - l.WithField("creation", latest.Creation). - Debug("found latest snapshot") - - since := now.Sub(latest.Creation) - if since < 0 { - l.WithField("snapshot", latest.Name). - WithField("creation", latest.Creation). - Error("snapshot is from the future") - continue - } - next := now - if since < a.SnapshotInterval { - next = latest.Creation.Add(a.SnapshotInterval) - } - snaptimes = append(snaptimes, snapTime{d, next}) - } - - if len(snaptimes) == 0 { - snaptimes = append(snaptimes, snapTime{nil, now}) - } - - sort.Slice(snaptimes, func(i, j int) bool { - return snaptimes[i].time.Before(snaptimes[j].time) - }) - - return snaptimes[0].time, nil - -} - -func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) { - a.task.Enter("wait_sync_point") - defer a.task.Finish() - - const LOG_TIME_FMT string = time.ANSIC - - a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)). - Info("wait for sync point") - - select { - case <-ctx.Done(): - a.task.Log().WithError(ctx.Err()).Info("context done") - return - case <-time.After(syncPoint.Sub(time.Now())): - } -} - -func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) { - a.task.Enter("sync_up") - defer a.task.Finish() - - fss, stop := a.filterFilesystems() - if stop { - return true - } - - syncPoint, err := a.findSyncPoint(fss) - if err != nil { - return true - } - - a.waitForSyncPoint(ctx, syncPoint) - - a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep") - a.doSnapshots(didSnaps) - return false -} - -func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) { - - if a.syncUpRun(ctx, didSnaps) { - a.task.Log().Error("stoppping autosnap after error in sync up") - return - } - - // task drops back to idle here - - a.task.Log().Debug("setting up ticker in SnapshotInterval") - ticker := time.NewTicker(a.SnapshotInterval) - for { - select { - case <-ctx.Done(): - ticker.Stop() - a.task.Log().WithError(ctx.Err()).Info("context done") - return - - case <-ticker.C: - a.doSnapshots(didSnaps) - } - } - -} - -func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) { - - a.task.Enter("do_snapshots") - defer a.task.Finish() - - // don't cache the result from previous run in case the user added - // a new dataset in the meantime - ds, stop := a.filterFilesystems() - if stop { - return - } - - // TODO channel programs -> allow a little jitter? - for _, d := range ds { - suffix := time.Now().In(time.UTC).Format("20060102_150405_000") - snapname := fmt.Sprintf("%s%s", a.Prefix, suffix) - - l := a.task.Log().WithField(logFSField, d.ToString()). - WithField("snapname", snapname) - - l.Info("create snapshot") - err := zfs.ZFSSnapshot(d, snapname, false) - if err != nil { - a.task.Log().WithError(err).Error("cannot create snapshot") - } - - l.Info("create corresponding bookmark") - err = zfs.ZFSBookmark(d, snapname, snapname) - if err != nil { - a.task.Log().WithError(err).Error("cannot create bookmark") - } - - } - - select { - case didSnaps <- struct{}{}: - default: - a.task.Log().Error("warning: callback channel is full, discarding") - } - -} diff --git a/cmd/bashcomp.go b/cmd/bashcomp.go deleted file mode 100644 index 3b39252..0000000 --- a/cmd/bashcomp.go +++ /dev/null @@ -1,29 +0,0 @@ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" - "os" -) - -var bashcompCmd = &cobra.Command{ - Use: "bashcomp path/to/out/file", - Short: "generate bash completions", - Run: func(cmd *cobra.Command, args []string) { - if len(args) != 1 { - fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n") - cmd.Usage() - os.Exit(1) - } - if err := RootCmd.GenBashCompletionFile(args[0]); err != nil { - fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err) - os.Exit(1) - } - }, - Hidden: true, -} - -func init() { - RootCmd.AddCommand(bashcompCmd) -} diff --git a/cmd/config.go b/cmd/config.go deleted file mode 100644 index 38a95be..0000000 --- a/cmd/config.go +++ /dev/null @@ -1,102 +0,0 @@ -package cmd - -import ( - "io" - - "fmt" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/zfs" -) - -type Config struct { - Global Global - Jobs map[string]Job -} - -func (c *Config) LookupJob(name string) (j Job, err error) { - j, ok := c.Jobs[name] - if !ok { - return nil, errors.Errorf("job '%s' is not defined", name) - } - return j, nil -} - -type Global struct { - Serve struct { - Stdinserver struct { - SockDir string - } - } - Control struct { - Sockpath string - } - logging *LoggingConfig -} - -type JobDebugSettings struct { - Conn struct { - ReadDump string `mapstructure:"read_dump"` - WriteDump string `mapstructure:"write_dump"` - } - RPC struct { - Log bool - } -} - -type RWCConnecter interface { - Connect() (io.ReadWriteCloser, error) -} -type AuthenticatedChannelListenerFactory interface { - Listen() (AuthenticatedChannelListener, error) -} - -type AuthenticatedChannelListener interface { - Accept() (ch io.ReadWriteCloser, err error) - Close() (err error) -} - -type SSHStdinServerConnectDescr struct { -} - -type PrunePolicy interface { - // Prune filters versions and decide which to keep and which to remove. - // Prune **does not** implement the actual removal of the versions. - Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) -} - -type PruningJob interface { - Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error) -} - -// A type for constants describing different prune policies of a PruningJob -// This is mostly a special-case for LocalJob, which is the only job that has two prune policies -// instead of one. -// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand -type PrunePolicySide string - -const ( - PrunePolicySideDefault PrunePolicySide = "" - PrunePolicySideLeft PrunePolicySide = "left" - PrunePolicySideRight PrunePolicySide = "right" -) - -func (s *PrunePolicySide) String() string { - return string(*s) -} - -func (s *PrunePolicySide) Set(news string) error { - p := PrunePolicySide(news) - switch p { - case PrunePolicySideRight: - fallthrough - case PrunePolicySideLeft: - *s = p - default: - return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight) - } - return nil -} - -func (s *PrunePolicySide) Type() string { - return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight) -} diff --git a/cmd/config_connect.go b/cmd/config_connect.go deleted file mode 100644 index b6d8435..0000000 --- a/cmd/config_connect.go +++ /dev/null @@ -1,66 +0,0 @@ -package cmd - -import ( - "fmt" - "io" - - "context" - "github.com/jinzhu/copier" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/problame/go-netssh" - "time" -) - -type SSHStdinserverConnecter struct { - Host string - User string - Port uint16 - IdentityFile string `mapstructure:"identity_file"` - TransportOpenCommand []string `mapstructure:"transport_open_command"` - SSHCommand string `mapstructure:"ssh_command"` - Options []string - DialTimeout string `mapstructure:"dial_timeout"` - dialTimeout time.Duration -} - -func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) { - - c = &SSHStdinserverConnecter{} - if err = mapstructure.Decode(i, c); err != nil { - err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err)) - return nil, err - } - - if c.DialTimeout != "" { - c.dialTimeout, err = time.ParseDuration(c.DialTimeout) - if err != nil { - return nil, errors.Wrap(err, "cannot parse dial_timeout") - } - } else { - c.dialTimeout = 10 * time.Second - } - - // TODO assert fields are filled - return - -} - -func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) { - - var endpoint netssh.Endpoint - if err = copier.Copy(&endpoint, c); err != nil { - return nil, errors.WithStack(err) - } - var dialCtx context.Context - dialCtx, dialCancel := context.WithTimeout(context.TODO(), c.dialTimeout) // context.TODO tied to error handling below - defer dialCancel() - if rwc, err = netssh.Dial(dialCtx, endpoint); err != nil { - if err == context.DeadlineExceeded { - err = errors.Errorf("dial_timeout of %s exceeded", c.dialTimeout) - } - err = errors.WithStack(err) - return - } - return -} diff --git a/cmd/config_fsvfilter.go b/cmd/config_fsvfilter.go deleted file mode 100644 index 8e13dd5..0000000 --- a/cmd/config_fsvfilter.go +++ /dev/null @@ -1,36 +0,0 @@ -package cmd - -import ( - "github.com/pkg/errors" - "github.com/zrepl/zrepl/zfs" - "strings" -) - -type PrefixFilter struct { - prefix string - fstype zfs.VersionType - fstypeSet bool // optionals anyone? -} - -func NewPrefixFilter(prefix string) *PrefixFilter { - return &PrefixFilter{prefix: prefix} -} - -func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter { - return &PrefixFilter{prefix, versionType, true} -} - -func parseSnapshotPrefix(i string) (p string, err error) { - if len(i) <= 0 { - err = errors.Errorf("snapshot prefix must not be empty string") - return - } - p = i - return -} - -func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) { - fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype) - prefixMatches := strings.HasPrefix(fsv.Name, f.prefix) - return fstypeMatches && prefixMatches, nil -} diff --git a/cmd/config_job_control.go b/cmd/config_job_control.go deleted file mode 100644 index 3baf715..0000000 --- a/cmd/config_job_control.go +++ /dev/null @@ -1,145 +0,0 @@ -package cmd - -import ( - "bytes" - "context" - "encoding/json" - "github.com/pkg/errors" - "io" - "net" - "net/http" -) - -type ControlJob struct { - Name string - sockaddr *net.UnixAddr -} - -func NewControlJob(name, sockpath string) (j *ControlJob, err error) { - j = &ControlJob{Name: name} - - j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath) - if err != nil { - err = errors.Wrap(err, "cannot resolve unix address") - return - } - - return -} - -func (j *ControlJob) JobName() string { - return j.Name -} - -func (j *ControlJob) JobType() JobType { return JobTypeControl } - -func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) { - return &JobStatus{Tasks: nil}, nil -} - -const ( - ControlJobEndpointPProf string = "/debug/pprof" - ControlJobEndpointVersion string = "/version" - ControlJobEndpointStatus string = "/status" -) - -func (j *ControlJob) JobStart(ctx context.Context) { - - log := ctx.Value(contextKeyLog).(Logger) - defer log.Info("control job finished") - - daemon := ctx.Value(contextKeyDaemon).(*Daemon) - - l, err := ListenUnixPrivate(j.sockaddr) - if err != nil { - log.WithError(err).Error("error listening") - return - } - - pprofServer := NewPProfServer(ctx) - - mux := http.NewServeMux() - mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) { - var msg PprofServerControlMsg - err := json.NewDecoder(r.Body).Decode(&msg) - if err != nil { - log.WithError(err).Error("bad pprof request from client") - w.WriteHeader(http.StatusBadRequest) - } - pprofServer.Control(msg) - w.WriteHeader(200) - }}) - mux.Handle(ControlJobEndpointVersion, - requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) { - return NewZreplVersionInformation(), nil - }}}) - mux.Handle(ControlJobEndpointStatus, - requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) { - return daemon.Status(), nil - }}}) - server := http.Server{Handler: mux} - -outer: - for { - - served := make(chan error) - go func() { - served <- server.Serve(l) - close(served) - }() - - select { - case <-ctx.Done(): - log.WithError(ctx.Err()).Info("context done") - server.Shutdown(context.Background()) - break outer - case err = <-served: - if err != nil { - log.WithError(err).Error("error serving") - break outer - } - } - - } - -} - -type jsonResponder struct { - producer func() (interface{}, error) -} - -func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) { - res, err := j.producer() - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, err.Error()) - return - } - var buf bytes.Buffer - err = json.NewEncoder(&buf).Encode(res) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, err.Error()) - } else { - io.Copy(w, &buf) - } -} - -type requestLogger struct { - log Logger - handler http.Handler - handlerFunc http.HandlerFunc -} - -func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log := l.log.WithField("method", r.Method).WithField("url", r.URL) - log.Info("start") - if l.handlerFunc != nil { - l.handlerFunc(w, r) - } else if l.handler != nil { - l.handler.ServeHTTP(w, r) - } else { - log.Error("no handler or handlerFunc configured") - } - log.Info("finish") -} diff --git a/cmd/config_job_local.go b/cmd/config_job_local.go deleted file mode 100644 index 7bbe517..0000000 --- a/cmd/config_job_local.go +++ /dev/null @@ -1,216 +0,0 @@ -package cmd - -import ( - "time" - - "context" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/rpc" - "github.com/zrepl/zrepl/zfs" - "sync" -) - -type LocalJob struct { - Name string - Mapping *DatasetMapFilter - SnapshotPrefix string - Interval time.Duration - InitialReplPolicy InitialReplPolicy - PruneLHS PrunePolicy - PruneRHS PrunePolicy - Debug JobDebugSettings - snapperTask *Task - mainTask *Task - handlerTask *Task - pruneRHSTask *Task - pruneLHSTask *Task -} - -func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) { - - var asMap struct { - Mapping map[string]string - SnapshotPrefix string `mapstructure:"snapshot_prefix"` - Interval string - InitialReplPolicy string `mapstructure:"initial_repl_policy"` - PruneLHS map[string]interface{} `mapstructure:"prune_lhs"` - PruneRHS map[string]interface{} `mapstructure:"prune_rhs"` - Debug map[string]interface{} - } - - if err = mapstructure.Decode(i, &asMap); err != nil { - err = errors.Wrap(err, "mapstructure error") - return nil, err - } - - j = &LocalJob{Name: name} - - if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil { - return - } - - if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil { - return - } - - if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil { - err = errors.Wrap(err, "cannot parse interval") - return - } - - if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil { - return - } - - if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil { - err = errors.Wrap(err, "cannot parse 'prune_lhs'") - return - } - if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil { - err = errors.Wrap(err, "cannot parse 'prune_rhs'") - return - } - - if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil { - err = errors.Wrap(err, "cannot parse 'debug'") - return - } - - return -} - -func (j *LocalJob) JobName() string { - return j.Name -} - -func (j *LocalJob) JobType() JobType { return JobTypeLocal } - -func (j *LocalJob) JobStart(ctx context.Context) { - - rootLog := ctx.Value(contextKeyLog).(Logger) - - j.snapperTask = NewTask("snapshot", j, rootLog) - j.mainTask = NewTask("main", j, rootLog) - j.handlerTask = NewTask("handler", j, rootLog) - j.pruneRHSTask = NewTask("prune_rhs", j, rootLog) - j.pruneLHSTask = NewTask("prune_lhs", j, rootLog) - - local := rpc.NewLocalRPC() - // Allow access to any dataset since we control what mapping - // is passed to the pull routine. - // All local datasets will be passed to its Map() function, - // but only those for which a mapping exists will actually be pulled. - // We can pay this small performance penalty for now. - handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix)) - - registerEndpoints(local, handler) - - snapper := IntervalAutosnap{ - task: j.snapperTask, - DatasetFilter: j.Mapping.AsFilter(), - Prefix: j.SnapshotPrefix, - SnapshotInterval: j.Interval, - } - - plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false) - if err != nil { - rootLog.WithError(err).Error("error creating lhs pruner") - return - } - prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false) - if err != nil { - rootLog.WithError(err).Error("error creating rhs pruner") - return - } - - didSnaps := make(chan struct{}) - go snapper.Run(ctx, didSnaps) - -outer: - for { - - select { - case <-ctx.Done(): - j.mainTask.Log().WithError(ctx.Err()).Info("context") - break outer - case <-didSnaps: - j.mainTask.Log().Debug("finished taking snapshots") - j.mainTask.Log().Info("starting replication procedure") - } - - j.mainTask.Log().Debug("replicating from lhs to rhs") - j.mainTask.Enter("replicate") - puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy} - puller.Pull() - j.mainTask.Finish() - - // use a ctx as soon as Pull gains ctx support - select { - case <-ctx.Done(): - break outer - default: - } - - var wg sync.WaitGroup - - j.mainTask.Log().Info("pruning lhs") - wg.Add(1) - go func() { - plhs.Run(ctx) - wg.Done() - }() - - j.mainTask.Log().Info("pruning rhs") - wg.Add(1) - go func() { - prhs.Run(ctx) - wg.Done() - }() - - wg.Wait() - - } - -} - -func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) { - return &JobStatus{Tasks: []*TaskStatus{ - j.snapperTask.Status(), - j.pruneLHSTask.Status(), - j.pruneRHSTask.Status(), - j.mainTask.Status(), - }}, nil -} - -func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) { - - var dsfilter zfs.DatasetFilter - var pp PrunePolicy - switch side { - case PrunePolicySideLeft: - pp = j.PruneLHS - dsfilter = j.Mapping.AsFilter() - case PrunePolicySideRight: - pp = j.PruneRHS - dsfilter, err = j.Mapping.InvertedFilter() - if err != nil { - err = errors.Wrap(err, "cannot invert mapping for prune_rhs") - return - } - default: - err = errors.Errorf("must be either left or right side") - return - } - - p = Pruner{ - task, - time.Now(), - dryRun, - dsfilter, - j.SnapshotPrefix, - pp, - } - - return -} diff --git a/cmd/config_job_prometheus.go b/cmd/config_job_prometheus.go deleted file mode 100644 index 512d01b..0000000 --- a/cmd/config_job_prometheus.go +++ /dev/null @@ -1,99 +0,0 @@ -package cmd - -import ( - "context" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/zrepl/zrepl/zfs" - "net" - "net/http" -) - -type PrometheusJob struct { - Name string - Listen string -} - -var prom struct { - taskLastActiveStart *prometheus.GaugeVec - taskLastActiveDuration *prometheus.GaugeVec - taskLogEntries *prometheus.CounterVec -} - -func init() { - prom.taskLastActiveStart = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "zrepl", - Subsystem: "daemon", - Name: "task_last_active_start", - Help: "point in time at which the job task last left idle state", - }, []string{"zrepl_job", "job_type", "task"}) - prom.taskLastActiveDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "zrepl", - Subsystem: "daemon", - Name: "task_last_active_duration", - Help: "seconds that the last run ob a job task spent between leaving and re-entering idle state", - }, []string{"zrepl_job", "job_type", "task"}) - prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "zrepl", - Subsystem: "daemon", - Name: "task_log_entries", - Help: "number of log entries per job task and level", - }, []string{"zrepl_job", "job_type", "task", "level"}) - prometheus.MustRegister(prom.taskLastActiveStart) - prometheus.MustRegister(prom.taskLastActiveDuration) - prometheus.MustRegister(prom.taskLogEntries) -} - -func parsePrometheusJob(c JobParsingContext, name string, i map[string]interface{}) (j *PrometheusJob, err error) { - var s struct { - Listen string - } - if err := mapstructure.Decode(i, &s); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - if s.Listen == "" { - return nil, errors.New("must specify 'listen' attribute") - } - return &PrometheusJob{name, s.Listen}, nil -} - -func (j *PrometheusJob) JobName() string { return j.Name } - -func (j *PrometheusJob) JobType() JobType { return JobTypePrometheus } - -func (j *PrometheusJob) JobStart(ctx context.Context) { - - if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil { - panic(err) - } - - log := ctx.Value(contextKeyLog).(Logger) - task := NewTask("main", j, log) - log = task.Log() - - l, err := net.Listen("tcp", j.Listen) - if err != nil { - log.WithError(err).Error("cannot listen") - } - go func() { - select { - case <-ctx.Done(): - l.Close() - } - }() - - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - - err = http.Serve(l, mux) - if err != nil { - log.WithError(err).Error("error while serving") - } - -} - -func (*PrometheusJob) JobStatus(ctxt context.Context) (*JobStatus, error) { - return &JobStatus{}, nil -} diff --git a/cmd/config_job_pull.go b/cmd/config_job_pull.go deleted file mode 100644 index c3e2413..0000000 --- a/cmd/config_job_pull.go +++ /dev/null @@ -1,197 +0,0 @@ -package cmd - -import ( - "time" - - "context" - "fmt" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/rpc" - "github.com/zrepl/zrepl/util" -) - -type PullJob struct { - Name string - Connect RWCConnecter - Interval time.Duration - Mapping *DatasetMapFilter - // constructed from mapping during parsing - pruneFilter *DatasetMapFilter - SnapshotPrefix string - InitialReplPolicy InitialReplPolicy - Prune PrunePolicy - Debug JobDebugSettings - - task *Task -} - -func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) { - - var asMap struct { - Connect map[string]interface{} - Interval string - Mapping map[string]string - InitialReplPolicy string `mapstructure:"initial_repl_policy"` - Prune map[string]interface{} - SnapshotPrefix string `mapstructure:"snapshot_prefix"` - Debug map[string]interface{} - } - - if err = mapstructure.Decode(i, &asMap); err != nil { - err = errors.Wrap(err, "mapstructure error") - return nil, err - } - - j = &PullJob{Name: name} - - j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect) - if err != nil { - err = errors.Wrap(err, "cannot parse 'connect'") - return nil, err - } - - if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil { - err = errors.Wrap(err, "cannot parse 'interval'") - return nil, err - } - - j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false) - if err != nil { - err = errors.Wrap(err, "cannot parse 'mapping'") - return nil, err - } - - if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil { - err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job") - return nil, err - } - - j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY) - if err != nil { - err = errors.Wrap(err, "cannot parse 'initial_repl_policy'") - return - } - - if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil { - return - } - - if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil { - err = errors.Wrap(err, "cannot parse prune policy") - return - } - - if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil { - err = errors.Wrap(err, "cannot parse 'debug'") - return - } - - return -} - -func (j *PullJob) JobName() string { - return j.Name -} - -func (j *PullJob) JobType() JobType { return JobTypePull } - -func (j *PullJob) JobStart(ctx context.Context) { - - log := ctx.Value(contextKeyLog).(Logger) - defer log.Info("exiting") - j.task = NewTask("main", j, log) - - // j.task is idle here idle here - - ticker := time.NewTicker(j.Interval) - for { - j.doRun(ctx) - select { - case <-ctx.Done(): - j.task.Log().WithError(ctx.Err()).Info("context") - return - case <-ticker.C: - } - } -} - -func (j *PullJob) doRun(ctx context.Context) { - - j.task.Enter("run") - defer j.task.Finish() - - j.task.Log().Info("connecting") - rwc, err := j.Connect.Connect() - if err != nil { - j.task.Log().WithError(err).Error("error connecting") - return - } - - rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump) - if err != nil { - return - } - - client := rpc.NewClient(rwc) - if j.Debug.RPC.Log { - client.SetLogger(j.task.Log(), true) - } - - j.task.Enter("pull") - puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy} - puller.Pull() - closeRPCWithTimeout(j.task, client, time.Second*1, "") - rwc.Close() - j.task.Finish() - - j.task.Enter("prune") - pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false) - if err != nil { - j.task.Log().WithError(err).Error("error creating pruner") - } else { - pruner.Run(ctx) - } - j.task.Finish() - -} - -func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) { - return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil -} - -func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) { - p = Pruner{ - task, - time.Now(), - dryRun, - j.pruneFilter, - j.SnapshotPrefix, - j.Prune, - } - return -} - -func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) { - - task.Log().Info("closing rpc connection") - - ch := make(chan error) - go func() { - ch <- remote.Close() - close(ch) - }() - - var err error - select { - case <-time.After(timeout): - err = fmt.Errorf("timeout exceeded (%s)", timeout) - case closeRequestErr := <-ch: - err = closeRequestErr - } - - if err != nil { - task.Log().WithError(err).Error("error closing connection") - } - return -} diff --git a/cmd/config_job_source.go b/cmd/config_job_source.go deleted file mode 100644 index 79b8559..0000000 --- a/cmd/config_job_source.go +++ /dev/null @@ -1,252 +0,0 @@ -package cmd - -import ( - "context" - "io" - "time" - - mapstructure "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/rpc" - "github.com/zrepl/zrepl/util" -) - -type SourceJob struct { - Name string - Serve AuthenticatedChannelListenerFactory - Filesystems *DatasetMapFilter - SnapshotPrefix string - Interval time.Duration - Prune PrunePolicy - Debug JobDebugSettings - serveTask *Task - autosnapTask *Task - pruneTask *Task -} - -func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) { - - var asMap struct { - Serve map[string]interface{} - Filesystems map[string]string - SnapshotPrefix string `mapstructure:"snapshot_prefix"` - Interval string - Prune map[string]interface{} - Debug map[string]interface{} - } - - if err = mapstructure.Decode(i, &asMap); err != nil { - err = errors.Wrap(err, "mapstructure error") - return nil, err - } - - j = &SourceJob{Name: name} - - if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil { - return - } - - if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil { - return - } - - if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil { - return - } - - if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil { - err = errors.Wrap(err, "cannot parse 'interval'") - return - } - - if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil { - err = errors.Wrap(err, "cannot parse 'prune'") - return - } - - if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil { - err = errors.Wrap(err, "cannot parse 'debug'") - return - } - - return -} - -func (j *SourceJob) JobName() string { - return j.Name -} - -func (j *SourceJob) JobType() JobType { return JobTypeSource } - -func (j *SourceJob) JobStart(ctx context.Context) { - - log := ctx.Value(contextKeyLog).(Logger) - defer log.Info("exiting") - - j.autosnapTask = NewTask("autosnap", j, log) - j.pruneTask = NewTask("prune", j, log) - j.serveTask = NewTask("serve", j, log) - - a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval} - p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false) - - if err != nil { - log.WithError(err).Error("error creating pruner") - return - } - - didSnaps := make(chan struct{}) - - go j.serve(ctx, j.serveTask) - go a.Run(ctx, didSnaps) - -outer: - for { - select { - case <-ctx.Done(): - break outer - case <-didSnaps: - log.Info("starting pruner") - p.Run(ctx) - log.Info("pruner done") - } - } - log.WithError(ctx.Err()).Info("context") - -} - -func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) { - return &JobStatus{ - Tasks: []*TaskStatus{ - j.autosnapTask.Status(), - j.pruneTask.Status(), - j.serveTask.Status(), - }}, nil -} - -func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) { - p = Pruner{ - task, - time.Now(), - dryRun, - j.Filesystems, - j.SnapshotPrefix, - j.Prune, - } - return -} - -func (j *SourceJob) serve(ctx context.Context, task *Task) { - - listener, err := j.Serve.Listen() - if err != nil { - task.Log().WithError(err).Error("error listening") - return - } - - type rwcChanMsg struct { - rwc io.ReadWriteCloser - err error - } - rwcChan := make(chan rwcChanMsg) - - // Serve connections until interrupted or error -outer: - for { - - go func() { - rwc, err := listener.Accept() - if err != nil { - rwcChan <- rwcChanMsg{rwc, err} - close(rwcChan) - return - } - rwcChan <- rwcChanMsg{rwc, err} - }() - - select { - - case rwcMsg := <-rwcChan: - - if rwcMsg.err != nil { - task.Log().WithError(err).Error("error accepting connection") - break outer - } - - j.handleConnection(rwcMsg.rwc, task) - - case <-ctx.Done(): - task.Log().WithError(ctx.Err()).Info("context") - break outer - - } - - } - - task.Enter("close_listener") - defer task.Finish() - err = listener.Close() - if err != nil { - task.Log().WithError(err).Error("error closing listener") - } - - return - -} - -func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) { - - task.Enter("handle_connection") - defer task.Finish() - - task.Log().Info("handling client connection") - - rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump) - if err != nil { - panic(err) - } - - // construct connection handler - handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix)) - - // handle connection - rpcServer := rpc.NewServer(rwc) - if j.Debug.RPC.Log { - rpclog := task.Log().WithField("subsystem", "rpc") - rpcServer.SetLogger(rpclog, true) - } - registerEndpoints(rpcServer, handler) - if err = rpcServer.Serve(); err != nil { - task.Log().WithError(err).Error("error serving connection") - } - - // wait for client to close connection - // FIXME: we cannot just close it like we would to with a TCP socket because - // FIXME: go-nettsh's Close() may overtake the remaining data in the pipe - const CLIENT_HANGUP_TIMEOUT = 1 * time.Second - task.Log(). - WithField("timeout", CLIENT_HANGUP_TIMEOUT). - Debug("waiting for client to hang up") - - wchan := make(chan error) - go func() { - var pseudo [1]byte - _, err := io.ReadFull(rwc, pseudo[:]) - wchan <- err - }() - var werr error - select { - case werr = <-wchan: - // all right - case <-time.After(CLIENT_HANGUP_TIMEOUT): - werr = errors.New("client did not close connection within timeout") - } - if werr != nil && werr != io.EOF { - task.Log().WithError(werr). - Error("error waiting for client to hang up") - } - task.Log().Info("closing client connection") - if err = rwc.Close(); err != nil { - task.Log().WithError(err).Error("error force-closing connection") - } -} diff --git a/cmd/config_logging.go b/cmd/config_logging.go deleted file mode 100644 index 927fbe6..0000000 --- a/cmd/config_logging.go +++ /dev/null @@ -1,251 +0,0 @@ -package cmd - -import ( - "crypto/tls" - "crypto/x509" - "github.com/mattn/go-isatty" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/logger" - "io/ioutil" - "os" - "time" -) - -type LoggingConfig struct { - Outlets *logger.Outlets -} - -type MetadataFlags int64 - -const ( - MetadataTime MetadataFlags = 1 << iota - MetadataLevel - - MetadataNone MetadataFlags = 0 - MetadataAll MetadataFlags = ^0 -) - -func parseLogging(i interface{}) (c *LoggingConfig, err error) { - - c = &LoggingConfig{} - c.Outlets = logger.NewOutlets() - - var asList []interface{} - if err = mapstructure.Decode(i, &asList); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - if len(asList) == 0 { - // Default config - out := WriterOutlet{&HumanFormatter{}, os.Stdout} - c.Outlets.Add(out, logger.Warn) - return - } - - var syslogOutlets, stdoutOutlets int - for lei, le := range asList { - - outlet, minLevel, err := parseOutlet(le) - if err != nil { - return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei) - } - var _ logger.Outlet = WriterOutlet{} - var _ logger.Outlet = &SyslogOutlet{} - switch outlet.(type) { - case *SyslogOutlet: - syslogOutlets++ - case WriterOutlet: - stdoutOutlets++ - } - - c.Outlets.Add(outlet, minLevel) - - } - - if syslogOutlets > 1 { - return nil, errors.Errorf("can only define one 'syslog' outlet") - } - if stdoutOutlets > 1 { - return nil, errors.Errorf("can only define one 'stdout' outlet") - } - - return c, nil - -} - -func parseLogFormat(i interface{}) (f EntryFormatter, err error) { - var is string - switch j := i.(type) { - case string: - is = j - default: - return nil, errors.Errorf("invalid log format: wrong type: %T", i) - } - - switch is { - case "human": - return &HumanFormatter{}, nil - case "logfmt": - return &LogfmtFormatter{}, nil - case "json": - return &JSONFormatter{}, nil - default: - return nil, errors.Errorf("invalid log format: '%s'", is) - } - -} - -func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) { - - var in struct { - Outlet string - Level string - Format string - } - if err = mapstructure.Decode(i, &in); err != nil { - err = errors.Wrap(err, "mapstructure error") - return - } - if in.Outlet == "" || in.Level == "" || in.Format == "" { - err = errors.Errorf("must specify 'outlet', 'level' and 'format' field") - return - } - - minLevel, err := logger.ParseLevel(in.Level) - if err != nil { - err = errors.Wrap(err, "cannot parse 'level' field") - return - } - formatter, err := parseLogFormat(in.Format) - if err != nil { - err = errors.Wrap(err, "cannot parse") - return - } - - switch in.Outlet { - case "stdout": - o, err = parseStdoutOutlet(i, formatter) - case "tcp": - o, err = parseTCPOutlet(i, formatter) - case "syslog": - o, err = parseSyslogOutlet(i, formatter) - default: - err = errors.Errorf("unknown outlet type '%s'", in.Outlet) - } - return o, minLevel, err - -} - -func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) { - - var in struct { - Time bool - } - if err := mapstructure.Decode(i, &in); err != nil { - return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet") - } - - flags := MetadataAll - writer := os.Stdout - if !isatty.IsTerminal(writer.Fd()) && !in.Time { - flags &= ^MetadataTime - } - - formatter.SetMetadataFlags(flags) - return WriterOutlet{ - formatter, - os.Stdout, - }, nil -} - -func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) { - - var in struct { - Net string - Address string - RetryInterval string `mapstructure:"retry_interval"` - TLS *struct { - CA string - Cert string - Key string - } - } - if err = mapstructure.Decode(i, &in); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - - retryInterval, err := time.ParseDuration(in.RetryInterval) - if err != nil { - return nil, errors.Wrap(err, "cannot parse 'retry_interval'") - } - - if len(in.Net) == 0 { - return nil, errors.New("field 'net' must not be empty") - } - if len(in.Address) == 0 { - return nil, errors.New("field 'address' must not be empty") - } - - var tlsConfig *tls.Config - if in.TLS != nil { - - cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key) - if err != nil { - return nil, errors.Wrap(err, "cannot load client cert") - } - - var rootCAs *x509.CertPool - if in.TLS.CA == "" { - if rootCAs, err = x509.SystemCertPool(); err != nil { - return nil, errors.Wrap(err, "cannot open system cert pool") - } - } else { - rootCAs = x509.NewCertPool() - rootCAPEM, err := ioutil.ReadFile(in.TLS.CA) - if err != nil { - return nil, errors.Wrap(err, "cannot load CA cert") - } - if !rootCAs.AppendCertsFromPEM(rootCAPEM) { - return nil, errors.New("cannot parse CA cert") - } - } - if err != nil && in.TLS.CA == "" { - return nil, errors.Wrap(err, "cannot load root ca pool") - } - - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: rootCAs, - } - - tlsConfig.BuildNameToCertificate() - } - - formatter.SetMetadataFlags(MetadataAll) - return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil - -} - -func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) { - - var in struct { - RetryInterval string `mapstructure:"retry_interval"` - } - if err = mapstructure.Decode(i, &in); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - - out = &SyslogOutlet{} - out.Formatter = formatter - out.Formatter.SetMetadataFlags(MetadataNone) - - out.RetryInterval = 0 // default to 0 as we assume local syslog will just work - if in.RetryInterval != "" { - out.RetryInterval, err = time.ParseDuration(in.RetryInterval) - if err != nil { - return nil, errors.Wrap(err, "cannot parse 'retry_interval'") - } - } - - return -} diff --git a/cmd/config_parse.go b/cmd/config_parse.go deleted file mode 100644 index cd733eb..0000000 --- a/cmd/config_parse.go +++ /dev/null @@ -1,322 +0,0 @@ -package cmd - -import ( - "io/ioutil" - - "fmt" - yaml "github.com/go-yaml/yaml" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "os" - "regexp" - "strconv" - "time" -) - -var ConfigFileDefaultLocations []string = []string{ - "/etc/zrepl/zrepl.yml", - "/usr/local/etc/zrepl/zrepl.yml", -} - -const ( - JobNameControl string = "control" -) - -var ReservedJobNames []string = []string{ - JobNameControl, -} - -type ConfigParsingContext struct { - Global *Global -} - -func ParseConfig(path string) (config *Config, err error) { - - if path == "" { - // Try default locations - for _, l := range ConfigFileDefaultLocations { - stat, err := os.Stat(l) - if err != nil { - continue - } - if !stat.Mode().IsRegular() { - err = errors.Errorf("file at default location is not a regular file: %s", l) - continue - } - path = l - break - } - } - - var i interface{} - - var bytes []byte - - if bytes, err = ioutil.ReadFile(path); err != nil { - err = errors.WithStack(err) - return - } - - if err = yaml.Unmarshal(bytes, &i); err != nil { - err = errors.WithStack(err) - return - } - - return parseConfig(i) -} - -func parseConfig(i interface{}) (c *Config, err error) { - - var asMap struct { - Global map[string]interface{} - Jobs []map[string]interface{} - } - if err := mapstructure.Decode(i, &asMap); err != nil { - return nil, errors.Wrap(err, "config root must be a dict") - } - - c = &Config{} - - // Parse global with defaults - c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver" - c.Global.Control.Sockpath = "/var/run/zrepl/control" - - err = mapstructure.Decode(asMap.Global, &c.Global) - if err != nil { - err = errors.Wrap(err, "mapstructure error on 'global' section: %s") - return - } - - if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil { - return nil, errors.Wrap(err, "cannot parse logging section") - } - - cpc := ConfigParsingContext{&c.Global} - jpc := JobParsingContext{cpc} - c.Jobs = make(map[string]Job, len(asMap.Jobs)) - - // FIXME internal jobs should not be mixed with user jobs - // Monitoring Jobs - var monJobs []map[string]interface{} - if err := mapstructure.Decode(asMap.Global["monitoring"], &monJobs); err != nil { - return nil, errors.Wrap(err, "cannot parse monitoring section") - } - for i, jc := range monJobs { - if jc["name"] == "" || jc["name"] == nil { - // FIXME internal jobs should not require a name... - jc["name"] = fmt.Sprintf("prometheus-%d", i) - } - job, err := parseJob(jpc, jc) - if err != nil { - return nil, errors.Wrapf(err, "cannot parse monitoring job #%d", i) - } - if job.JobType() != JobTypePrometheus { - return nil, errors.Errorf("monitoring job #%d has invalid job type", i) - } - c.Jobs[job.JobName()] = job - } - - // Regular Jobs - for i := range asMap.Jobs { - job, err := parseJob(jpc, asMap.Jobs[i]) - if err != nil { - // Try to find its name - namei, ok := asMap.Jobs[i]["name"] - if !ok { - namei = fmt.Sprintf("", i) - } - err = errors.Wrapf(err, "cannot parse job '%v'", namei) - return nil, err - } - jn := job.JobName() - if _, ok := c.Jobs[jn]; ok { - err = errors.Errorf("duplicate or invalid job name: %s", jn) - return nil, err - } - c.Jobs[job.JobName()] = job - } - - cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath) - if err != nil { - err = errors.Wrap(err, "cannot create control job") - return - } - c.Jobs[JobNameControl] = cj - - return c, nil - -} - -func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) { - vi, ok := i[key] - if !ok { - err = errors.Errorf("must have field '%s'", key) - return "", err - } - field, ok = vi.(string) - if !ok { - err = errors.Errorf("'%s' field must have type string", key) - return "", err - } - if notempty && len(field) <= 0 { - err = errors.Errorf("'%s' field must not be empty", key) - return "", err - } - return -} - -type JobParsingContext struct { - ConfigParsingContext -} - -func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) { - - name, err := extractStringField(i, "name", true) - if err != nil { - return nil, err - } - - for _, r := range ReservedJobNames { - if name == r { - err = errors.Errorf("job name '%s' is reserved", name) - return nil, err - } - } - - jobtypeStr, err := extractStringField(i, "type", true) - if err != nil { - return nil, err - } - - jobtype, err := ParseUserJobType(jobtypeStr) - if err != nil { - return nil, err - } - - switch jobtype { - case JobTypePull: - return parsePullJob(c, name, i) - case JobTypeSource: - return parseSourceJob(c, name, i) - case JobTypeLocal: - return parseLocalJob(c, name, i) - case JobTypePrometheus: - return parsePrometheusJob(c, name, i) - default: - panic(fmt.Sprintf("implementation error: unknown job type %s", jobtype)) - } - -} - -func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) { - - t, err := extractStringField(i, "type", true) - if err != nil { - return nil, err - } - - switch t { - case "ssh+stdinserver": - return parseSSHStdinserverConnecter(i) - default: - return nil, errors.Errorf("unknown connection type '%s'", t) - } - -} - -func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) { - s, ok := v.(string) - if !ok { - goto err - } - - switch { - case s == "": - p = defaultPolicy - case s == "most_recent": - p = InitialReplPolicyMostRecent - case s == "all": - p = InitialReplPolicyAll - default: - goto err - } - - return - -err: - err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v)) - return -} - -func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) { - - policyName, err := extractStringField(v, "policy", true) - if err != nil { - return - } - - switch policyName { - case "grid": - return parseGridPrunePolicy(v, willSeeBookmarks) - case "noprune": - return NoPrunePolicy{}, nil - default: - err = errors.Errorf("unknown policy '%s'", policyName) - return - } -} - -func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) { - - t, err := extractStringField(v, "type", true) - if err != nil { - return nil, err - } - - switch t { - case "stdinserver": - return parseStdinserverListenerFactory(c, v) - default: - err = errors.Errorf("unknown type '%s'", t) - return - } - -} - -var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`) - -func parsePostitiveDuration(e string) (d time.Duration, err error) { - comps := durationStringRegex.FindStringSubmatch(e) - if len(comps) != 3 { - err = fmt.Errorf("does not match regex: %s %#v", e, comps) - return - } - - durationFactor, err := strconv.ParseInt(comps[1], 10, 64) - if err != nil { - return 0, err - } - if durationFactor <= 0 { - return 0, errors.New("duration must be positive integer") - } - - var durationUnit time.Duration - switch comps[2] { - case "s": - durationUnit = time.Second - case "m": - durationUnit = time.Minute - case "h": - durationUnit = time.Hour - case "d": - durationUnit = 24 * time.Hour - case "w": - durationUnit = 24 * 7 * time.Hour - default: - err = fmt.Errorf("contains unknown time unit '%s'", comps[2]) - return - } - - d = time.Duration(durationFactor) * durationUnit - return -} diff --git a/cmd/config_prune_grid.go b/cmd/config_prune_grid.go deleted file mode 100644 index 5b567f3..0000000 --- a/cmd/config_prune_grid.go +++ /dev/null @@ -1,245 +0,0 @@ -package cmd - -import ( - "fmt" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/util" - "github.com/zrepl/zrepl/zfs" - "math" - "regexp" - "sort" - "strconv" - "strings" - "time" -) - -type GridPrunePolicy struct { - RetentionGrid *util.RetentionGrid - MaxBookmarks int -} - -const GridPrunePolicyMaxBookmarksKeepAll = -1 - -type retentionGridAdaptor struct { - zfs.FilesystemVersion -} - -func (a retentionGridAdaptor) Date() time.Time { - return a.Creation -} - -func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool { - return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG -} - -// Prune filters snapshots with the retention grid. -// Bookmarks are deleted such that KeepBookmarks are kept in the end. -// The oldest bookmarks are removed first. -func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) { - skeep, sremove := p.pruneSnapshots(versions) - keep, remove = p.pruneBookmarks(skeep) - remove = append(remove, sremove...) - return keep, remove, nil -} - -func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) { - - // Build adaptors for retention grid - keep = []zfs.FilesystemVersion{} - adaptors := make([]util.RetentionGridEntry, 0) - for fsv := range versions { - if versions[fsv].Type != zfs.Snapshot { - keep = append(keep, versions[fsv]) - continue - } - adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]}) - } - - sort.SliceStable(adaptors, func(i, j int) bool { - return adaptors[i].LessThan(adaptors[j]) - }) - now := adaptors[len(adaptors)-1].Date() - - // Evaluate retention grid - keepa, removea := p.RetentionGrid.FitEntries(now, adaptors) - - // Revert adaptors - for i := range keepa { - keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion) - } - remove = make([]zfs.FilesystemVersion, len(removea)) - for i := range removea { - remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion - } - return - -} - -func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) { - - if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll { - return versions, []zfs.FilesystemVersion{} - } - - keep = []zfs.FilesystemVersion{} - bookmarks := make([]zfs.FilesystemVersion, 0) - for fsv := range versions { - if versions[fsv].Type != zfs.Bookmark { - keep = append(keep, versions[fsv]) - continue - } - bookmarks = append(bookmarks, versions[fsv]) - } - - if len(bookmarks) == 0 { - return keep, []zfs.FilesystemVersion{} - } - if len(bookmarks) < p.MaxBookmarks { - keep = append(keep, bookmarks...) - return keep, []zfs.FilesystemVersion{} - } - - // NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time - sort.SliceStable(bookmarks, func(i, j int) bool { - return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG) - }) - - keep = append(keep, bookmarks[:p.MaxBookmarks]...) - remove = bookmarks[p.MaxBookmarks:] - - return keep, remove -} - -func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) { - - const KeepBookmarksAllString = "all" - var i struct { - Grid string - KeepBookmarks string `mapstructure:"keep_bookmarks"` - } - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true}) - if err != nil { - err = errors.Wrap(err, "mapstructure error") - return - } - if err = dec.Decode(e); err != nil { - err = errors.Wrapf(err, "mapstructure error") - return - } - - // Parse grid - intervals, err := parseRetentionGridIntervalsString(i.Grid) - if err != nil { - err = fmt.Errorf("cannot parse retention grid: %s", err) - return - } - // Assert intervals are of increasing length (not necessarily required, but indicates config mistake) - lastDuration := time.Duration(0) - for i := range intervals { - - if intervals[i].Length < lastDuration { - // If all intervals before were keep=all, this is ok - allPrevKeepCountAll := true - for j := i - 1; allPrevKeepCountAll && j >= 0; j-- { - allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll - } - if allPrevKeepCountAll { - goto isMonotonicIncrease - } - err = errors.New("retention grid interval length must be monotonically increasing") - return - } - isMonotonicIncrease: - lastDuration = intervals[i].Length - - } - - // Parse KeepBookmarks - keepBookmarks := 0 - if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) { - keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll - } else { - i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32) - if err != nil || i <= 0 || i > math.MaxInt32 { - return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'") - } - keepBookmarks = int(i) - } - return &GridPrunePolicy{ - util.NewRetentionGrid(intervals), - keepBookmarks, - }, nil -} - -var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`) - -func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) { - - comps := retentionStringIntervalRegex.FindStringSubmatch(e) - if comps == nil { - err = fmt.Errorf("retention string does not match expected format") - return - } - - times, err := strconv.Atoi(comps[1]) - if err != nil { - return nil, err - } else if times <= 0 { - return nil, fmt.Errorf("contains factor <= 0") - } - - duration, err := parsePostitiveDuration(comps[2]) - if err != nil { - return nil, err - } - - keepCount := 1 - if comps[3] != "" { - // Decompose key=value, comma separated - // For now, only keep_count is supported - re := regexp.MustCompile(`^\s*keep=(.+)\s*$`) - res := re.FindStringSubmatch(comps[4]) - if res == nil || len(res) != 2 { - err = fmt.Errorf("interval parameter contains unknown parameters") - return - } - if res[1] == "all" { - keepCount = util.RetentionGridKeepCountAll - } else { - keepCount, err = strconv.Atoi(res[1]) - if err != nil { - err = fmt.Errorf("cannot parse keep_count value") - return - } - } - } - - intervals = make([]util.RetentionInterval, times) - for i := range intervals { - intervals[i] = util.RetentionInterval{ - Length: duration, - KeepCount: keepCount, - } - } - - return - -} - -func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) { - - ges := strings.Split(s, "|") - intervals = make([]util.RetentionInterval, 0, 7*len(ges)) - - for intervalIdx, e := range ges { - parsed, err := parseRetentionGridIntervalString(e) - if err != nil { - return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e)) - } - intervals = append(intervals, parsed...) - } - - return -} diff --git a/cmd/config_prune_none.go b/cmd/config_prune_none.go deleted file mode 100644 index 785941b..0000000 --- a/cmd/config_prune_none.go +++ /dev/null @@ -1,11 +0,0 @@ -package cmd - -import "github.com/zrepl/zrepl/zfs" - -type NoPrunePolicy struct{} - -func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) { - keep = versions - remove = []zfs.FilesystemVersion{} - return -} diff --git a/cmd/config_serve_stdinserver.go b/cmd/config_serve_stdinserver.go deleted file mode 100644 index ed7b68d..0000000 --- a/cmd/config_serve_stdinserver.go +++ /dev/null @@ -1,56 +0,0 @@ -package cmd - -import ( - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/problame/go-netssh" - "io" - "path" -) - -type StdinserverListenerFactory struct { - ClientIdentity string `mapstructure:"client_identity"` - sockpath string -} - -func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) { - - f = &StdinserverListenerFactory{} - - if err = mapstructure.Decode(i, f); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - if !(len(f.ClientIdentity) > 0) { - err = errors.Errorf("must specify 'client_identity'") - return - } - - f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity) - - return -} - -func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) { - - if err = PreparePrivateSockpath(f.sockpath); err != nil { - return nil, err - } - - l, err := netssh.Listen(f.sockpath) - if err != nil { - return nil, err - } - return StdinserverListener{l}, nil -} - -type StdinserverListener struct { - l *netssh.Listener -} - -func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) { - return l.l.Accept() -} - -func (l StdinserverListener) Close() (err error) { - return l.l.Close() -} diff --git a/cmd/config_test.go b/cmd/config_test.go deleted file mode 100644 index 5f666a1..0000000 --- a/cmd/config_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package cmd - -import ( - "testing" - "time" - - "github.com/kr/pretty" - "github.com/stretchr/testify/assert" - "github.com/zrepl/zrepl/util" - "github.com/zrepl/zrepl/zfs" -) - -func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { - - paths := []string{ - "./sampleconf/localbackup/host1.yml", - "./sampleconf/pullbackup/backuphost.yml", - "./sampleconf/pullbackup/productionhost.yml", - "./sampleconf/random/debugging.yml", - "./sampleconf/random/logging_and_monitoring.yml", - } - - for _, p := range paths { - - c, err := ParseConfig(p) - if err != nil { - t.Errorf("error parsing %s:\n%+v", p, err) - } - - t.Logf("file: %s", p) - t.Log(pretty.Sprint(c)) - - } - -} - -func TestParseRetentionGridStringParsing(t *testing.T) { - - intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w") - - assert.Nil(t, err) - assert.Len(t, intervals, 6) - proto := util.RetentionInterval{ - KeepCount: 2, - Length: 10 * time.Minute, - } - assert.EqualValues(t, proto, intervals[0]) - assert.EqualValues(t, proto, intervals[1]) - - proto.KeepCount = 1 - proto.Length = 1 * time.Hour - assert.EqualValues(t, proto, intervals[2]) - - proto.Length = 7 * 24 * time.Hour - assert.EqualValues(t, proto, intervals[3]) - assert.EqualValues(t, proto, intervals[4]) - assert.EqualValues(t, proto, intervals[5]) - - intervals, err = parseRetentionGridIntervalsString("|") - assert.Error(t, err) - intervals, err = parseRetentionGridIntervalsString("2x10m") - assert.NoError(t, err) - - intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)") - assert.NoError(t, err) - assert.Len(t, intervals, 1) - assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount) - -} - -func TestDatasetMapFilter(t *testing.T) { - - expectMapping := func(m map[string]string, from, to string) { - dmf, err := parseDatasetMapFilter(m, false) - if err != nil { - t.Logf("expect test map to be valid: %s", err) - t.FailNow() - } - fromPath, err := zfs.NewDatasetPath(from) - if err != nil { - t.Logf("expect test from path to be valid: %s", err) - t.FailNow() - } - - res, err := dmf.Map(fromPath) - if to == "" { - assert.Nil(t, res) - assert.Nil(t, err) - t.Logf("%s => NOT MAPPED", fromPath.ToString()) - return - } - - assert.Nil(t, err) - toPath, err := zfs.NewDatasetPath(to) - if err != nil { - t.Logf("expect test to path to be valid: %s", err) - t.FailNow() - } - assert.True(t, res.Equal(toPath)) - } - - expectFilter := func(m map[string]string, path string, pass bool) { - dmf, err := parseDatasetMapFilter(m, true) - if err != nil { - t.Logf("expect test filter to be valid: %s", err) - t.FailNow() - } - p, err := zfs.NewDatasetPath(path) - if err != nil { - t.Logf("expect test path to be valid: %s", err) - t.FailNow() - } - res, err := dmf.Filter(p) - assert.Nil(t, err) - assert.Equal(t, pass, res) - } - - map1 := map[string]string{ - "a/b/c<": "root1", - "a/b<": "root2", - "<": "root3/b/c", - "b": "!", - "a/b/c/d/e<": "!", - "q<": "root4/1/2", - } - - expectMapping(map1, "a/b/c", "root1") - expectMapping(map1, "a/b/c/d", "root1/d") - expectMapping(map1, "a/b/c/d/e", "") - expectMapping(map1, "a/b/e", "root2/e") - expectMapping(map1, "a/b", "root2") - expectMapping(map1, "x", "root3/b/c/x") - expectMapping(map1, "x/y", "root3/b/c/x/y") - expectMapping(map1, "q", "root4/1/2") - expectMapping(map1, "b", "") - expectMapping(map1, "q/r", "root4/1/2/r") - - filter1 := map[string]string{ - "<": "!", - "a<": "ok", - "a/b<": "!", - } - - expectFilter(filter1, "b", false) - expectFilter(filter1, "a", true) - expectFilter(filter1, "a/d", true) - expectFilter(filter1, "a/b", false) - expectFilter(filter1, "a/b/c", false) - - filter2 := map[string]string{} - expectFilter(filter2, "foo", false) // default to omit - -} - -func TestDatasetMapFilter_AsFilter(t *testing.T) { - - mapspec := map[string]string{ - "a/b/c<": "root1", - "a/b<": "root2", - "<": "root3/b/c", - "b": "!", - "a/b/c/d/e<": "!", - "q<": "root4/1/2", - } - - m, err := parseDatasetMapFilter(mapspec, false) - assert.Nil(t, err) - - f := m.AsFilter() - - t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f)) - - tf := func(f zfs.DatasetFilter, path string, pass bool) { - p, err := zfs.NewDatasetPath(path) - assert.Nil(t, err) - r, err := f.Filter(p) - assert.Nil(t, err) - assert.Equal(t, pass, r) - } - - tf(f, "a/b/c", true) - tf(f, "a/b", true) - tf(f, "b", false) - tf(f, "a/b/c/d/e", false) - tf(f, "a/b/c/d/e/f", false) - tf(f, "a", true) - -} - -func TestDatasetMapFilter_InvertedFilter(t *testing.T) { - mapspec := map[string]string{ - "a/b": "1/2", - "a/b/c<": "3", - "a/b/c/d<": "1/2/a", - "a/b/d": "!", - } - - m, err := parseDatasetMapFilter(mapspec, false) - assert.Nil(t, err) - - inv, err := m.InvertedFilter() - assert.Nil(t, err) - - t.Log(pretty.Sprint(inv)) - - expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) { - p, err := zfs.NewDatasetPath(ps) - assert.Nil(t, err) - r, err := m.Filter(p) - assert.Nil(t, err) - assert.Equal(t, expRes, r) - } - - expectMapping(inv, "4", false) - expectMapping(inv, "3", true) - expectMapping(inv, "3/x", true) - expectMapping(inv, "1", false) - expectMapping(inv, "1/2", true) - expectMapping(inv, "1/2/3", false) - expectMapping(inv, "1/2/a/b", true) - -} diff --git a/cmd/control.go b/cmd/control.go deleted file mode 100644 index 3c9e7df..0000000 --- a/cmd/control.go +++ /dev/null @@ -1,305 +0,0 @@ -package cmd - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/dustin/go-humanize" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/zrepl/zrepl/logger" - "io" - golog "log" - "net" - "net/http" - "os" - "sort" - "strings" - "time" -) - -var controlCmd = &cobra.Command{ - Use: "control", - Short: "control zrepl daemon", -} - -var pprofCmd = &cobra.Command{ - Use: "pprof off | [on TCP_LISTEN_ADDRESS]", - Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS", - Run: doControlPProf, - PreRunE: func(cmd *cobra.Command, args []string) error { - if cmd.Flags().NArg() < 1 { - goto enargs - } - switch cmd.Flags().Arg(0) { - case "on": - pprofCmdArgs.msg.Run = true - if cmd.Flags().NArg() != 2 { - return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument") - } - pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1) - case "off": - if cmd.Flags().NArg() != 1 { - goto enargs - } - pprofCmdArgs.msg.Run = false - } - return nil - enargs: - return errors.New("invalid number of positional arguments") - - }, -} -var pprofCmdArgs struct { - msg PprofServerControlMsg -} - -var controlVersionCmd = &cobra.Command{ - Use: "version", - Short: "print version of running zrepl daemon", - Run: doControLVersionCmd, -} - -var controlStatusCmdArgs struct { - format string - level logger.Level - onlyShowJob string -} - -var controlStatusCmd = &cobra.Command{ - Use: "status [JOB_NAME]", - Short: "get current status", - Run: doControlStatusCmd, -} - -func init() { - RootCmd.AddCommand(controlCmd) - controlCmd.AddCommand(pprofCmd) - controlCmd.AddCommand(controlVersionCmd) - controlCmd.AddCommand(controlStatusCmd) - controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)") - controlStatusCmdArgs.level = logger.Warn - controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show") -} - -func controlHttpClient() (client http.Client, err error) { - - conf, err := ParseConfig(rootArgs.configFile) - if err != nil { - return http.Client{}, err - } - - return http.Client{ - Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", conf.Global.Control.Sockpath) - }, - }, - }, nil -} - -func doControlPProf(cmd *cobra.Command, args []string) { - - log := golog.New(os.Stderr, "", 0) - - die := func() { - log.Printf("exiting after error") - os.Exit(1) - } - - log.Printf("connecting to zrepl daemon") - httpc, err := controlHttpClient() - if err != nil { - log.Printf("error parsing config: %s", err) - die() - } - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil { - log.Printf("error marshaling request: %s", err) - die() - } - _, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf) - if err != nil { - log.Printf("error: %s", err) - die() - } - - log.Printf("finished") -} - -func doControLVersionCmd(cmd *cobra.Command, args []string) { - - log := golog.New(os.Stderr, "", 0) - - die := func() { - log.Printf("exiting after error") - os.Exit(1) - } - - httpc, err := controlHttpClient() - if err != nil { - log.Printf("could not connect to daemon: %s", err) - die() - } - - resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion) - if err != nil { - log.Printf("error: %s", err) - die() - } else if resp.StatusCode != http.StatusOK { - var msg bytes.Buffer - io.CopyN(&msg, resp.Body, 4096) - log.Printf("error: %s", msg.String()) - die() - } - - var info ZreplVersionInformation - err = json.NewDecoder(resp.Body).Decode(&info) - if err != nil { - log.Printf("error unmarshaling response: %s", err) - die() - } - - fmt.Println(info.String()) - -} - -func doControlStatusCmd(cmd *cobra.Command, args []string) { - - log := golog.New(os.Stderr, "", 0) - - die := func() { - log.Print("exiting after error") - os.Exit(1) - } - - if len(args) == 1 { - controlStatusCmdArgs.onlyShowJob = args[0] - } else if len(args) > 1 { - log.Print("can only specify one job as positional argument") - cmd.Usage() - die() - } - - httpc, err := controlHttpClient() - if err != nil { - log.Printf("could not connect to daemon: %s", err) - die() - } - - resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus) - if err != nil { - log.Printf("error: %s", err) - die() - } else if resp.StatusCode != http.StatusOK { - var msg bytes.Buffer - io.CopyN(&msg, resp.Body, 4096) - log.Printf("error: %s", msg.String()) - die() - } - - var status DaemonStatus - err = json.NewDecoder(resp.Body).Decode(&status) - if err != nil { - log.Printf("error unmarshaling response: %s", err) - die() - } - - switch controlStatusCmdArgs.format { - case "raw": - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - if err := enc.Encode(status); err != nil { - log.Panic(err) - } - case "human": - - formatter := HumanFormatter{} - formatter.SetMetadataFlags(MetadataAll) - formatter.SetIgnoreFields([]string{ - logJobField, - }) - jobNames := make([]string, 0, len(status.Jobs)) - for name, _ := range status.Jobs { - jobNames = append(jobNames, name) - } - sort.Slice(jobNames, func(i, j int) bool { - return strings.Compare(jobNames[i], jobNames[j]) == -1 - }) - now := time.Now() - for _, name := range jobNames { - - if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob { - continue - } - - job := status.Jobs[name] - jobLogEntries := make([]logger.Entry, 0) - informAboutError := false - - fmt.Printf("Job '%s':\n", name) - for _, task := range job.Tasks { - - var header bytes.Buffer - fmt.Fprintf(&header, " Task '%s': ", task.Name) - if !task.Idle { - fmt.Fprint(&header, strings.Join(task.ActivityStack, ".")) - } else { - fmt.Fprint(&header, "") - } - fmt.Fprint(&header, " ") - const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second - sinceLastUpdate := now.Sub(task.LastUpdate) - if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 { - fmt.Fprintf(&header, "(%s / %s , Rx/Tx", - humanize.Bytes(uint64(task.ProgressRx)), - humanize.Bytes(uint64(task.ProgressTx))) - if task.Idle { - fmt.Fprint(&header, ", values from last run") - } - fmt.Fprint(&header, ")") - } - fmt.Fprint(&header, "\n") - if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION { - informAboutError = true - fmt.Fprintf(&header, " WARNING: last update %s ago at %s)", - sinceLastUpdate.String(), - task.LastUpdate.Format(HumanFormatterDateFormat)) - fmt.Fprint(&header, "\n") - } - io.Copy(os.Stdout, &header) - - jobLogEntries = append(jobLogEntries, task.LogEntries...) - informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn - } - - sort.Slice(jobLogEntries, func(i, j int) bool { - return jobLogEntries[i].Time.Before(jobLogEntries[j].Time) - }) - if informAboutError { - fmt.Println(" WARNING: Some tasks encountered problems since the last time they left idle state:") - fmt.Println(" check the logs below or your log file for more information.") - fmt.Println(" Use the --level flag if you need debug information.") - fmt.Println() - } - for _, e := range jobLogEntries { - if e.Level < controlStatusCmdArgs.level { - continue - } - formatted, err := formatter.Format(&e) - if err != nil { - panic(err) - } - fmt.Printf(" %s\n", string(formatted)) - } - fmt.Println() - - } - default: - log.Printf("invalid output format '%s'", controlStatusCmdArgs.format) - die() - } - -} diff --git a/cmd/daemon.go b/cmd/daemon.go deleted file mode 100644 index 46746a2..0000000 --- a/cmd/daemon.go +++ /dev/null @@ -1,461 +0,0 @@ -package cmd - -import ( - "container/list" - "context" - "fmt" - "github.com/spf13/cobra" - "github.com/zrepl/zrepl/logger" - "io" - "os" - "os/signal" - "strings" - "sync" - "syscall" - "time" -) - -// daemonCmd represents the daemon command -var daemonCmd = &cobra.Command{ - Use: "daemon", - Short: "start daemon", - Run: doDaemon, -} - -func init() { - RootCmd.AddCommand(daemonCmd) -} - -type Job interface { - JobName() string - JobType() JobType - JobStart(ctxt context.Context) - JobStatus(ctxt context.Context) (*JobStatus, error) -} - -type JobType string - -const ( - JobTypePull JobType = "pull" - JobTypeSource JobType = "source" - JobTypeLocal JobType = "local" - JobTypePrometheus JobType = "prometheus" - JobTypeControl JobType = "control" -) - -func ParseUserJobType(s string) (JobType, error) { - switch s { - case "pull": - return JobTypePull, nil - case "source": - return JobTypeSource, nil - case "local": - return JobTypeLocal, nil - case "prometheus": - return JobTypePrometheus, nil - } - return "", fmt.Errorf("unknown job type '%s'", s) -} - -func (j JobType) String() string { - return string(j) -} - -func doDaemon(cmd *cobra.Command, args []string) { - - conf, err := ParseConfig(rootArgs.configFile) - if err != nil { - fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err) - os.Exit(1) - } - - log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second) - - log.Info(NewZreplVersionInformation().String()) - log.Debug("starting daemon") - ctx := context.WithValue(context.Background(), contextKeyLog, log) - ctx = context.WithValue(ctx, contextKeyLog, log) - - d := NewDaemon(conf) - d.Loop(ctx) - -} - -type contextKey string - -const ( - contextKeyLog contextKey = contextKey("log") - contextKeyDaemon contextKey = contextKey("daemon") -) - -type Daemon struct { - conf *Config - startedAt time.Time -} - -func NewDaemon(initialConf *Config) *Daemon { - return &Daemon{conf: initialConf} -} - -func (d *Daemon) Loop(ctx context.Context) { - - d.startedAt = time.Now() - - log := ctx.Value(contextKeyLog).(Logger) - - ctx, cancel := context.WithCancel(ctx) - ctx = context.WithValue(ctx, contextKeyDaemon, d) - - sigChan := make(chan os.Signal, 1) - finishs := make(chan Job) - - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - log.Info("starting jobs from config") - i := 0 - for _, job := range d.conf.Jobs { - logger := log.WithField(logJobField, job.JobName()) - logger.Info("starting") - i++ - jobCtx := context.WithValue(ctx, contextKeyLog, logger) - go func(j Job) { - j.JobStart(jobCtx) - finishs <- j - }(job) - } - - finishCount := 0 -outer: - for { - select { - case <-finishs: - finishCount++ - if finishCount == len(d.conf.Jobs) { - log.Info("all jobs finished") - break outer - } - - case sig := <-sigChan: - log.WithField("signal", sig).Info("received signal") - log.Info("cancelling all jobs") - cancel() - } - } - - signal.Stop(sigChan) - cancel() // make go vet happy - - log.Info("exiting") - -} - -// Representation of a Job's status that is composed of Tasks -type JobStatus struct { - // Statuses of all tasks of this job - Tasks []*TaskStatus - // Error != "" if JobStatus() returned an error - JobStatusError string -} - -// Representation of a Daemon's status that is composed of Jobs -type DaemonStatus struct { - StartedAt time.Time - Jobs map[string]*JobStatus -} - -func (d *Daemon) Status() (s *DaemonStatus) { - - s = &DaemonStatus{} - s.StartedAt = d.startedAt - - s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs)) - - for name, j := range d.conf.Jobs { - status, err := j.JobStatus(context.TODO()) - if err != nil { - s.Jobs[name] = &JobStatus{nil, err.Error()} - continue - } - s.Jobs[name] = status - } - - return -} - -// Representation of a Task's status -type TaskStatus struct { - Name string - // Whether the task is idle. - Idle bool - // The stack of activities the task is currently executing. - // The first element is the root activity and equal to Name. - ActivityStack []string - // Number of bytes received by the task since it last left idle state. - ProgressRx int64 - // Number of bytes sent by the task since it last left idle state. - ProgressTx int64 - // Log entries emitted by the task since it last left idle state. - // Only contains the log entries emitted through the task's logger - // (provided by Task.Log()). - LogEntries []logger.Entry - // The maximum log level of LogEntries. - // Only valid if len(LogEntries) > 0. - MaxLogLevel logger.Level - // Last time something about the Task changed - LastUpdate time.Time -} - -// An instance of Task tracks a single thread of activity that is part of a Job. -type Task struct { - name string // immutable - parent Job // immutable - - // Stack of activities the task is currently in - // Members are instances of taskActivity - activities *list.List - // Last time activities was changed (not the activities inside, the list) - activitiesLastUpdate time.Time - // Protects Task members from modification - rwl sync.RWMutex -} - -// Structure that describes the progress a Task has made -type taskProgress struct { - rx int64 - tx int64 - creation time.Time - lastUpdate time.Time - logEntries []logger.Entry - mtx sync.RWMutex -} - -func newTaskProgress() (p *taskProgress) { - return &taskProgress{ - creation: time.Now(), - logEntries: make([]logger.Entry, 0), - } -} - -func (p *taskProgress) UpdateIO(drx, dtx int64) { - p.mtx.Lock() - defer p.mtx.Unlock() - p.rx += drx - p.tx += dtx - p.lastUpdate = time.Now() -} - -func (p *taskProgress) UpdateLogEntry(entry logger.Entry) { - p.mtx.Lock() - defer p.mtx.Unlock() - // FIXME: ensure maximum size (issue #48) - p.logEntries = append(p.logEntries, entry) - p.lastUpdate = time.Now() -} - -func (p *taskProgress) DeepCopy() (out taskProgress) { - p.mtx.RLock() - defer p.mtx.RUnlock() - out.rx, out.tx = p.rx, p.tx - out.creation = p.creation - out.lastUpdate = p.lastUpdate - out.logEntries = make([]logger.Entry, len(p.logEntries)) - for i := range p.logEntries { - out.logEntries[i] = p.logEntries[i] - } - return -} - -// returns a copy of this taskProgress, the mutex carries no semantic value -func (p *taskProgress) Read() (out taskProgress) { - p.mtx.RLock() - defer p.mtx.RUnlock() - return p.DeepCopy() -} - -// Element of a Task's activity stack -type taskActivity struct { - name string - idle bool - logger *logger.Logger - // The progress of the task that is updated by UpdateIO() and UpdateLogEntry() - // - // Progress happens on a task-level and is thus global to the task. - // That's why progress is just a pointer to the current taskProgress: - // we reset progress when leaving the idle root activity - progress *taskProgress -} - -func NewTask(name string, parent Job, lg *logger.Logger) *Task { - t := &Task{ - name: name, - parent: parent, - activities: list.New(), - } - rootLogger := lg.ReplaceField(logTaskField, name). - WithOutlet(t, logger.Debug) - rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()} - t.activities.PushFront(rootAct) - return t -} - -// callers must hold t.rwl -func (t *Task) cur() *taskActivity { - return t.activities.Front().Value.(*taskActivity) -} - -// buildActivityStack returns the stack of activity names -// t.rwl must be held, but the slice can be returned since strings are immutable -func (t *Task) buildActivityStack() []string { - comps := make([]string, 0, t.activities.Len()) - for e := t.activities.Back(); e != nil; e = e.Prev() { - act := e.Value.(*taskActivity) - comps = append(comps, act.name) - } - return comps -} - -// Start a sub-activity. -// Must always be matched with a call to t.Finish() -// --- consider using defer for this purpose. -func (t *Task) Enter(activity string) { - t.rwl.Lock() - defer t.rwl.Unlock() - - prev := t.cur() - if prev.idle { - // reset progress when leaving idle task - // we leave the old progress dangling to have the user not worry about - prev.progress = newTaskProgress() - - prom.taskLastActiveStart.WithLabelValues( - t.parent.JobName(), - t.parent.JobType().String(), - t.name). - Set(float64(prev.progress.creation.UnixNano()) / 1e9) - - } - act := &taskActivity{activity, false, nil, prev.progress} - t.activities.PushFront(act) - stack := t.buildActivityStack() - activityField := strings.Join(stack, ".") - act.logger = prev.logger.ReplaceField(logTaskField, activityField) - - t.activitiesLastUpdate = time.Now() -} - -func (t *Task) UpdateProgress(dtx, drx int64) { - t.rwl.RLock() - p := t.cur().progress // protected by own rwlock - t.rwl.RUnlock() - p.UpdateIO(dtx, drx) -} - -// Returns a wrapper io.Reader that updates this task's _current_ progress value. -// Progress updates after this task resets its progress value are discarded. -func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater { - t.rwl.RLock() - defer t.rwl.RUnlock() - return &IOProgressUpdater{r, t.cur().progress} -} - -func (t *Task) Status() *TaskStatus { - t.rwl.RLock() - defer t.rwl.RUnlock() - // NOTE - // do not return any state in TaskStatus that is protected by t.rwl - - cur := t.cur() - stack := t.buildActivityStack() - prog := cur.progress.Read() - - var maxLevel logger.Level - for _, entry := range prog.logEntries { - if maxLevel < entry.Level { - maxLevel = entry.Level - } - } - - lastUpdate := prog.lastUpdate - if lastUpdate.Before(t.activitiesLastUpdate) { - lastUpdate = t.activitiesLastUpdate - } - - s := &TaskStatus{ - Name: stack[0], - ActivityStack: stack, - Idle: cur.idle, - ProgressRx: prog.rx, - ProgressTx: prog.tx, - LogEntries: prog.logEntries, - MaxLogLevel: maxLevel, - LastUpdate: lastUpdate, - } - - return s -} - -// Finish a sub-activity. -// Corresponds to a preceding call to t.Enter() -func (t *Task) Finish() { - t.rwl.Lock() - defer t.rwl.Unlock() - top := t.activities.Front() - if top.Next() == nil { - return // cannot remove root activity - } - t.activities.Remove(top) - t.activitiesLastUpdate = time.Now() - - // prometheus - front := t.activities.Front() - if front != nil && front == t.activities.Back() { - idleAct := front.Value.(*taskActivity) - if !idleAct.idle { - panic("inconsistent implementation") - } - progress := idleAct.progress.Read() - non_idle_time := t.activitiesLastUpdate.Sub(progress.creation) // use same time - prom.taskLastActiveDuration.WithLabelValues( - t.parent.JobName(), - t.parent.JobType().String(), - t.name).Set(non_idle_time.Seconds()) - } - -} - -// Returns a logger derived from the logger passed to the constructor function. -// The logger's task field contains the current activity stack joined by '.'. -func (t *Task) Log() *logger.Logger { - t.rwl.RLock() - defer t.rwl.RUnlock() - // FIXME should influence TaskStatus's LastUpdate field - return t.cur().logger -} - -// implement logger.Outlet interface -func (t *Task) WriteEntry(entry logger.Entry) error { - t.rwl.RLock() - defer t.rwl.RUnlock() - t.cur().progress.UpdateLogEntry(entry) - - prom.taskLogEntries.WithLabelValues( - t.parent.JobName(), - t.parent.JobType().String(), - t.name, - entry.Level.String()). - Inc() - - return nil -} - -type IOProgressUpdater struct { - r io.Reader - p *taskProgress -} - -func (u *IOProgressUpdater) Read(p []byte) (n int, err error) { - n, err = u.r.Read(p) - u.p.UpdateIO(int64(n), 0) - return - -} diff --git a/cmd/handler.go b/cmd/handler.go deleted file mode 100644 index 5e9a6bb..0000000 --- a/cmd/handler.go +++ /dev/null @@ -1,181 +0,0 @@ -package cmd - -import ( - "fmt" - "io" - - "github.com/pkg/errors" - "github.com/zrepl/zrepl/rpc" - "github.com/zrepl/zrepl/zfs" -) - -type DatasetMapping interface { - Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) -} - -type FilesystemRequest struct { - Roots []string // may be nil, indicating interest in all filesystems -} - -type FilesystemVersionsRequest struct { - Filesystem *zfs.DatasetPath -} - -type InitialTransferRequest struct { - Filesystem *zfs.DatasetPath - FilesystemVersion zfs.FilesystemVersion -} - -type IncrementalTransferRequest struct { - Filesystem *zfs.DatasetPath - From zfs.FilesystemVersion - To zfs.FilesystemVersion -} - -type Handler struct { - logger Logger - dsf zfs.DatasetFilter - fsvf zfs.FilesystemVersionFilter -} - -func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) { - return Handler{logger, dsfilter, snapfilter} -} - -func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) { - err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest) - if err != nil { - panic(err) - } - err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest) - if err != nil { - panic(err) - } - err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest) - if err != nil { - panic(err) - } - err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest) - if err != nil { - panic(err) - } - return nil -} - -func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) { - - log := h.logger.WithField("endpoint", "FilesystemRequest") - - log.WithField("request", r).Debug("request") - log.WithField("dataset_filter", h.dsf).Debug("dsf") - - allowed, err := zfs.ZFSListMapping(h.dsf) - if err != nil { - log.WithError(err).Error("error listing filesystems") - return - } - - log.WithField("response", allowed).Debug("response") - *roots = allowed - return -} - -func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) { - - log := h.logger.WithField("endpoint", "FilesystemVersionsRequest") - - log.WithField("request", r).Debug("request") - - // allowed to request that? - if h.pullACLCheck(r.Filesystem, nil); err != nil { - log.WithError(err).Warn("pull ACL check failed") - return - } - - // find our versions - vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf) - if err != nil { - log.WithError(err).Error("cannot list filesystem versions") - return - } - - log.WithField("response", vs).Debug("response") - - *versions = vs - return - -} - -func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) { - - log := h.logger.WithField("endpoint", "InitialTransferRequest") - - log.WithField("request", r).Debug("request") - if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil { - log.WithError(err).Warn("pull ACL check failed") - return - } - - log.Debug("invoking zfs send") - - s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil) - if err != nil { - log.WithError(err).Error("cannot send filesystem") - } - *stream = s - - return - -} - -func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) { - - log := h.logger.WithField("endpoint", "IncrementalTransferRequest") - log.WithField("request", r).Debug("request") - if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil { - log.WithError(err).Warn("pull ACL check failed") - return - } - if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil { - log.WithError(err).Warn("pull ACL check failed") - return - } - - log.Debug("invoking zfs send") - - s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To) - if err != nil { - log.WithError(err).Error("cannot send filesystem") - } - - *stream = s - return - -} - -func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) { - var fsAllowed, vAllowed bool - fsAllowed, err = h.dsf.Filter(p) - if err != nil { - err = fmt.Errorf("error evaluating ACL: %s", err) - return - } - if !fsAllowed { - err = fmt.Errorf("ACL prohibits access to %s", p.ToString()) - return - } - if v == nil { - return - } - - vAllowed, err = h.fsvf.Filter(*v) - if err != nil { - err = errors.Wrap(err, "error evaluating version filter") - return - } - if !vAllowed { - err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p)) - return - } - return -} diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 4a5c626..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,47 +0,0 @@ -// zrepl replicates ZFS filesystems & volumes between pools -// -// Code Organization -// -// The cmd package uses github.com/spf13/cobra for its CLI. -// -// It combines the other packages in the zrepl project to implement zrepl functionality. -// -// Each subcommand's code is in the corresponding *.go file. -// All other *.go files contain code shared by the subcommands. -package cmd - -import ( - "github.com/spf13/cobra" - "github.com/zrepl/zrepl/logger" -) - -// -//type Logger interface { -// Printf(format string, v ...interface{}) -//} - -var ( - zreplVersion string // set by build infrastructure -) - -type Logger = *logger.Logger - -var RootCmd = &cobra.Command{ - Use: "zrepl", - Short: "ZFS dataset replication", - Long: `Replicate ZFS filesystems & volumes between pools: - - - push & pull mode - - automatic snapshot creation & pruning - - local / over the network - - ACLs instead of blank SSH access`, -} - -var rootArgs struct { - configFile string -} - -func init() { - //cobra.OnInitialize(initConfig) - RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path") -} diff --git a/cmd/prune.go b/cmd/prune.go deleted file mode 100644 index 73ea7e1..0000000 --- a/cmd/prune.go +++ /dev/null @@ -1,138 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "github.com/zrepl/zrepl/zfs" - "time" -) - -type Pruner struct { - task *Task - Now time.Time - DryRun bool - DatasetFilter zfs.DatasetFilter - SnapshotPrefix string - PrunePolicy PrunePolicy -} - -type PruneResult struct { - Filesystem *zfs.DatasetPath - All []zfs.FilesystemVersion - Keep []zfs.FilesystemVersion - Remove []zfs.FilesystemVersion -} - -func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) { - p.task.Enter("filter_fs") - defer p.task.Finish() - filesystems, err := zfs.ZFSListMapping(p.DatasetFilter) - if err != nil { - p.task.Log().WithError(err).Error("error applying filesystem filter") - return nil, true - } - if len(filesystems) <= 0 { - p.task.Log().Info("no filesystems matching filter") - return nil, true - } - return filesystems, false -} - -func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) { - p.task.Enter("filter_versions") - defer p.task.Finish() - log := p.task.Log().WithField(logFSField, fs.ToString()) - - filter := NewPrefixFilter(p.SnapshotPrefix) - fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter) - if err != nil { - log.WithError(err).Error("error listing filesytem versions") - return nil, true - } - if len(fsversions) == 0 { - log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix") - return nil, true - } - return fsversions, false -} - -func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool) { - p.task.Enter("prune_fs") - defer p.task.Finish() - log := p.task.Log().WithField(logFSField, fs.ToString()) - - fsversions, stop := p.filterVersions(fs) - if stop { - return - } - - p.task.Enter("prune_policy") - keep, remove, err := p.PrunePolicy.Prune(fs, fsversions) - p.task.Finish() - if err != nil { - log.WithError(err).Error("error evaluating prune policy") - return - } - - log.WithField("fsversions", fsversions). - WithField("keep", keep). - WithField("remove", remove). - Debug("prune policy debug dump") - - r = PruneResult{fs, fsversions, keep, remove} - - makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) { - fields = make(map[string]interface{}) - fields["version"] = v.ToAbsPath(fs) - timeSince := v.Creation.Sub(p.Now) - fields["age_ns"] = timeSince - const day time.Duration = 24 * time.Hour - days := timeSince / day - remainder := timeSince % day - fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder) - return - } - - for _, v := range remove { - fields := makeFields(v) - log.WithFields(fields).Info("destroying version") - // echo what we'll do and exec zfs destroy if not dry run - // TODO special handling for EBUSY (zfs hold) - // TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent) - if !p.DryRun { - p.task.Enter("destroy") - err := zfs.ZFSDestroyFilesystemVersion(fs, v) - p.task.Finish() - if err != nil { - log.WithFields(fields).WithError(err).Error("error destroying version") - } - } - } - return r, true -} - -func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) { - p.task.Enter("run") - defer p.task.Finish() - - if p.DryRun { - p.task.Log().Info("doing dry run") - } - - filesystems, stop := p.filterFilesystems() - if stop { - return - } - - r = make([]PruneResult, 0, len(filesystems)) - - for _, fs := range filesystems { - res, ok := p.pruneFilesystem(fs) - if ok { - r = append(r, res) - } - } - - return - -} diff --git a/cmd/replication.go b/cmd/replication.go deleted file mode 100644 index 2cb5626..0000000 --- a/cmd/replication.go +++ /dev/null @@ -1,323 +0,0 @@ -package cmd - -import ( - "fmt" - "io" - - "bytes" - "encoding/json" - "github.com/zrepl/zrepl/rpc" - "github.com/zrepl/zrepl/zfs" -) - -type localPullACL struct{} - -func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) { - return true, nil -} - -const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent - -type InitialReplPolicy string - -const ( - InitialReplPolicyMostRecent InitialReplPolicy = "most_recent" - InitialReplPolicyAll InitialReplPolicy = "all" -) - -type Puller struct { - task *Task - Remote rpc.RPCClient - Mapping DatasetMapping - InitialReplPolicy InitialReplPolicy -} - -type remoteLocalMapping struct { - Remote *zfs.DatasetPath - Local *zfs.DatasetPath -} - -func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) { - p.task.Enter("fetch_remote_fs_list") - defer p.task.Finish() - - fsr := FilesystemRequest{} - if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil { - p.task.Log().WithError(err).Error("cannot fetch remote filesystem list") - return nil, false - } - return rfs, true -} - -func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) { - p.task.Enter("build_repl_mapping") - defer p.task.Finish() - - replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems)) - for fs := range remoteFilesystems { - var err error - var localFs *zfs.DatasetPath - localFs, err = p.Mapping.Map(remoteFilesystems[fs]) - if err != nil { - err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err) - p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map") - return nil, false - } - if localFs == nil { - continue - } - p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()). - WithField(logMapToField, localFs.ToString()).Debug("mapping") - m := remoteLocalMapping{remoteFilesystems[fs], localFs} - replMapping[m.Local.ToString()] = m - } - return replMapping, true -} - -// returns true if the receiving filesystem (local side) exists and can have child filesystems -func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) { - - p.task.Enter("repl_fs") - defer p.task.Finish() - var err error - remote := p.Remote - - log := p.task.Log(). - WithField(logMapFromField, m.Remote.ToString()). - WithField(logMapToField, m.Local.ToString()) - - log.Debug("examining local filesystem state") - localState, localExists := localFilesystemState[m.Local.ToString()] - var versions []zfs.FilesystemVersion - switch { - case !localExists: - log.Info("local filesystem does not exist") - case localState.Placeholder: - log.Info("local filesystem is marked as placeholder") - default: - log.Debug("local filesystem exists") - log.Debug("requesting local filesystem versions") - if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil { - log.WithError(err).Error("cannot get local filesystem versions") - return false - } - } - - log.Info("requesting remote filesystem versions") - r := FilesystemVersionsRequest{ - Filesystem: m.Remote, - } - var theirVersions []zfs.FilesystemVersion - if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil { - log.WithError(err).Error("cannot get remote filesystem versions") - log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem") - return false - } - - log.Debug("computing diff between remote and local filesystem versions") - diff := zfs.MakeFilesystemDiff(versions, theirVersions) - log.WithField("diff", diff).Debug("diff between local and remote filesystem") - - if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight { - panic("internal inconsistency: local placeholder implies ConflictAllRight") - } - - switch diff.Conflict { - case zfs.ConflictAllRight: - - log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy") - - if p.InitialReplPolicy != InitialReplPolicyMostRecent { - panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy)) - } - - snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight)) - for s := range diff.MRCAPathRight { - if diff.MRCAPathRight[s].Type == zfs.Snapshot { - snapsOnly = append(snapsOnly, diff.MRCAPathRight[s]) - } - } - - if len(snapsOnly) < 1 { - log.Warn("cannot perform initial sync: no remote snapshots") - return false - } - - r := InitialTransferRequest{ - Filesystem: m.Remote, - FilesystemVersion: snapsOnly[len(snapsOnly)-1], - } - - log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream") - - var stream io.Reader - - if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil { - log.WithError(err).Error("cannot request initial transfer") - return false - } - log.Debug("received initial transfer request response") - - log.Debug("invoke zfs receive") - recvArgs := []string{"-u"} - if localState.Placeholder { - log.Info("receive with forced rollback to replace placeholder filesystem") - recvArgs = append(recvArgs, "-F") - } - progressStream := p.task.ProgressUpdater(stream) - if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil { - log.WithError(err).Error("cannot receive stream") - return false - } - log.Info("finished receiving stream") // TODO rx delta - - // TODO unify with recv path of ConflictIncremental - log.Debug("configuring properties of received filesystem") - props := zfs.NewZFSProperties() - props.Set("readonly", "on") - if err = zfs.ZFSSet(m.Local, props); err != nil { - log.WithError(err).Error("cannot set readonly property") - } - - log.Info("finished initial transfer") - return true - - case zfs.ConflictIncremental: - - if len(diff.IncrementalPath) < 2 { - log.Info("remote and local are in sync") - return true - } - - log.Info("following incremental path from diff") - for i := 0; i < len(diff.IncrementalPath)-1; i++ { - - from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1] - - log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0 - - log.Debug("requesting incremental snapshot stream") - r := IncrementalTransferRequest{ - Filesystem: m.Remote, - From: from, - To: to, - } - var stream io.Reader - if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil { - log.WithError(err).Error("cannot request incremental snapshot stream") - return false - } - - log.Debug("invoking zfs receive") - progressStream := p.task.ProgressUpdater(stream) - // TODO protect against malicious incremental stream - if err = zfs.ZFSRecv(m.Local, progressStream); err != nil { - log.WithError(err).Error("cannot receive stream") - return false - } - log.Info("finished incremental transfer") // TODO increment rx - - } - log.Info("finished following incremental path") // TODO path rx - return true - - case zfs.ConflictNoCommonAncestor: - fallthrough - case zfs.ConflictDiverged: - - var jsonDiff bytes.Buffer - if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil { - log.WithError(err).Error("cannot JSON-encode diff") - return false - } - - var problem, resolution string - - switch diff.Conflict { - case zfs.ConflictNoCommonAncestor: - problem = "remote and local filesystem have snapshots, but no common one" - resolution = "perform manual establish a common snapshot history" - case zfs.ConflictDiverged: - problem = "remote and local filesystem share a history but have diverged" - resolution = "perform manual replication or delete snapshots on the receiving" + - "side to establish an incremental replication parse" - } - - log.WithField("diff", jsonDiff.String()). - WithField("problem", problem). - WithField("resolution", resolution). - Error("manual conflict resolution required") - - return false - - } - - panic("should not be reached") -} - -func (p *Puller) Pull() { - p.task.Enter("run") - defer p.task.Finish() - - p.task.Log().Info("request remote filesystem list") - remoteFilesystems, ok := p.getRemoteFilesystems() - if !ok { - return - } - - p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync") - replMapping, ok := p.buildReplMapping(remoteFilesystems) - if !ok { - - } - - p.task.Log().Debug("build cache for already present local filesystem state") - p.task.Enter("cache_local_fs_state") - localFilesystemState, err := zfs.ZFSListFilesystemState() - p.task.Finish() - if err != nil { - p.task.Log().WithError(err).Error("cannot request local filesystem state") - return - } - - localTraversal := zfs.NewDatasetPathForest() - for _, m := range replMapping { - localTraversal.Add(m.Local) - } - - p.task.Log().Info("start per-filesystem sync") - localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool { - - p.task.Enter("tree_walk") - defer p.task.Finish() - - log := p.task.Log().WithField(logFSField, v.Path.ToString()) - - if v.FilledIn { - if _, exists := localFilesystemState[v.Path.ToString()]; exists { - // No need to verify if this is a placeholder or not. It is sufficient - // to know we can add child filesystems to it - return true - } - log.Debug("create placeholder filesystem") - p.task.Enter("create_placeholder") - err = zfs.ZFSCreatePlaceholderFilesystem(v.Path) - p.task.Finish() - if err != nil { - log.Error("cannot create placeholder filesystem") - return false - } - return true - } - - m, ok := replMapping[v.Path.ToString()] - if !ok { - panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()") - } - - return p.replFilesystem(m, localFilesystemState) - }) - - return - -} diff --git a/cmd/sampleconf/localbackup/host1.yml b/cmd/sampleconf/localbackup/host1.yml deleted file mode 100644 index e48cf02..0000000 --- a/cmd/sampleconf/localbackup/host1.yml +++ /dev/null @@ -1,29 +0,0 @@ -jobs: -- name: mirror_local - type: local - - # snapshot the filesystems matched by the left-hand-side of the mapping - # every 10m with zrepl_ as prefix - mapping: { - "zroot/var/db<": "storage/backups/local/zroot/var/db", - "zroot/usr/home<": "storage/backups/local/zroot/usr/home", - "zroot/usr/home/paranoid": "!", #don't backup paranoid user - "zroot/poudriere/ports<": "!", #don't backup the ports trees - } - snapshot_prefix: zrepl_ - interval: 10m - initial_repl_policy: most_recent - - # keep one hour of 10m interval snapshots of filesystems matched by - # the left-hand-side of the mapping - prune_lhs: - policy: grid - grid: 1x1h(keep=all) - keep_bookmarks: all - - # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping - prune_rhs: - policy: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d - - diff --git a/cmd/sampleconf/pullbackup/backuphost.yml b/cmd/sampleconf/pullbackup/backuphost.yml deleted file mode 100644 index f6871d0..0000000 --- a/cmd/sampleconf/pullbackup/backuphost.yml +++ /dev/null @@ -1,27 +0,0 @@ -jobs: -- name: fullbackup_prod1 - type: pull - # connect to remote using ssh / stdinserver command - connect: - type: ssh+stdinserver - host: prod1.example.com - user: root - port: 22 - identity_file: /root/.ssh/id_ed25519 - - # pull (=ask for new snapshots) every 10m, prune afterwards - # this will leave us at most 10m behind production - interval: 10m - - # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com - mapping: { - "<":"storage/backups/zrepl/pull/prod1.example.com" - } - initial_repl_policy: most_recent - - # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping - snapshot_prefix: zrepl_ - prune: - policy: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d - diff --git a/cmd/sampleconf/pullbackup/productionhost.yml b/cmd/sampleconf/pullbackup/productionhost.yml deleted file mode 100644 index d2914e5..0000000 --- a/cmd/sampleconf/pullbackup/productionhost.yml +++ /dev/null @@ -1,47 +0,0 @@ -global: - serve: - stdinserver: - # Directory where AF_UNIX sockets for stdinserver command are placed. - # - # `zrepl stdinserver CLIENT_IDENTITY` - # * connects to the socket in $sockdir/CLIENT_IDENTITY - # * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3)) - # * does nothing more - # - # This enables a setup where `zrepl daemon` is not directly exposed to the internet - # but instead all traffic is tunnelled through SSH. - # The server with the source job has an authorized_keys file entry for the public key - # used by the corresponding pull job - # - # command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver - # - # Below is the default value. - sockdir: /var/run/zrepl/stdinserver - -jobs: - -- name: fullbackup_prod1 - # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity - type: source - serve: - type: stdinserver # see global.serve.stdinserver for explanation - client_identity: fullbackup_prod1 - - # snapshot these filesystems every 10m with zrepl_ as prefix - filesystems: { - "zroot/var/db<": "ok", - "zroot/usr/home<": "ok", - "zroot/var/tmp": "!", #don't backup /tmp - } - snapshot_prefix: zrepl_ - interval: 10m - - - # keep 1 hour of snapshots (6 at 10m interval) - # and one day of bookmarks in case pull doesn't work (link down, etc) - # => keep_bookmarks = 24h / interval = 24h / 10m = 144 - prune: - policy: grid - grid: 1x1h(keep=all) - keep_bookmarks: 144 - diff --git a/cmd/sampleconf/pushbackup/backuphost.yml b/cmd/sampleconf/pushbackup/backuphost.yml deleted file mode 100644 index 25431a6..0000000 --- a/cmd/sampleconf/pushbackup/backuphost.yml +++ /dev/null @@ -1,20 +0,0 @@ -jobs: -- name: fullbackup_prod1 - - # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity - type: push-sink - serve: - type: stdinserver - client_identity: fullbackup_prod1 - - # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com - mapping: { - "<":"storage/backups/zrepl/sink/prod1.example.com" - } - - # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping - prune: - policy: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d - - diff --git a/cmd/sampleconf/pushbackup/productionhost.yml b/cmd/sampleconf/pushbackup/productionhost.yml deleted file mode 100644 index 5b81ecd..0000000 --- a/cmd/sampleconf/pushbackup/productionhost.yml +++ /dev/null @@ -1,26 +0,0 @@ -jobs: -- name: fullbackup_prod1 - - # connect to remote using ssh / stdinserver command - type: push - connect: - type: ssh+stdinserver - host: prod1.example.com - user: root - port: 22 - identity_file: /root/.ssh/id_ed25519 - - # snapshot these datsets every 10m with zrepl_ as prefix - filesystems: { - "zroot/var/db<": "ok", - "zroot/usr/home<": "!", - } - snapshot_prefix: zrepl_ - interval: 10m - - # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc) - # (we cannot keep more than one day because this host will run out of disk space) - prune: - policy: grid - grid: 1x1d(keep=all) - diff --git a/cmd/sampleconf/random/debugging.yml b/cmd/sampleconf/random/debugging.yml deleted file mode 100644 index 237f3fe..0000000 --- a/cmd/sampleconf/random/debugging.yml +++ /dev/null @@ -1,33 +0,0 @@ -global: - serve: - stdinserver: - sockdir: /var/run/zrepl/stdinserver - -jobs: - -- name: debian2_pull - # JOB DEBUGGING OPTIONS - # should be equal for all job types, but each job implements the debugging itself - # => consult job documentation for supported options - debug: - conn: # debug the io.ReadWriteCloser connection - read_dump: /tmp/connlog_read # dump results of Read() invocations to this file - write_dump: /tmp/connlog_write # dump results of Write() invocations to this file - rpc: # debug the RPC protocol implementation - log: true # log output from rpc layer to the job log - - # ... just to make the unit tests pass. - # check other examples, e.g. localbackup or pullbackup for what the sutff below means - type: source - serve: - type: stdinserver - client_identity: debian2 - filesystems: { - "pool1/db<": ok - } - snapshot_prefix: zrepl_ - interval: 1s - prune: - policy: grid - grid: 1x10s(keep=all) - keep_bookmarks: all diff --git a/cmd/sampleconf/random/logging/client.crt b/cmd/sampleconf/random/logging/client.crt deleted file mode 100644 index 58dbfa5..0000000 --- a/cmd/sampleconf/random/logging/client.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV -BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 -ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy -MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV -BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS -Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F -uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS -Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp -kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK -f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID -AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml -YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST -3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh -M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd -Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM -j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM ------END CERTIFICATE----- diff --git a/cmd/sampleconf/random/logging/client.csr b/cmd/sampleconf/random/logging/client.csr deleted file mode 100644 index 380bb7f..0000000 --- a/cmd/sampleconf/random/logging/client.csr +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx -ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp -ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD -+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj -ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P -0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ -uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj -muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv -ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL -HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ -mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr -PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj -Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV -yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U= ------END CERTIFICATE REQUEST----- diff --git a/cmd/sampleconf/random/logging/client.key b/cmd/sampleconf/random/logging/client.key deleted file mode 100644 index a7f1565..0000000 --- a/cmd/sampleconf/random/logging/client.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb -NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF -MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9 -30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62 -JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK -cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs -5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1 -Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd -iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU -/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX -smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2 -Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg -NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf -S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7 -eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D -RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl -CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi -twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc -OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw -DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi -Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW -eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/ -Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8 -yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG -dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G -ifnel8XWUG0ROybsr1tk4mzW ------END PRIVATE KEY----- diff --git a/cmd/sampleconf/random/logging/logserver.crt b/cmd/sampleconf/random/logging/logserver.crt deleted file mode 100644 index 87ec4bb..0000000 --- a/cmd/sampleconf/random/logging/logserver.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3 -MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l -LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV -BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3 -TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b -IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge -364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos -rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM -M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM -8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP -BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR -mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw -PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM -aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH -VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r -K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY= ------END CERTIFICATE----- diff --git a/cmd/sampleconf/random/logging/logserver.key b/cmd/sampleconf/random/logging/logserver.key deleted file mode 100644 index 8eb8faa..0000000 --- a/cmd/sampleconf/random/logging/logserver.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq -e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz -lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7 -txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52 -r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24 -dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5 -9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o -uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis -O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7 -FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na -Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc -1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C -nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi -atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx -u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ -FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw -wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0 -09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H -G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM -Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11 -SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD -qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/ -NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq -8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG -kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p -5CbIivB0QCjeQIA8dOQpE8E= ------END PRIVATE KEY----- diff --git a/cmd/sampleconf/random/logging_and_monitoring.yml b/cmd/sampleconf/random/logging_and_monitoring.yml deleted file mode 100644 index d739f99..0000000 --- a/cmd/sampleconf/random/logging_and_monitoring.yml +++ /dev/null @@ -1,28 +0,0 @@ -global: - logging: - - - outlet: stdout - level: warn - format: human - - - outlet: tcp - level: debug - format: json - net: tcp - address: 127.0.0.1:8080 - retry_interval: 1s - tls: # if not specified, use plain TCP - ca: sampleconf/random/logging/logserver.crt - cert: sampleconf/random/logging/client.crt - key: sampleconf/random/logging/client.key - - - outlet: syslog - level: debug - format: logfmt - - monitoring: - - - type: prometheus - listen: ':9090' - -jobs: [] diff --git a/cmd/test.go b/cmd/test.go deleted file mode 100644 index d6b1f80..0000000 --- a/cmd/test.go +++ /dev/null @@ -1,215 +0,0 @@ -package cmd - -import ( - "os" - - "bytes" - "context" - "fmt" - "sort" - "strings" - - "github.com/kr/pretty" - "github.com/spf13/cobra" - "github.com/zrepl/zrepl/logger" - "github.com/zrepl/zrepl/zfs" - "time" -) - -var testCmd = &cobra.Command{ - Use: "test", - Short: "test configuration", - PersistentPreRun: testCmdGlobalInit, -} - -var testCmdGlobal struct { - log Logger - conf *Config -} - -var testConfigSyntaxCmd = &cobra.Command{ - Use: "config", - Short: "parse config file and dump parsed datastructure", - Run: doTestConfig, -} - -var testDatasetMapFilter = &cobra.Command{ - Use: "pattern jobname test/zfs/dataset/path", - Short: "test dataset mapping / filter specified in config", - Example: ` zrepl test pattern my_pull_job tank/tmp`, - Run: doTestDatasetMapFilter, -} - -var testPrunePolicyArgs struct { - side PrunePolicySide - showKept bool - showRemoved bool -} - -var testPrunePolicyCmd = &cobra.Command{ - Use: "prune jobname", - Short: "do a dry-run of the pruning part of a job", - Run: doTestPrunePolicy, -} - -func init() { - RootCmd.AddCommand(testCmd) - testCmd.AddCommand(testConfigSyntaxCmd) - testCmd.AddCommand(testDatasetMapFilter) - - testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)") - testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots") - testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots") - testCmd.AddCommand(testPrunePolicyCmd) -} - -func testCmdGlobalInit(cmd *cobra.Command, args []string) { - - out := logger.NewOutlets() - out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info) - log := logger.NewLogger(out, 1*time.Second) - testCmdGlobal.log = log - - var err error - if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil { - testCmdGlobal.log.Printf("error parsing config file: %s", err) - os.Exit(1) - } - -} - -func doTestConfig(cmd *cobra.Command, args []string) { - - log, conf := testCmdGlobal.log, testCmdGlobal.conf - - log.Printf("config ok") - log.Printf("%# v", pretty.Formatter(conf)) - return -} - -func doTestDatasetMapFilter(cmd *cobra.Command, args []string) { - - log, conf := testCmdGlobal.log, testCmdGlobal.conf - - if len(args) != 2 { - log.Printf("specify job name as first postitional argument, test input as second") - log.Printf(cmd.UsageString()) - os.Exit(1) - } - n, i := args[0], args[1] - - jobi, err := conf.LookupJob(n) - if err != nil { - log.Printf("%s", err) - os.Exit(1) - } - - var mf *DatasetMapFilter - switch j := jobi.(type) { - case *PullJob: - mf = j.Mapping - case *SourceJob: - mf = j.Filesystems - case *LocalJob: - mf = j.Mapping - default: - panic("incomplete implementation") - } - - ip, err := zfs.NewDatasetPath(i) - if err != nil { - log.Printf("cannot parse test input as ZFS dataset path: %s", err) - os.Exit(1) - } - - if mf.filterMode { - pass, err := mf.Filter(ip) - if err != nil { - log.Printf("error evaluating filter: %s", err) - os.Exit(1) - } - log.Printf("filter result: %v", pass) - } else { - res, err := mf.Map(ip) - if err != nil { - log.Printf("error evaluating mapping: %s", err) - os.Exit(1) - } - toStr := "NO MAPPING" - if res != nil { - toStr = res.ToString() - } - log.Printf("%s => %s", ip.ToString(), toStr) - - } - -} - -func doTestPrunePolicy(cmd *cobra.Command, args []string) { - - log, conf := testCmdGlobal.log, testCmdGlobal.conf - - if cmd.Flags().NArg() != 1 { - log.Printf("specify job name as first positional argument") - log.Printf(cmd.UsageString()) - os.Exit(1) - } - - jobname := cmd.Flags().Arg(0) - jobi, err := conf.LookupJob(jobname) - if err != nil { - log.Printf("%s", err) - os.Exit(1) - } - - jobp, ok := jobi.(PruningJob) - if !ok { - log.Printf("job doesn't do any prunes") - os.Exit(0) - } - - log.Printf("job dump:\n%s", pretty.Sprint(jobp)) - - task := NewTask("", jobi, log) - pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true) - if err != nil { - log.Printf("cannot create test pruner: %s", err) - os.Exit(1) - } - - log.Printf("start pruning") - - ctx := context.WithValue(context.Background(), contextKeyLog, log) - result, err := pruner.Run(ctx) - if err != nil { - log.Printf("error running pruner: %s", err) - os.Exit(1) - } - - sort.Slice(result, func(i, j int) bool { - return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1 - }) - - var b bytes.Buffer - for _, r := range result { - fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString()) - - if testPrunePolicyArgs.showKept { - fmt.Fprintf(&b, "\tkept:\n") - for _, v := range r.Keep { - fmt.Fprintf(&b, "\t- %s\n", v.Name) - } - } - - if testPrunePolicyArgs.showRemoved { - fmt.Fprintf(&b, "\tremoved:\n") - for _, v := range r.Remove { - fmt.Fprintf(&b, "\t- %s\n", v.Name) - } - } - - } - - log.Printf("pruning result:\n%s", b.String()) - -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..1b14dba --- /dev/null +++ b/config/config.go @@ -0,0 +1,483 @@ +package config + +import ( + "fmt" + "github.com/pkg/errors" + "github.com/zrepl/yaml-config" + "io/ioutil" + "os" + "reflect" + "regexp" + "strconv" + "time" +) + +type Config struct { + Jobs []JobEnum `yaml:"jobs"` + Global *Global `yaml:"global,optional,fromdefaults"` +} + +func (c *Config) Job(name string) (*JobEnum, error) { + for _, j := range c.Jobs { + if j.Name() == name { + return &j, nil + } + } + return nil, fmt.Errorf("job %q not defined in config", name) +} + +type JobEnum struct { + Ret interface{} +} + +func (j JobEnum) Name() string { + var name string + switch v := j.Ret.(type) { + case *PushJob: name = v.Name + case *SinkJob: name = v.Name + case *PullJob: name = v.Name + case *SourceJob: name = v.Name + default: + panic(fmt.Sprintf("unknownn job type %T", v)) + } + return name +} + +type ActiveJob struct { + Type string `yaml:"type"` + Name string `yaml:"name"` + Connect ConnectEnum `yaml:"connect"` + Pruning PruningSenderReceiver `yaml:"pruning"` + Debug JobDebugSettings `yaml:"debug,optional"` +} + +type PassiveJob struct { + Type string `yaml:"type"` + Name string `yaml:"name"` + Serve ServeEnum `yaml:"serve"` + Debug JobDebugSettings `yaml:"debug,optional"` +} + +type PushJob struct { + ActiveJob `yaml:",inline"` + Snapshotting SnapshottingEnum `yaml:"snapshotting"` + Filesystems FilesystemsFilter `yaml:"filesystems"` +} + +type PullJob struct { + ActiveJob `yaml:",inline"` + RootFS string `yaml:"root_fs"` + Interval time.Duration `yaml:"interval,positive"` +} + +type SinkJob struct { + PassiveJob `yaml:",inline"` + RootFS string `yaml:"root_fs"` +} + +type SourceJob struct { + PassiveJob `yaml:",inline"` + Snapshotting SnapshottingEnum `yaml:"snapshotting"` + Filesystems FilesystemsFilter `yaml:"filesystems"` +} + +type FilesystemsFilter map[string]bool + +type SnapshottingEnum struct { + Ret interface{} +} + +type SnapshottingPeriodic struct { + Type string `yaml:"type"` + Prefix string `yaml:"prefix"` + Interval time.Duration `yaml:"interval,positive"` +} + +type SnapshottingManual struct { + Type string `yaml:"type"` +} + +type PruningSenderReceiver struct { + KeepSender []PruningEnum `yaml:"keep_sender"` + KeepReceiver []PruningEnum `yaml:"keep_receiver"` +} + +type PruningLocal struct { + Keep []PruningEnum `yaml:"keep"` +} + +type LoggingOutletEnumList []LoggingOutletEnum + +func (l *LoggingOutletEnumList) SetDefault() { + def := ` +type: "stdout" +time: true +level: "warn" +format: "human" +` + s := StdoutLoggingOutlet{} + err := yaml.UnmarshalStrict([]byte(def), &s) + if err != nil { + panic(err) + } + *l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}} +} + +var _ yaml.Defaulter = &LoggingOutletEnumList{} + +type Global struct { + Logging *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"` + Monitoring []MonitoringEnum `yaml:"monitoring,optional"` + Control *GlobalControl `yaml:"control,optional,fromdefaults"` + Serve *GlobalServe `yaml:"serve,optional,fromdefaults"` + RPC *RPCConfig `yaml:"rpc,optional,fromdefaults"` +} + +func Default(i interface{}) { + v := reflect.ValueOf(i) + if v.Kind() != reflect.Ptr { + panic(v) + } + y := `{}` + err := yaml.Unmarshal([]byte(y), v.Interface()) + if err != nil { + panic(err) + } +} + +type RPCConfig struct { + Timeout time.Duration `yaml:"timeout,optional,positive,default=10s"` + TxChunkSize uint32 `yaml:"tx_chunk_size,optional,default=32768"` + RxStructuredMaxLen uint32 `yaml:"rx_structured_max,optional,default=16777216"` + RxStreamChunkMaxLen uint32 `yaml:"rx_stream_chunk_max,optional,default=16777216"` + RxHeaderMaxLen uint32 `yaml:"rx_header_max,optional,default=40960"` + SendHeartbeatInterval time.Duration `yaml:"send_heartbeat_interval,optional,positive,default=5s"` + +} + +type ConnectEnum struct { + Ret interface{} +} + +type ConnectCommon struct { + Type string `yaml:"type"` + RPC *RPCConfig `yaml:"rpc,optional"` +} + +type TCPConnect struct { + ConnectCommon `yaml:",inline"` + Address string `yaml:"address"` + DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` +} + +type TLSConnect struct { + ConnectCommon `yaml:",inline"` + Address string `yaml:"address"` + Ca string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` + ServerCN string `yaml:"server_cn"` + DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` +} + +type SSHStdinserverConnect struct { + ConnectCommon `yaml:",inline"` + Host string `yaml:"host"` + User string `yaml:"user"` + Port uint16 `yaml:"port"` + IdentityFile string `yaml:"identity_file"` + TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused + SSHCommand string `yaml:"ssh_command,optional"` //TODO unused + Options []string `yaml:"options,optional"` + DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` +} + +type LocalConnect struct { + ConnectCommon `yaml:",inline"` + ListenerName string `yaml:"listener_name"` + ClientIdentity string `yaml:"client_identity"` +} + +type ServeEnum struct { + Ret interface{} +} + +type ServeCommon struct { + Type string `yaml:"type"` + RPC *RPCConfig `yaml:"rpc,optional"` +} + +type TCPServe struct { + ServeCommon `yaml:",inline"` + Listen string `yaml:"listen"` + Clients map[string]string `yaml:"clients"` +} + +type TLSServe struct { + ServeCommon `yaml:",inline"` + Listen string `yaml:"listen"` + Ca string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` + ClientCNs []string `yaml:"client_cns"` + HandshakeTimeout time.Duration `yaml:"handshake_timeout,positive,default=10s"` +} + +type StdinserverServer struct { + ServeCommon `yaml:",inline"` + ClientIdentities []string `yaml:"client_identities"` +} + +type LocalServe struct { + ServeCommon `yaml:",inline"` + ListenerName string `yaml:"listener_name"` +} + +type PruningEnum struct { + Ret interface{} +} + +type PruneKeepNotReplicated struct { + Type string `yaml:"type"` + KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"` +} + +type PruneKeepLastN struct { + Type string `yaml:"type"` + Count int `yaml:"count"` +} + +type PruneKeepRegex struct { // FIXME rename to KeepRegex + Type string `yaml:"type"` + Regex string `yaml:"regex"` +} + +type LoggingOutletEnum struct { + Ret interface{} +} + +type LoggingOutletCommon struct { + Type string `yaml:"type"` + Level string `yaml:"level"` + Format string `yaml:"format"` +} + +type StdoutLoggingOutlet struct { + LoggingOutletCommon `yaml:",inline"` + Time bool `yaml:"time,default=true"` + Color bool `yaml:"color,default=true"` +} + +type SyslogLoggingOutlet struct { + LoggingOutletCommon `yaml:",inline"` + RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` +} + +type TCPLoggingOutlet struct { + LoggingOutletCommon `yaml:",inline"` + Address string `yaml:"address"` + Net string `yaml:"net,default=tcp"` + RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` + TLS *TCPLoggingOutletTLS `yaml:"tls,optional"` +} + +type TCPLoggingOutletTLS struct { + CA string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` +} + +type MonitoringEnum struct { + Ret interface{} +} + +type PrometheusMonitoring struct { + Type string `yaml:"type"` + Listen string `yaml:"listen"` +} + +type GlobalControl struct { + SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"` +} + +type GlobalServe struct { + StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"` +} + +type GlobalStdinServer struct { + SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"` +} + +type JobDebugSettings struct { + Conn *struct { + ReadDump string `yaml:"read_dump"` + WriteDump string `yaml:"write_dump"` + } `yaml:"conn,optional"` + RPCLog bool `yaml:"rpc_log,optional,default=false"` +} + +func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) { + var in struct { + Type string + } + if err := u(&in, true); err != nil { + return nil, err + } + if in.Type == "" { + return nil, &yaml.TypeError{Errors: []string{"must specify type"}} + } + + v, ok := types[in.Type] + if !ok { + return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}} + } + if err := u(v, false); err != nil { + return nil, err + } + return v, nil +} + +func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "push": &PushJob{}, + "sink": &SinkJob{}, + "pull": &PullJob{}, + "source": &SourceJob{}, + }) + return +} + +func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "tcp": &TCPConnect{}, + "tls": &TLSConnect{}, + "ssh+stdinserver": &SSHStdinserverConnect{}, + "local": &LocalConnect{}, + }) + return +} + +func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "tcp": &TCPServe{}, + "tls": &TLSServe{}, + "stdinserver": &StdinserverServer{}, + "local" : &LocalServe{}, + }) + return +} + +func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "not_replicated": &PruneKeepNotReplicated{}, + "last_n": &PruneKeepLastN{}, + "grid": &PruneGrid{}, + "regex": &PruneKeepRegex{}, + }) + return +} + +func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "periodic": &SnapshottingPeriodic{}, + "manual": &SnapshottingManual{}, + }) + return +} + +func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "stdout": &StdoutLoggingOutlet{}, + "syslog": &SyslogLoggingOutlet{}, + "tcp": &TCPLoggingOutlet{}, + }) + return +} + +func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + t.Ret, err = enumUnmarshal(u, map[string]interface{}{ + "prometheus": &PrometheusMonitoring{}, + }) + return +} + +var ConfigFileDefaultLocations = []string{ + "/etc/zrepl/zrepl.yml", + "/usr/local/etc/zrepl/zrepl.yml", +} + +func ParseConfig(path string) (i *Config, err error) { + + if path == "" { + // Try default locations + for _, l := range ConfigFileDefaultLocations { + stat, statErr := os.Stat(l) + if statErr != nil { + continue + } + if !stat.Mode().IsRegular() { + err = errors.Errorf("file at default location is not a regular file: %s", l) + return + } + path = l + break + } + } + + var bytes []byte + + if bytes, err = ioutil.ReadFile(path); err != nil { + return + } + + return ParseConfigBytes(bytes) +} + +func ParseConfigBytes(bytes []byte) (*Config, error) { + var c *Config + if err := yaml.UnmarshalStrict(bytes, &c); err != nil { + return nil, err + } + if c == nil { + return nil, fmt.Errorf("config is empty or only consists of comments") + } + return c, nil +} + +var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`) + +func parsePostitiveDuration(e string) (d time.Duration, err error) { + comps := durationStringRegex.FindStringSubmatch(e) + if len(comps) != 3 { + err = fmt.Errorf("does not match regex: %s %#v", e, comps) + return + } + + durationFactor, err := strconv.ParseInt(comps[1], 10, 64) + if err != nil { + return 0, err + } + if durationFactor <= 0 { + return 0, errors.New("duration must be positive integer") + } + + var durationUnit time.Duration + switch comps[2] { + case "s": + durationUnit = time.Second + case "m": + durationUnit = time.Minute + case "h": + durationUnit = time.Hour + case "d": + durationUnit = 24 * time.Hour + case "w": + durationUnit = 24 * 7 * time.Hour + default: + err = fmt.Errorf("contains unknown time unit '%s'", comps[2]) + return + } + + d = time.Duration(durationFactor) * durationUnit + return +} diff --git a/config/config_global_test.go b/config/config_global_test.go new file mode 100644 index 0000000..f73219e --- /dev/null +++ b/config/config_global_test.go @@ -0,0 +1,82 @@ +package config + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zrepl/yaml-config" + "testing" +) + +func testValidGlobalSection(t *testing.T, s string) *Config { + jobdef := ` +jobs: +- name: dummyjob + type: sink + serve: + type: tcp + listen: ":2342" + clients: { + "10.0.0.1":"foo" + } + root_fs: zoot/foo +` + _, err := ParseConfigBytes([]byte(jobdef)) + require.NoError(t, err) + return testValidConfig(t, s + jobdef) +} + +func TestOutletTypes(t *testing.T) { + conf := testValidGlobalSection(t, ` +global: + logging: + - type: stdout + level: debug + format: human + - type: syslog + level: info + retry_interval: 20s + format: human + - type: tcp + level: debug + format: json + address: logserver.example.com:1234 + - type: tcp + level: debug + format: json + address: encryptedlogserver.example.com:1234 + retry_interval: 20s + tls: + ca: /etc/zrepl/log/ca.crt + cert: /etc/zrepl/log/key.pem + key: /etc/zrepl/log/cert.pem +`) + assert.Equal(t, 4, len(*conf.Global.Logging)) + assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS) +} + +func TestDefaultLoggingOutlet(t *testing.T) { + conf := testValidGlobalSection(t, "") + assert.Equal(t, 1, len(*conf.Global.Logging)) + o := (*conf.Global.Logging)[0].Ret.(StdoutLoggingOutlet) + assert.Equal(t, "warn", o.Level) + assert.Equal(t, "human", o.Format) +} + +func TestPrometheusMonitoring(t *testing.T) { + conf := testValidGlobalSection(t, ` +global: + monitoring: + - type: prometheus + listen: ':9091' +`) + assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen) +} + +func TestLoggingOutletEnumList_SetDefaults(t *testing.T) { + e := &LoggingOutletEnumList{} + var i yaml.Defaulter = e + require.NotPanics(t, func() { + i.SetDefault() + assert.Equal(t, "warn", (*e)[0].Ret.(StdoutLoggingOutlet).Level) + }) +} diff --git a/config/config_minimal_test.go b/config/config_minimal_test.go new file mode 100644 index 0000000..72f8752 --- /dev/null +++ b/config/config_minimal_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "testing" + "github.com/stretchr/testify/assert" +) + +func TestConfigEmptyFails(t *testing.T) { + conf, err := testConfig(t, "\n") + assert.Nil(t, conf) + assert.Error(t, err) +} + +func TestJobsOnlyWorks(t *testing.T) { + testValidConfig(t, ` +jobs: +- name: push + type: push + # snapshot the filesystems matched by the left-hand-side of the mapping + # every 10m with zrepl_ as prefix + connect: + type: tcp + address: localhost:2342 + filesystems: { + "pool1/var/db<": true, + "pool1/usr/home<": true, + "pool1/usr/home/paranoid": false, #don't backup paranoid user + "pool1/poudriere/ports<": false #don't backup the ports trees + } + snapshotting: + type: manual + pruning: + keep_sender: + - type: not_replicated + keep_receiver: + - type: last_n + count: 1 +`) +} \ No newline at end of file diff --git a/config/config_rpc_test.go b/config/config_rpc_test.go new file mode 100644 index 0000000..f02311e --- /dev/null +++ b/config/config_rpc_test.go @@ -0,0 +1,86 @@ +package config + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestRPC(t *testing.T) { + conf := testValidConfig(t, ` +jobs: +- name: pull_servers + type: pull + connect: + type: tcp + address: "server1.foo.bar:8888" + rpc: + timeout: 20s # different form default, should merge + root_fs: "pool2/backup_servers" + interval: 10m + pruning: + keep_sender: + - type: not_replicated + keep_receiver: + - type: last_n + count: 100 + +- name: pull_servers2 + type: pull + connect: + type: tcp + address: "server1.foo.bar:8888" + rpc: + tx_chunk_size: 0xabcd # different from default, should merge + root_fs: "pool2/backup_servers" + interval: 10m + pruning: + keep_sender: + - type: not_replicated + keep_receiver: + - type: last_n + count: 100 + +- type: sink + name: "laptop_sink" + root_fs: "pool2/backup_laptops" + serve: + type: tcp + listen: "192.168.122.189:8888" + clients: { + "10.23.42.23":"client1" + } + rpc: + rx_structured_max: 0x2342 + +- type: sink + name: "other_sink" + root_fs: "pool2/backup_laptops" + serve: + type: tcp + listen: "192.168.122.189:8888" + clients: { + "10.23.42.23":"client1" + } + rpc: + send_heartbeat_interval: 10s + +`) + + assert.Equal(t, 20*time.Second, conf.Jobs[0].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.Timeout) + assert.Equal(t, uint32(0xabcd), conf.Jobs[1].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.TxChunkSize) + assert.Equal(t, uint32(0x2342), conf.Jobs[2].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.RxStructuredMaxLen) + assert.Equal(t, 10*time.Second, conf.Jobs[3].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.SendHeartbeatInterval) + defConf := RPCConfig{} + Default(&defConf) + assert.Equal(t, defConf.Timeout, conf.Global.RPC.Timeout) +} + +func TestGlobal_DefaultRPCConfig(t *testing.T) { + assert.NotPanics(t, func() { + var c RPCConfig + Default(&c) + assert.NotNil(t, c) + assert.Equal(t, c.TxChunkSize, uint32(1)<<15) + }) +} diff --git a/config/config_snapshotting_test.go b/config/config_snapshotting_test.go new file mode 100644 index 0000000..e0f826b --- /dev/null +++ b/config/config_snapshotting_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestSnapshotting(t *testing.T) { + tmpl := ` +jobs: +- name: foo + type: push + connect: + type: local + listener_name: foo + client_identity: bar + filesystems: {"<": true} + %s + pruning: + keep_sender: + - type: last_n + count: 10 + keep_receiver: + - type: last_n + count: 10 +` + manual := ` + snapshotting: + type: manual +` + periodic := ` + snapshotting: + type: periodic + prefix: zrepl_ + interval: 10m +` + + fillSnapshotting := func(s string) string {return fmt.Sprintf(tmpl, s)} + var c *Config + + t.Run("manual", func(t *testing.T) { + c = testValidConfig(t, fillSnapshotting(manual)) + snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual) + assert.Equal(t, "manual", snm.Type) + }) + + t.Run("periodic", func(t *testing.T) { + c = testValidConfig(t, fillSnapshotting(periodic)) + snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic) + assert.Equal(t, "periodic", snp.Type) + assert.Equal(t, 10*time.Minute, snp.Interval) + assert.Equal(t, "zrepl_" , snp.Prefix) + }) + +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..d8e1e2c --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "github.com/kr/pretty" + "github.com/stretchr/testify/require" + "path" + "path/filepath" + "testing" +) + +func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { + paths, err := filepath.Glob("./samples/*") + if err != nil { + t.Errorf("glob failed: %+v", err) + } + + for _, p := range paths { + + if path.Ext(p) != ".yml" { + t.Logf("skipping file %s", p) + continue + } + + t.Run(p, func(t *testing.T) { + c, err := ParseConfig(p) + if err != nil { + t.Errorf("error parsing %s:\n%+v", p, err) + } + + t.Logf("file: %s", p) + t.Log(pretty.Sprint(c)) + }) + + } + +} + + +func testValidConfig(t *testing.T, input string) (*Config) { + t.Helper() + conf, err := testConfig(t, input) + require.NoError(t, err) + require.NotNil(t, conf) + return conf +} + +func testConfig(t *testing.T, input string) (*Config, error) { + t.Helper() + return ParseConfigBytes([]byte(input)) +} \ No newline at end of file diff --git a/config/retentiongrid.go b/config/retentiongrid.go new file mode 100644 index 0000000..58b2ff3 --- /dev/null +++ b/config/retentiongrid.go @@ -0,0 +1,123 @@ +package config + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +type RetentionIntervalList []RetentionInterval + +type PruneGrid struct { + Type string `yaml:"type"` + Grid RetentionIntervalList `yaml:"grid"` + Regex string `yaml:"regex"` +} + +type RetentionInterval struct { + length time.Duration + keepCount int +} + +func (i *RetentionInterval) Length() time.Duration { + return i.length +} + +func (i *RetentionInterval) KeepCount() int { + return i.keepCount +} + +const RetentionGridKeepCountAll int = -1 + +type RetentionGrid struct { + intervals []RetentionInterval +} + +func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) { + var in string + if err := u(&in, true); err != nil { + return err + } + + intervals, err := parseRetentionGridIntervalsString(in) + if err != nil { + return err + } + + *t = intervals + + return nil +} + +var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`) + +func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) { + + comps := retentionStringIntervalRegex.FindStringSubmatch(e) + if comps == nil { + err = fmt.Errorf("retention string does not match expected format") + return + } + + times, err := strconv.Atoi(comps[1]) + if err != nil { + return nil, err + } else if times <= 0 { + return nil, fmt.Errorf("contains factor <= 0") + } + + duration, err := parsePostitiveDuration(comps[2]) + if err != nil { + return nil, err + } + + keepCount := 1 + if comps[3] != "" { + // Decompose key=value, comma separated + // For now, only keep_count is supported + re := regexp.MustCompile(`^\s*keep=(.+)\s*$`) + res := re.FindStringSubmatch(comps[4]) + if res == nil || len(res) != 2 { + err = fmt.Errorf("interval parameter contains unknown parameters") + return + } + if res[1] == "all" { + keepCount = RetentionGridKeepCountAll + } else { + keepCount, err = strconv.Atoi(res[1]) + if err != nil { + err = fmt.Errorf("cannot parse keep_count value") + return + } + } + } + + intervals = make([]RetentionInterval, times) + for i := range intervals { + intervals[i] = RetentionInterval{ + length: duration, + keepCount: keepCount, + } + } + + return + +} + +func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) { + + ges := strings.Split(s, "|") + intervals = make([]RetentionInterval, 0, 7*len(ges)) + + for intervalIdx, e := range ges { + parsed, err := parseRetentionGridIntervalString(e) + if err != nil { + return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e)) + } + intervals = append(intervals, parsed...) + } + + return +} diff --git a/config/samples/local.yml b/config/samples/local.yml new file mode 100644 index 0000000..fbf7861 --- /dev/null +++ b/config/samples/local.yml @@ -0,0 +1,31 @@ + +jobs: + - type: sink + name: "local_sink" + root_fs: "storage/zrepl/sink" + serve: + type: local + listener_name: localsink + + - type: push + name: "backup_system" + connect: + type: local + listener_name: localsink + client_identity: local_backup + filesystems: { + "system<": true, + } + snapshotting: + type: periodic + interval: 10m + prefix: zrepl_ + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + regex: "zrepl_.*" \ No newline at end of file diff --git a/config/samples/pull.yml b/config/samples/pull.yml new file mode 100644 index 0000000..9a32a43 --- /dev/null +++ b/config/samples/pull.yml @@ -0,0 +1,24 @@ +jobs: +- name: pull_servers + type: pull + connect: + type: tls + address: "server1.foo.bar:8888" + ca: "/certs/ca.crt" + cert: "/certs/cert.crt" + key: "/certs/key.pem" + server_cn: "server1" + root_fs: "pool2/backup_servers" + interval: 10m + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + regex: "zrepl_.*" + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + regex: "zrepl_.*" \ No newline at end of file diff --git a/config/samples/pull_ssh.yml b/config/samples/pull_ssh.yml new file mode 100644 index 0000000..cbe8559 --- /dev/null +++ b/config/samples/pull_ssh.yml @@ -0,0 +1,28 @@ +jobs: + +- name: pull_servers + type: pull + connect: + type: ssh+stdinserver + host: app-srv.example.com + user: root + port: 22 + identity_file: /etc/zrepl/ssh/identity + options: # optional, default [], `-o` arguments passed to ssh + - "Compression=on" + root_fs: "pool2/backup_servers" + interval: 10m + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + regex: "^zrepl_.*" + keep_receiver: + - type: regex + regex: keep_ + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + regex: "^zrepl_.*" diff --git a/config/samples/push.yml b/config/samples/push.yml new file mode 100644 index 0000000..6045d67 --- /dev/null +++ b/config/samples/push.yml @@ -0,0 +1,24 @@ +jobs: + - type: push + name: "push" + filesystems: { + "<": true, + "tmp": false + } + connect: + type: tcp + address: "backup-server.foo.bar:8888" + snapshotting: + type: manual + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + regex: "^zrepl_.*" + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + regex: "^zrepl_.*" \ No newline at end of file diff --git a/config/samples/sink.yml b/config/samples/sink.yml new file mode 100644 index 0000000..917a3cf --- /dev/null +++ b/config/samples/sink.yml @@ -0,0 +1,13 @@ +jobs: + - type: sink + name: "laptop_sink" + root_fs: "pool2/backup_laptops" + serve: + type: tls + listen: "192.168.122.189:8888" + ca: "ca.pem" + cert: "cert.pem" + key: "key.pem" + client_cns: + - "laptop1" + - "homeserver" diff --git a/config/samples/source.yml b/config/samples/source.yml new file mode 100644 index 0000000..f10f14b --- /dev/null +++ b/config/samples/source.yml @@ -0,0 +1,17 @@ +jobs: +- name: pull_source + type: source + serve: + type: tcp + listen: "0.0.0.0:8888" + clients: { + "192.168.122.123" : "client1" + } + filesystems: { + "<": true, + "secret": false + } + snapshotting: + type: periodic + interval: 10m + prefix: zrepl_ diff --git a/config/samples/source_ssh.yml b/config/samples/source_ssh.yml new file mode 100644 index 0000000..f81f44d --- /dev/null +++ b/config/samples/source_ssh.yml @@ -0,0 +1,17 @@ +jobs: +- name: pull_source + type: source + serve: + type: stdinserver + client_identities: + - "client1" + - "client2" + filesystems: { + "<": true, + "secret": false + } + snapshotting: + type: periodic + interval: 10m + prefix: zrepl_ + diff --git a/daemon/control.go b/daemon/control.go new file mode 100644 index 0000000..9163dbc --- /dev/null +++ b/daemon/control.go @@ -0,0 +1,240 @@ +package daemon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/daemon/job" + "github.com/zrepl/zrepl/daemon/nethelpers" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/version" + "io" + "net" + "net/http" + "time" +) + +type controlJob struct { + sockaddr *net.UnixAddr + jobs *jobs +} + +func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) { + j = &controlJob{jobs: jobs} + + j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath) + if err != nil { + err = errors.Wrap(err, "cannot resolve unix address") + return + } + + return +} + +func (j *controlJob) Name() string { return jobNameControl } + +func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} } + +var promControl struct { + requestBegin *prometheus.CounterVec + requestFinished *prometheus.HistogramVec +} + +func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) { + promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "zrepl", + Subsystem: "control", + Name: "request_begin", + Help: "number of request we started to handle", + }, []string{"endpoint"}) + + promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "zrepl", + Subsystem: "control", + Name: "request_finished", + Help: "time it took a request to finih", + Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20}, + }, []string{"endpoint"}) + registerer.MustRegister(promControl.requestBegin) + registerer.MustRegister(promControl.requestFinished) +} + +const ( + ControlJobEndpointPProf string = "/debug/pprof" + ControlJobEndpointVersion string = "/version" + ControlJobEndpointStatus string = "/status" + ControlJobEndpointSignal string = "/signal" +) + +func (j *controlJob) Run(ctx context.Context) { + + log := job.GetLogger(ctx) + defer log.Info("control job finished") + + l, err := nethelpers.ListenUnixPrivate(j.sockaddr) + if err != nil { + log.WithError(err).Error("error listening") + return + } + + pprofServer := NewPProfServer(ctx) + + mux := http.NewServeMux() + mux.Handle(ControlJobEndpointPProf, + requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) { + var msg PprofServerControlMsg + err := decoder(&msg) + if err != nil { + return nil, errors.Errorf("decode failed") + } + pprofServer.Control(msg) + return struct{}{}, nil + }}}) + + mux.Handle(ControlJobEndpointVersion, + requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) { + return version.NewZreplVersionInformation(), nil + }}}) + + mux.Handle(ControlJobEndpointStatus, + requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) { + s := j.jobs.status() + return s, nil + }}}) + + mux.Handle(ControlJobEndpointSignal, + requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) { + type reqT struct { + Name string + Op string + } + var req reqT + if decoder(&req) != nil { + return nil, errors.Errorf("decode failed") + } + + var err error + switch req.Op { + case "wakeup": + err = j.jobs.wakeup(req.Name) + case "reset": + err = j.jobs.reset(req.Name) + default: + err = fmt.Errorf("operation %q is invalid", req.Op) + } + + return struct{}{}, err + }}}) + server := http.Server{ + Handler: mux, + // control socket is local, 1s timeout should be more than sufficient, even on a loaded system + WriteTimeout: 1*time.Second, + ReadTimeout: 1*time.Second, + } + +outer: + for { + + served := make(chan error) + go func() { + served <- server.Serve(l) + close(served) + }() + + select { + case <-ctx.Done(): + log.WithError(ctx.Err()).Info("context done") + server.Shutdown(context.Background()) + break outer + case err = <-served: + if err != nil { + log.WithError(err).Error("error serving") + break outer + } + } + + } + +} + +type jsonResponder struct { + producer func() (interface{}, error) +} + +func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + res, err := j.producer() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, err.Error()) + return + } + var buf bytes.Buffer + err = json.NewEncoder(&buf).Encode(res) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, err.Error()) + } else { + io.Copy(w, &buf) + } +} + +type jsonDecoder = func(interface{}) error + +type jsonRequestResponder struct { + producer func(decoder jsonDecoder) (interface{}, error) +} + +func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var decodeError error + decoder := func(i interface{}) error { + err := json.NewDecoder(r.Body).Decode(&i) + decodeError = err + return err + } + res, producerErr := j.producer(decoder) + + //If we had a decode error ignore output of producer and return error + if decodeError != nil { + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, decodeError.Error()) + return + } + if producerErr != nil { + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, producerErr.Error()) + return + } + + var buf bytes.Buffer + encodeErr := json.NewEncoder(&buf).Encode(res) + if encodeErr != nil { + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, encodeErr.Error()) + } else { + io.Copy(w, &buf) + } +} + +type requestLogger struct { + log logger.Logger + handler http.Handler + handlerFunc http.HandlerFunc +} + +func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log := l.log.WithField("method", r.Method).WithField("url", r.URL) + log.Debug("start") + promControl.requestBegin.WithLabelValues(r.URL.Path).Inc() + defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration() + if l.handlerFunc != nil { + l.handlerFunc(w, r) + } else if l.handler != nil { + l.handler.ServeHTTP(w, r) + } else { + log.Error("no handler or handlerFunc configured") + } + log.Debug("finish") +} diff --git a/daemon/daemon.go b/daemon/daemon.go new file mode 100644 index 0000000..9f0e185 --- /dev/null +++ b/daemon/daemon.go @@ -0,0 +1,223 @@ +package daemon + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/job" + "github.com/zrepl/zrepl/daemon/job/reset" + "github.com/zrepl/zrepl/daemon/job/wakeup" + "github.com/zrepl/zrepl/daemon/logging" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/version" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" +) + +func Run(conf *config.Config) error { + + ctx, cancel := context.WithCancel(context.Background()) + + defer cancel() + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigChan + cancel() + }() + + outlets, err := logging.OutletsFromConfig(*conf.Global.Logging) + if err != nil { + return errors.Wrap(err, "cannot build logging from config") + } + + confJobs, err := job.JobsFromConfig(conf) + if err != nil { + return errors.Wrap(err, "cannot build jobs from config") + } + + log := logger.NewLogger(outlets, 1*time.Second) + log.Info(version.NewZreplVersionInformation().String()) + + for _, job := range confJobs { + if IsInternalJobName(job.Name()) { + panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME + } + } + + ctx = job.WithLogger(ctx, log) + + jobs := newJobs() + + // start control socket + controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs) + if err != nil { + panic(err) // FIXME + } + jobs.start(ctx, controlJob, true) + + for i, jc := range conf.Global.Monitoring { + var ( + job job.Job + err error + ) + switch v := jc.Ret.(type) { + case *config.PrometheusMonitoring: + job, err = newPrometheusJobFromConfig(v) + default: + return errors.Errorf("unknown monitoring job #%d (type %T)", i, v) + } + if err != nil { + return errors.Wrapf(err,"cannot build monitorin gjob #%d", i) + } + jobs.start(ctx, job, true) + } + + + log.Info("starting daemon") + + // start regular jobs + for _, j := range confJobs { + jobs.start(ctx, j, false) + } + + select { + case <-jobs.wait(): + log.Info("all jobs finished") + case <-ctx.Done(): + log.WithError(ctx.Err()).Info("context finished") + } + log.Info("daemon exiting") + return nil +} + +type jobs struct { + wg sync.WaitGroup + + // m protects all fields below it + m sync.RWMutex + wakeups map[string]wakeup.Func // by Job.Name + resets map[string]reset.Func // by Job.Name + jobs map[string]job.Job +} + +func newJobs() *jobs { + return &jobs{ + wakeups: make(map[string]wakeup.Func), + resets: make(map[string]reset.Func), + jobs: make(map[string]job.Job), + } +} + +const ( + logJobField string = "job" + logTaskField string = "task" + logSubsysField string = "subsystem" +) + +func (s *jobs) wait() <-chan struct{} { + ch := make(chan struct{}) + go func() { + s.wg.Wait() + }() + return ch +} + +func (s *jobs) status() map[string]*job.Status { + s.m.RLock() + defer s.m.RUnlock() + + type res struct { + name string + status *job.Status + } + var wg sync.WaitGroup + c := make(chan res, len(s.jobs)) + for name, j := range s.jobs { + wg.Add(1) + go func(name string, j job.Job) { + defer wg.Done() + c <- res{name: name, status: j.Status()} + }(name, j) + } + wg.Wait() + close(c) + ret := make(map[string]*job.Status, len(s.jobs)) + for res := range c { + ret[res.name] = res.status + } + return ret +} + +func (s *jobs) wakeup(job string) error { + s.m.RLock() + defer s.m.RUnlock() + + wu, ok := s.wakeups[job] + if !ok { + return errors.Errorf("Job %s does not exist", job) + } + return wu() +} + +func (s *jobs) reset(job string) error { + s.m.RLock() + defer s.m.RUnlock() + + wu, ok := s.resets[job] + if !ok { + return errors.Errorf("Job %s does not exist", job) + } + return wu() +} + +const ( + jobNamePrometheus = "_prometheus" + jobNameControl = "_control" +) + +func IsInternalJobName(s string) bool { + return strings.HasPrefix(s, "_") +} + +func (s *jobs) start(ctx context.Context, j job.Job, internal bool) { + s.m.Lock() + defer s.m.Unlock() + + jobLog := job.GetLogger(ctx). + WithField(logJobField, j.Name()). + WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug) + jobName := j.Name() + if !internal && IsInternalJobName(jobName) { + panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName)) + } + if internal && !IsInternalJobName(jobName) { + panic(fmt.Sprintf("internal job does not use internal job name %s", jobName)) + } + if _, ok := s.jobs[jobName]; ok { + panic(fmt.Sprintf("duplicate job name %s", jobName)) + } + + j.RegisterMetrics(prometheus.DefaultRegisterer) + + s.jobs[jobName] = j + ctx = job.WithLogger(ctx, jobLog) + ctx, wakeup := wakeup.Context(ctx) + ctx, resetFunc := reset.Context(ctx) + s.wakeups[jobName] = wakeup + s.resets[jobName] = resetFunc + + s.wg.Add(1) + go func() { + defer s.wg.Done() + jobLog.Info("starting job") + defer jobLog.Info("job exited") + j.Run(ctx) + }() +} diff --git a/cmd/config_mapfilter.go b/daemon/filters/fsmapfilter.go similarity index 79% rename from cmd/config_mapfilter.go rename to daemon/filters/fsmapfilter.go index c625fc8..7fc6bbd 100644 --- a/cmd/config_mapfilter.go +++ b/daemon/filters/fsmapfilter.go @@ -1,12 +1,11 @@ -package cmd +package filters import ( "fmt" - "strings" - - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" + "github.com/zrepl/zrepl/endpoint" "github.com/zrepl/zrepl/zfs" + "strings" ) type DatasetMapFilter struct { @@ -101,6 +100,7 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx return } +// Returns target == nil if there is no mapping func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) { if m.filterMode { @@ -114,9 +114,17 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, } me := m.entries[mi] - if strings.HasPrefix("!", me.mapping) { - // reject mapping - return nil, nil + if me.mapping == "" { + // Special case treatment: 'foo/bar<' => '' + if !me.subtreeMatch { + return nil, fmt.Errorf("mapping to '' must be a subtree match") + } + // ok... + } else { + if strings.HasPrefix("!", me.mapping) { + // reject mapping + return nil, nil + } } target, err = zfs.NewDatasetPath(me.mapping) @@ -177,12 +185,43 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) { return inv, nil } +// FIXME investigate whether we can support more... +func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) { + + if m.filterMode { + return nil, errors.Errorf("can only invert mappings") + } + + if len(m.entries) != 1 { + return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME + } + + e := m.entries[0] + + inv := &DatasetMapFilter{ + make([]datasetMapFilterEntry, len(m.entries)), + false, + } + mp, err := zfs.NewDatasetPath(e.mapping) + if err != nil { + return nil, err + } + + inv.entries[0] = datasetMapFilterEntry{ + path: mp, + mapping: e.path.ToString(), + subtreeMatch: e.subtreeMatch, + } + + return inv, nil +} + // Creates a new DatasetMapFilter in filter mode from a mapping // All accepting mapping results are mapped to accepting filter results // All rejecting mapping results are mapped to rejecting filter results -func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) { +func (m DatasetMapFilter) AsFilter() endpoint.FSFilter { - f = &DatasetMapFilter{ + f := &DatasetMapFilter{ make([]datasetMapFilterEntry, len(m.entries)), true, } @@ -217,16 +256,14 @@ func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, er return false, fmt.Errorf("'%s' is not a valid filter result", result) } -func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) { +func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) { - var m map[string]string - if err = mapstructure.Decode(mi, &m); err != nil { - err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err) - return - } - - f = NewDatasetMapFilter(len(m), filterMode) - for pathPattern, mapping := range m { + f = NewDatasetMapFilter(len(in), true) + for pathPattern, accept := range in { + mapping := MapFilterResultOmit + if accept { + mapping = MapFilterResultOk + } if err = f.Add(pathPattern, mapping); err != nil { err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err) return diff --git a/daemon/filters/fsvfilter.go b/daemon/filters/fsvfilter.go new file mode 100644 index 0000000..3abcca8 --- /dev/null +++ b/daemon/filters/fsvfilter.go @@ -0,0 +1,41 @@ +package filters + +import ( + "github.com/zrepl/zrepl/zfs" + "strings" +) + +type AnyFSVFilter struct{} + +func NewAnyFSVFilter() AnyFSVFilter { + return AnyFSVFilter{} +} + +var _ zfs.FilesystemVersionFilter = AnyFSVFilter{} + +func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) { + return true, nil +} + + +type PrefixFilter struct { + prefix string + fstype zfs.VersionType + fstypeSet bool // optionals anyone? +} + +var _ zfs.FilesystemVersionFilter = &PrefixFilter{} + +func NewPrefixFilter(prefix string) *PrefixFilter { + return &PrefixFilter{prefix: prefix} +} + +func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter { + return &PrefixFilter{prefix, versionType, true} +} + +func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) { + fstypeMatches := (!f.fstypeSet || t == f.fstype) + prefixMatches := strings.HasPrefix(name, f.prefix) + return fstypeMatches && prefixMatches, nil +} diff --git a/daemon/job/active.go b/daemon/job/active.go new file mode 100644 index 0000000..2822eba --- /dev/null +++ b/daemon/job/active.go @@ -0,0 +1,405 @@ +package job + +import ( + "context" + "github.com/pkg/errors" + "github.com/problame/go-streamrpc" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/job/reset" + "github.com/zrepl/zrepl/daemon/job/wakeup" + "github.com/zrepl/zrepl/daemon/transport/connecter" + "github.com/zrepl/zrepl/daemon/filters" + "github.com/zrepl/zrepl/daemon/pruner" + "github.com/zrepl/zrepl/endpoint" + "github.com/zrepl/zrepl/replication" + "github.com/zrepl/zrepl/zfs" + "sync" + "github.com/zrepl/zrepl/daemon/logging" + "github.com/zrepl/zrepl/daemon/snapper" + "time" +) + +type ActiveSide struct { + mode activeMode + name string + clientFactory *connecter.ClientFactory + + prunerFactory *pruner.PrunerFactory + + + promRepStateSecs *prometheus.HistogramVec // labels: state + promPruneSecs *prometheus.HistogramVec // labels: prune_side + promBytesReplicated *prometheus.CounterVec // labels: filesystem + + tasksMtx sync.Mutex + tasks activeSideTasks +} + +type activeSideTasks struct { + replication *replication.Replication + prunerSender, prunerReceiver *pruner.Pruner +} + +func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks { + a.tasksMtx.Lock() + defer a.tasksMtx.Unlock() + var copy activeSideTasks + copy = a.tasks + if u == nil { + return copy + } + u(©) + a.tasks = copy + return copy +} + +type activeMode interface { + SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) + Type() Type + RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) +} + +type modePush struct { + fsfilter endpoint.FSFilter + snapper *snapper.PeriodicOrManual +} + +func (m *modePush) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) { + sender := endpoint.NewSender(m.fsfilter) + receiver := endpoint.NewRemote(client) + return sender, receiver, nil +} + +func (m *modePush) Type() Type { return TypePush } + +func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan <- struct{}) { + m.snapper.Run(ctx, wakeUpCommon) +} + + +func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) { + m := &modePush{} + fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems) + if err != nil { + return nil, errors.Wrap(err, "cannnot build filesystem filter") + } + m.fsfilter = fsf + + if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil { + return nil, errors.Wrap(err, "cannot build snapper") + } + + return m, nil +} + +type modePull struct { + rootFS *zfs.DatasetPath + interval time.Duration +} + +func (m *modePull) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) { + sender := endpoint.NewRemote(client) + receiver, err := endpoint.NewReceiver(m.rootFS) + return sender, receiver, err +} + +func (*modePull) Type() Type { return TypePull } + +func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) { + t := time.NewTicker(m.interval) + defer t.Stop() + for { + select { + case <-t.C: + select { + case wakeUpCommon <- struct{}{}: + default: + GetLogger(ctx). + WithField("pull_interval", m.interval). + Warn("pull job took longer than pull interval") + wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup + } + case <-ctx.Done(): + return + } + } +} + +func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) { + m = &modePull{} + if in.Interval <= 0 { + return nil, errors.New("interval must be positive") + } + m.interval = in.Interval + + m.rootFS, err = zfs.NewDatasetPath(in.RootFS) + if err != nil { + return nil, errors.New("RootFS is not a valid zfs filesystem path") + } + if m.rootFS.Length() <= 0 { + return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver + } + + return m, nil +} + +func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) { + + j = &ActiveSide{mode: mode} + j.name = in.Name + j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "zrepl", + Subsystem: "replication", + Name: "state_time", + Help: "seconds spent during replication", + ConstLabels: prometheus.Labels{"zrepl_job":j.name}, + }, []string{"state"}) + j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "zrepl", + Subsystem: "replication", + Name: "bytes_replicated", + Help: "number of bytes replicated from sender to receiver per filesystem", + ConstLabels: prometheus.Labels{"zrepl_job":j.name}, + }, []string{"filesystem"}) + + j.clientFactory, err = connecter.FromConfig(g, in.Connect) + if err != nil { + return nil, errors.Wrap(err, "cannot build client") + } + + j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "zrepl", + Subsystem: "pruning", + Name: "time", + Help: "seconds spent in pruner", + ConstLabels: prometheus.Labels{"zrepl_job":j.name}, + }, []string{"prune_side"}) + j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs) + if err != nil { + return nil, err + } + + return j, nil +} + +func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) { + registerer.MustRegister(j.promRepStateSecs) + registerer.MustRegister(j.promPruneSecs) + registerer.MustRegister(j.promBytesReplicated) +} + +func (j *ActiveSide) Name() string { return j.name } + +type ActiveSideStatus struct { + Replication *replication.Report + PruningSender, PruningReceiver *pruner.Report +} + +func (j *ActiveSide) Status() *Status { + tasks := j.updateTasks(nil) + + s := &ActiveSideStatus{} + t := j.mode.Type() + if tasks.replication != nil { + s.Replication = tasks.replication.Report() + } + if tasks.prunerSender != nil { + s.PruningSender = tasks.prunerSender.Report() + } + if tasks.prunerReceiver != nil { + s.PruningReceiver = tasks.prunerReceiver.Report() + } + return &Status{Type: t, JobSpecific: s} +} + +func (j *ActiveSide) Run(ctx context.Context) { + log := GetLogger(ctx) + ctx = logging.WithSubsystemLoggers(ctx, log) + + defer log.Info("job exiting") + + periodicDone := make(chan struct{}) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + go j.mode.RunPeriodic(ctx, periodicDone) + + invocationCount := 0 +outer: + for { + log.Info("wait for wakeups") + select { + case <-ctx.Done(): + log.WithError(ctx.Err()).Info("context") + break outer + + case <-wakeup.Wait(ctx): + case <-periodicDone: + } + invocationCount++ + invLog := log.WithField("invocation", invocationCount) + j.do(WithLogger(ctx, invLog), periodicDone) + } +} + +func (j *ActiveSide) do(ctx context.Context, periodicWakeup <-chan struct{}) { + + log := GetLogger(ctx) + ctx = logging.WithSubsystemLoggers(ctx, log) + + // allow cancellation of an invocation (this function) + ctx, cancelThisRun := context.WithCancel(ctx) + defer cancelThisRun() + runDone := make(chan struct{}) + defer close(runDone) + go func() { + select { + case <-runDone: + case <-reset.Wait(ctx): + log.Info("reset received, cancelling current invocation") + cancelThisRun() + case <-ctx.Done(): + } + }() + + client, err := j.clientFactory.NewClient() + if err != nil { + log.WithError(err).Error("factory cannot instantiate streamrpc client") + } + defer client.Close(ctx) + + sender, receiver, err := j.mode.SenderReceiver(client) + + tasks := j.updateTasks(func(tasks *activeSideTasks) { + // reset it + *tasks = activeSideTasks{} + tasks.replication = replication.NewReplication(j.promRepStateSecs, j.promBytesReplicated) + }) + + log.Info("start replication") + replicationDone := make(chan struct{}) + replicationCtx, replicationCancel := context.WithCancel(ctx) + defer replicationCancel() + go func() { + tasks.replication.Drive(replicationCtx, sender, receiver) + close(replicationDone) + }() + outer: + for { + select { + case <-replicationDone: + // fine! + break outer + case <-periodicWakeup: + // Replication took longer than the periodic interval. + // + // For pull jobs, this isn't so bad because nothing changes on the active side + // if replication doesn't go forward. + // + // For push jobs, this means snapshots were taken. + // We need to invoke the pruner now, because otherwise an infinitely stuck replication + // will cause this side to fill up with snapshots. + // + // However, there are cases where replication progresses and just takes longer, + // and we don't want these situations be interrupted by a prune, which will require + // re-planning and starting over (think of initial replication as an example). + // + // Therefore, we prohibit pruning of snapshots that are part of the current replication plan. + // If there is no such plan, we kill the replication. + + if j.mode.Type() == TypePush { + + rep := tasks.replication.Report() + state, err := replication.StateString(rep.Status) + if err != nil { + panic(err) + } + + switch state { + case replication.Planning: + fallthrough + case replication.PlanningError: + fallthrough + case replication.WorkingWait: + log.WithField("repl_state", state.String()). + Info("cancelling replication after new snapshots invalidated its current state") + replicationCancel() + log.Info("waiting for replication to stop") + <-replicationDone // no need to wait for ctx.Done, replication is already bound to global cancel + break outer + default: + log.WithField("repl_state", state.String()). + Warn("new snapshots while replication is running and making progress") + } + + } + + } + } + + + var pruningWg sync.WaitGroup + log.Info("start pruning sender") + pruningWg.Add(1) + go func() { + defer pruningWg.Done() + tasks := j.updateTasks(func(tasks *activeSideTasks) { + tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender) + }) + tasks.prunerSender.Prune() + // FIXME no need to do the cancellation dance with sender, we know it's local for push + // FIXME and we don't worry about pull ATM + }() + log.Info("start pruning receiver") + pruningWg.Add(1) + go func() { + defer pruningWg.Done() + + receiverPrunerCtx, receiverPrunerCancel := context.WithCancel(ctx) + defer receiverPrunerCancel() + tasks := j.updateTasks(func(tasks *activeSideTasks) { + tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(receiverPrunerCtx, receiver, sender) + }) + receiverPrunerDone := make(chan struct{}) + go func() { + defer close(receiverPrunerDone) + tasks.prunerReceiver.Prune() + }() + + outer: + for { + select { + case <-receiverPrunerDone: + // fine! + break outer + case <-periodicWakeup: + // see comments for similar apporach with replication above + if j.mode.Type() == TypePush { + rep := tasks.prunerReceiver.Report() + state, err := pruner.StateString(rep.State) + if err != nil { + panic(err) + } + switch state { + case pruner.PlanWait: + fallthrough + case pruner.ExecWait: + log.WithField("pruner_state", state.String()). + Info("cancelling failing prune on receiver because new snapshots were taken on sender") + receiverPrunerCancel() + log.Info("waiting for receiver pruner to stop") + <-receiverPrunerDone + break outer + default: + log.WithField("pruner_state", state.String()). + Warn("new snapshots while prune on receiver is still running") + } + } + } + } + + }() + + pruningWg.Wait() // if pruners handle ctx cancellation correctly, we don't need to wait for it here + +} diff --git a/daemon/job/build_jobs.go b/daemon/job/build_jobs.go new file mode 100644 index 0000000..372f51a --- /dev/null +++ b/daemon/job/build_jobs.go @@ -0,0 +1,68 @@ +package job + +import ( + "fmt" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" +) + +func JobsFromConfig(c *config.Config) ([]Job, error) { + js := make([]Job, len(c.Jobs)) + for i := range c.Jobs { + j, err := buildJob(c.Global, c.Jobs[i]) + if err != nil { + return nil, err + } + js[i] = j + } + return js, nil +} + +func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) { + cannotBuildJob := func(e error, name string) (Job, error) { + return nil, errors.Wrapf(err, "cannot build job %q", name) + } + // FIXME prettify this + switch v := in.Ret.(type) { + case *config.SinkJob: + m, err := modeSinkFromConfig(c, v) + if err != nil { + return cannotBuildJob(err, v.Name) + } + j, err = passiveSideFromConfig(c, &v.PassiveJob, m) + if err != nil { + return cannotBuildJob(err, v.Name) + } + case *config.SourceJob: + m, err := modeSourceFromConfig(c, v) + if err != nil { + return cannotBuildJob(err, v.Name) + } + j, err = passiveSideFromConfig(c, &v.PassiveJob, m) + if err != nil { + return cannotBuildJob(err, v.Name) + } + case *config.PushJob: + m, err := modePushFromConfig(c, v) + if err != nil { + return cannotBuildJob(err, v.Name) + } + j, err = activeSide(c, &v.ActiveJob, m) + if err != nil { + return cannotBuildJob(err, v.Name) + } + case *config.PullJob: + m, err := modePullFromConfig(c, v) + if err != nil { + return cannotBuildJob(err, v.Name) + } + j, err = activeSide(c, &v.ActiveJob, m) + if err != nil { + return cannotBuildJob(err, v.Name) + } + default: + panic(fmt.Sprintf("implementation error: unknown job type %T", v)) + } + return j, nil + +} diff --git a/daemon/job/job.go b/daemon/job/job.go new file mode 100644 index 0000000..5b3a684 --- /dev/null +++ b/daemon/job/job.go @@ -0,0 +1,103 @@ +package job + +import ( + "context" + "encoding/json" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/logger" +) + +type Logger = logger.Logger + +type contextKey int + +const ( + contextKeyLog contextKey = iota +) + +func GetLogger(ctx context.Context) Logger { + if l, ok := ctx.Value(contextKeyLog).(Logger); ok { + return l + } + return logger.NewNullLogger() +} + +func WithLogger(ctx context.Context, l Logger) context.Context { + return context.WithValue(ctx, contextKeyLog, l) +} + + +type Job interface { + Name() string + Run(ctx context.Context) + Status() *Status + RegisterMetrics(registerer prometheus.Registerer) +} + +type Type string + +const ( + TypeInternal Type = "internal" + TypePush Type = "push" + TypeSink Type = "sink" + TypePull Type = "pull" + TypeSource Type = "source" +) + +type Status struct { + Type Type + JobSpecific interface{} +} + +func (s *Status) MarshalJSON() ([]byte, error) { + typeJson, err := json.Marshal(s.Type) + if err != nil { + return nil, err + } + jobJSON, err := json.Marshal(s.JobSpecific) + if err != nil { + return nil, err + } + m := map[string]json.RawMessage { + "type": typeJson, + string(s.Type): jobJSON, + } + return json.Marshal(m) +} + +func (s *Status) UnmarshalJSON(in []byte) (err error) { + var m map[string]json.RawMessage + if err := json.Unmarshal(in, &m); err != nil { + return err + } + tJSON, ok := m["type"] + if !ok { + return fmt.Errorf("field 'type' not found") + } + if err := json.Unmarshal(tJSON, &s.Type); err != nil { + return err + } + key := string(s.Type) + jobJSON, ok := m[key] + if !ok { + return fmt.Errorf("field '%s', not found", key) + } + switch s.Type { + case TypePull: fallthrough + case TypePush: + var st ActiveSideStatus + err = json.Unmarshal(jobJSON, &st) + s.JobSpecific = &st + case TypeSource: fallthrough + case TypeSink: + var st PassiveStatus + err = json.Unmarshal(jobJSON, &st) + s.JobSpecific = &st + case TypeInternal: + // internal jobs do not report specifics + default: + err = fmt.Errorf("unknown job type '%s'", key) + } + return err +} diff --git a/daemon/job/passive.go b/daemon/job/passive.go new file mode 100644 index 0000000..99071a8 --- /dev/null +++ b/daemon/job/passive.go @@ -0,0 +1,196 @@ +package job + +import ( + "context" + "github.com/pkg/errors" + "github.com/problame/go-streamrpc" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/filters" + "github.com/zrepl/zrepl/daemon/logging" + "github.com/zrepl/zrepl/daemon/transport/serve" + "github.com/zrepl/zrepl/daemon/snapper" + "github.com/zrepl/zrepl/endpoint" + "github.com/zrepl/zrepl/zfs" + "path" +) + +type PassiveSide struct { + mode passiveMode + name string + l serve.ListenerFactory + rpcConf *streamrpc.ConnConfig +} + +type passiveMode interface { + ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc + RunPeriodic(ctx context.Context) + Type() Type +} + +type modeSink struct { + rootDataset *zfs.DatasetPath +} + +func (m *modeSink) Type() Type { return TypeSink } + +func (m *modeSink) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc { + log := GetLogger(ctx) + + clientRootStr := path.Join(m.rootDataset.ToString(), conn.ClientIdentity()) + clientRoot, err := zfs.NewDatasetPath(clientRootStr) + if err != nil { + log.WithError(err). + WithField("client_identity", conn.ClientIdentity()). + Error("cannot build client filesystem map (client identity must be a valid ZFS FS name") + } + log.WithField("client_root", clientRoot).Debug("client root") + + local, err := endpoint.NewReceiver(clientRoot) + if err != nil { + log.WithError(err).Error("unexpected error: cannot convert mapping to filter") + return nil + } + + h := endpoint.NewHandler(local) + return h.Handle +} + +func (m *modeSink) RunPeriodic(_ context.Context) {} + +func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) { + m = &modeSink{} + m.rootDataset, err = zfs.NewDatasetPath(in.RootFS) + if err != nil { + return nil, errors.New("root dataset is not a valid zfs filesystem path") + } + if m.rootDataset.Length() <= 0 { + return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver + } + return m, nil +} + +type modeSource struct { + fsfilter zfs.DatasetFilter + snapper *snapper.PeriodicOrManual +} + +func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) { + // FIXME exact dedup of modePush + m = &modeSource{} + fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems) + if err != nil { + return nil, errors.Wrap(err, "cannnot build filesystem filter") + } + m.fsfilter = fsf + + if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil { + return nil, errors.Wrap(err, "cannot build snapper") + } + + return m, nil +} + +func (m *modeSource) Type() Type { return TypeSource } + +func (m *modeSource) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc { + sender := endpoint.NewSender(m.fsfilter) + h := endpoint.NewHandler(sender) + return h.Handle +} + +func (m *modeSource) RunPeriodic(ctx context.Context) { + m.snapper.Run(ctx, nil) +} + +func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) { + + s = &PassiveSide{mode: mode, name: in.Name} + if s.l, s.rpcConf, err = serve.FromConfig(g, in.Serve); err != nil { + return nil, errors.Wrap(err, "cannot build server") + } + + return s, nil +} + +func (j *PassiveSide) Name() string { return j.name } + +type PassiveStatus struct {} + +func (s *PassiveSide) Status() *Status { + return &Status{Type: s.mode.Type()} // FIXME PassiveStatus +} + +func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {} + +func (j *PassiveSide) Run(ctx context.Context) { + + log := GetLogger(ctx) + defer log.Info("job exiting") + + l, err := j.l.Listen() + if err != nil { + log.WithError(err).Error("cannot listen") + return + } + defer l.Close() + + { + ctx, cancel := context.WithCancel(logging.WithSubsystemLoggers(ctx, log)) // shadowing + defer cancel() + go j.mode.RunPeriodic(ctx) + } + + log.WithField("addr", l.Addr()).Debug("accepting connections") + var connId int +outer: + for { + + select { + case res := <-accept(ctx, l): + if res.err != nil { + log.WithError(res.err).Info("accept error") + continue + } + conn := res.conn + connId++ + connLog := log. + WithField("connID", connId) + connLog. + WithField("addr", conn.RemoteAddr()). + WithField("client_identity", conn.ClientIdentity()). + Info("handling connection") + go func() { + defer connLog.Info("finished handling connection") + defer conn.Close() + ctx := logging.WithSubsystemLoggers(ctx, connLog) + handleFunc := j.mode.ConnHandleFunc(ctx, conn) + if handleFunc == nil { + return + } + if err := streamrpc.ServeConn(ctx, conn, j.rpcConf, handleFunc); err != nil { + log.WithError(err).Error("error serving client") + } + }() + + case <-ctx.Done(): + break outer + } + + } + +} + +type acceptResult struct { + conn serve.AuthenticatedConn + err error +} + +func accept(ctx context.Context, listener serve.AuthenticatedListener) <-chan acceptResult { + c := make(chan acceptResult, 1) + go func() { + conn, err := listener.Accept(ctx) + c <- acceptResult{conn, err} + }() + return c +} diff --git a/daemon/job/reset/reset.go b/daemon/job/reset/reset.go new file mode 100644 index 0000000..b7322ed --- /dev/null +++ b/daemon/job/reset/reset.go @@ -0,0 +1,35 @@ +package reset + +import ( + "context" + "errors" +) + +type contextKey int + +const contextKeyReset contextKey = iota + +func Wait(ctx context.Context) <-chan struct{} { + wc, ok := ctx.Value(contextKeyReset).(chan struct{}) + if !ok { + wc = make(chan struct{}) + } + return wc +} + +type Func func() error + +var AlreadyReset = errors.New("already reset") + +func Context(ctx context.Context) (context.Context, Func) { + wc := make(chan struct{}) + wuf := func() error { + select { + case wc <- struct{}{}: + return nil + default: + return AlreadyReset + } + } + return context.WithValue(ctx, contextKeyReset, wc), wuf +} diff --git a/daemon/job/wakeup/wakeup.go b/daemon/job/wakeup/wakeup.go new file mode 100644 index 0000000..a099b53 --- /dev/null +++ b/daemon/job/wakeup/wakeup.go @@ -0,0 +1,35 @@ +package wakeup + +import ( + "context" + "errors" +) + +type contextKey int + +const contextKeyWakeup contextKey = iota + +func Wait(ctx context.Context) <-chan struct{} { + wc, ok := ctx.Value(contextKeyWakeup).(chan struct{}) + if !ok { + wc = make(chan struct{}) + } + return wc +} + +type Func func() error + +var AlreadyWokenUp = errors.New("already woken up") + +func Context(ctx context.Context) (context.Context, Func) { + wc := make(chan struct{}) + wuf := func() error { + select { + case wc <- struct{}{}: + return nil + default: + return AlreadyWokenUp + } + } + return context.WithValue(ctx, contextKeyWakeup, wc), wuf +} diff --git a/daemon/logging/adaptors.go b/daemon/logging/adaptors.go new file mode 100644 index 0000000..c5a7196 --- /dev/null +++ b/daemon/logging/adaptors.go @@ -0,0 +1,32 @@ +package logging + +import ( + "fmt" + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/logger" + "strings" +) + +type streamrpcLogAdaptor = twoClassLogAdaptor + +type twoClassLogAdaptor struct { + logger.Logger +} + +var _ streamrpc.Logger = twoClassLogAdaptor{} + +func (a twoClassLogAdaptor) Errorf(fmtStr string, args ...interface{}) { + const errorSuffix = ": %s" + if len(args) == 1 { + if err, ok := args[0].(error); ok && strings.HasSuffix(fmtStr, errorSuffix) { + msg := strings.TrimSuffix(fmtStr, errorSuffix) + a.WithError(err).Error(msg) + return + } + } + a.Logger.Error(fmt.Sprintf(fmtStr, args...)) +} + +func (a twoClassLogAdaptor) Infof(fmtStr string, args ...interface{}) { + a.Logger.Debug(fmt.Sprintf(fmtStr, args...)) +} diff --git a/daemon/logging/build_logging.go b/daemon/logging/build_logging.go new file mode 100644 index 0000000..fcc1fa4 --- /dev/null +++ b/daemon/logging/build_logging.go @@ -0,0 +1,205 @@ +package logging + +import ( + "context" + "crypto/tls" + "crypto/x509" + "github.com/mattn/go-isatty" + "github.com/pkg/errors" + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/pruner" + "github.com/zrepl/zrepl/endpoint" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/replication" + "github.com/zrepl/zrepl/tlsconf" + "os" + "github.com/zrepl/zrepl/daemon/snapper" + "github.com/zrepl/zrepl/daemon/transport/serve" +) + +func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) { + + outlets := logger.NewOutlets() + + if len(in) == 0 { + // Default config + out := WriterOutlet{&HumanFormatter{}, os.Stdout} + outlets.Add(out, logger.Warn) + return outlets, nil + } + + var syslogOutlets, stdoutOutlets int + for lei, le := range in { + + outlet, minLevel, err := parseOutlet(le) + if err != nil { + return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei) + } + var _ logger.Outlet = WriterOutlet{} + var _ logger.Outlet = &SyslogOutlet{} + switch outlet.(type) { + case *SyslogOutlet: + syslogOutlets++ + case WriterOutlet: + stdoutOutlets++ + } + + outlets.Add(outlet, minLevel) + + } + + if syslogOutlets > 1 { + return nil, errors.Errorf("can only define one 'syslog' outlet") + } + if stdoutOutlets > 1 { + return nil, errors.Errorf("can only define one 'stdout' outlet") + } + + return outlets, nil + +} + +const ( + SubsysReplication = "repl" + SubsysStreamrpc = "rpc" + SubsyEndpoint = "endpoint" +) + +func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context { + ctx = replication.WithLogger(ctx, log.WithField(SubsysField, "repl")) + ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{log.WithField(SubsysField, "rpc")}) + ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, "endpoint")) + ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, "pruning")) + ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, "snapshot")) + ctx = serve.WithLogger(ctx, log.WithField(SubsysField, "serve")) + return ctx +} + +func parseLogFormat(i interface{}) (f EntryFormatter, err error) { + var is string + switch j := i.(type) { + case string: + is = j + default: + return nil, errors.Errorf("invalid log format: wrong type: %T", i) + } + + switch is { + case "human": + return &HumanFormatter{}, nil + case "logfmt": + return &LogfmtFormatter{}, nil + case "json": + return &JSONFormatter{}, nil + default: + return nil, errors.Errorf("invalid log format: '%s'", is) + } + +} + +func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) { + + parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) { + if common.Level == "" || common.Format == "" { + return 0, nil, errors.Errorf("must specify 'level' and 'format' field") + } + + minLevel, err := logger.ParseLevel(common.Level) + if err != nil { + return 0, nil, errors.Wrap(err, "cannot parse 'level' field") + } + formatter, err := parseLogFormat(common.Format) + if err != nil { + return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field") + } + return minLevel, formatter, nil + } + + var f EntryFormatter + + switch v := in.Ret.(type) { + case *config.StdoutLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseStdoutOutlet(v, f) + case *config.TCPLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseTCPOutlet(v, f) + case *config.SyslogLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseSyslogOutlet(v, f) + default: + panic(v) + } + return o, level, err +} + +func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) { + flags := MetadataAll + writer := os.Stdout + if !isatty.IsTerminal(writer.Fd()) && !in.Time { + flags &= ^MetadataTime + } + if isatty.IsTerminal(writer.Fd()) && !in.Color { + flags &= ^MetadataColor + } + + formatter.SetMetadataFlags(flags) + return WriterOutlet{ + formatter, + os.Stdout, + }, nil +} + +func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) { + var tlsConfig *tls.Config + if in.TLS != nil { + tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) { + clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key) + if err != nil { + return nil, errors.Wrap(err, "cannot load client cert") + } + + var rootCAs *x509.CertPool + if m.CA == "" { + if rootCAs, err = x509.SystemCertPool(); err != nil { + return nil, errors.Wrap(err, "cannot open system cert pool") + } + } else { + rootCAs, err = tlsconf.ParseCAFile(m.CA) + if err != nil { + return nil, errors.Wrap(err, "cannot parse CA cert") + } + } + if rootCAs == nil { + panic("invariant violated") + } + + return tlsconf.ClientAuthClient(host, rootCAs, clientCert) + }(in.TLS, in.Address) + if err != nil { + return nil, errors.New("cannot not parse TLS config in field 'tls'") + } + } + + formatter.SetMetadataFlags(MetadataAll) + return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil + +} + +func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) { + out = &SyslogOutlet{} + out.Formatter = formatter + out.Formatter.SetMetadataFlags(MetadataNone) + out.RetryInterval = in.RetryInterval + return out, nil +} diff --git a/cmd/logging_formatters.go b/daemon/logging/logging_formatters.go similarity index 71% rename from cmd/logging_formatters.go rename to daemon/logging/logging_formatters.go index cf0f985..3f3a7a8 100644 --- a/cmd/logging_formatters.go +++ b/daemon/logging/logging_formatters.go @@ -1,20 +1,16 @@ -package cmd +package logging import ( "bytes" "encoding/json" "fmt" + "github.com/fatih/color" "github.com/go-logfmt/logfmt" "github.com/pkg/errors" "github.com/zrepl/zrepl/logger" "time" ) -type EntryFormatter interface { - SetMetadataFlags(flags MetadataFlags) - Format(e *logger.Entry) ([]byte, error) -} - const ( FieldLevel = "level" FieldMessage = "msg" @@ -22,13 +18,19 @@ const ( ) const ( - logJobField string = "job" - logTaskField string = "task" - logFSField string = "filesystem" - logMapFromField string = "map_from" - logMapToField string = "map_to" - logIncFromField string = "inc_from" - logIncToField string = "inc_to" + JobField string = "job" + SubsysField string = "subsystem" +) + +type MetadataFlags int64 + +const ( + MetadataTime MetadataFlags = 1 << iota + MetadataLevel + MetadataColor + + MetadataNone MetadataFlags = 0 + MetadataAll MetadataFlags = ^0 ) type NoFormatter struct{} @@ -69,39 +71,29 @@ func (f *HumanFormatter) ignored(field string) bool { func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) { var line bytes.Buffer + col := color.New() + if f.metadataFlags&MetadataColor != 0 { + col = e.Color() + } if f.metadataFlags&MetadataTime != 0 { fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat)) } if f.metadataFlags&MetadataLevel != 0 { - fmt.Fprintf(&line, "[%s]", e.Level.Short()) + fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short())) } - prefixFields := []string{logJobField, logTaskField, logFSField} + prefixFields := []string{JobField, SubsysField} prefixed := make(map[string]bool, len(prefixFields)+2) for _, field := range prefixFields { val, ok := e.Fields[field].(string) - if ok { - if !f.ignored(field) { - fmt.Fprintf(&line, "[%s]", val) - prefixed[field] = true - } - } else { - break + if !ok { + continue + } + if !f.ignored(field) { + fmt.Fprintf(&line, "[%s]", col.Sprint(val)) + prefixed[field] = true } - } - // even more prefix fields - mapFrom, mapFromOk := e.Fields[logMapFromField].(string) - mapTo, mapToOk := e.Fields[logMapToField].(string) - if mapFromOk && mapToOk && !f.ignored(logMapFromField) && !f.ignored(logMapToField) { - fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo) - prefixed[logMapFromField], prefixed[logMapToField] = true, true - } - incFrom, incFromOk := e.Fields[logIncFromField].(string) - incTo, incToOk := e.Fields[logIncToField].(string) - if incFromOk && incToOk && !f.ignored(logIncFromField) && !f.ignored(logMapToField) { - fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo) - prefixed[logIncFromField], prefixed[logIncToField] = true, true } if line.Len() > 0 { @@ -110,15 +102,11 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) { fmt.Fprint(&line, e.Message) if len(e.Fields)-len(prefixed) > 0 { - fmt.Fprint(&line, " ") - enc := logfmt.NewEncoder(&line) for field, value := range e.Fields { if prefixed[field] || f.ignored(field) { continue } - if err := logfmtTryEncodeKeyval(enc, field, value); err != nil { - return nil, err - } + fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value)) } } @@ -179,7 +167,7 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) { // at least try and put job and task in front prefixed := make(map[string]bool, 2) - prefix := []string{logJobField, logTaskField} + prefix := []string{JobField, SubsysField} for _, pf := range prefix { v, ok := e.Fields[pf] if !ok { diff --git a/cmd/logging_outlets.go b/daemon/logging/logging_outlets.go similarity index 92% rename from cmd/logging_outlets.go rename to daemon/logging/logging_outlets.go index d201ddc..5a00d42 100644 --- a/cmd/logging_outlets.go +++ b/daemon/logging/logging_outlets.go @@ -1,4 +1,4 @@ -package cmd +package logging import ( "bytes" @@ -12,18 +12,23 @@ import ( "time" ) +type EntryFormatter interface { + SetMetadataFlags(flags MetadataFlags) + Format(e *logger.Entry) ([]byte, error) +} + type WriterOutlet struct { - Formatter EntryFormatter - Writer io.Writer + formatter EntryFormatter + writer io.Writer } func (h WriterOutlet) WriteEntry(entry logger.Entry) error { - bytes, err := h.Formatter.Format(&entry) + bytes, err := h.formatter.Format(&entry) if err != nil { return err } - _, err = h.Writer.Write(bytes) - h.Writer.Write([]byte("\n")) + _, err = h.writer.Write(bytes) + h.writer.Write([]byte("\n")) return err } diff --git a/daemon/main.go b/daemon/main.go new file mode 100644 index 0000000..fae8c12 --- /dev/null +++ b/daemon/main.go @@ -0,0 +1,16 @@ +package daemon + +import ( + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/logger" +) + +type Logger = logger.Logger + +var DaemonCmd = &cli.Subcommand { + Use: "daemon", + Short: "run the zrepl daemon", + Run: func(subcommand *cli.Subcommand, args []string) error { + return Run(subcommand.Config()) + }, +} diff --git a/cmd/helpers.go b/daemon/nethelpers/helpers.go similarity index 98% rename from cmd/helpers.go rename to daemon/nethelpers/helpers.go index be3864e..994b9d2 100644 --- a/cmd/helpers.go +++ b/daemon/nethelpers/helpers.go @@ -1,4 +1,4 @@ -package cmd +package nethelpers import ( "github.com/pkg/errors" diff --git a/cmd/control_pprof.go b/daemon/pprof.go similarity index 86% rename from cmd/control_pprof.go rename to daemon/pprof.go index a6b6939..6c96251 100644 --- a/cmd/control_pprof.go +++ b/daemon/pprof.go @@ -1,4 +1,4 @@ -package cmd +package daemon import ( "net/http" @@ -9,7 +9,7 @@ import ( "net/http/pprof" ) -type PProfServer struct { +type pprofServer struct { cc chan PprofServerControlMsg state PprofServerControlMsg listener net.Listener @@ -22,9 +22,9 @@ type PprofServerControlMsg struct { HttpListenAddress string } -func NewPProfServer(ctx context.Context) *PProfServer { +func NewPProfServer(ctx context.Context) *pprofServer { - s := &PProfServer{ + s := &pprofServer{ cc: make(chan PprofServerControlMsg), } @@ -32,7 +32,7 @@ func NewPProfServer(ctx context.Context) *PProfServer { return s } -func (s *PProfServer) controlLoop(ctx context.Context) { +func (s *pprofServer) controlLoop(ctx context.Context) { outer: for { @@ -75,6 +75,6 @@ outer: } } -func (s *PProfServer) Control(msg PprofServerControlMsg) { +func (s *pprofServer) Control(msg PprofServerControlMsg) { s.cc <- msg } diff --git a/daemon/prometheus.go b/daemon/prometheus.go new file mode 100644 index 0000000..7607b94 --- /dev/null +++ b/daemon/prometheus.go @@ -0,0 +1,89 @@ +package daemon + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/job" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/zfs" + "net" + "net/http" +) + +type prometheusJob struct { + listen string +} + +func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) { + if _, _, err := net.SplitHostPort(in.Listen); err != nil { + return nil, err + } + return &prometheusJob{in.Listen}, nil +} + +var prom struct { + taskLogEntries *prometheus.CounterVec +} + +func init() { + prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "zrepl", + Subsystem: "daemon", + Name: "log_entries", + Help: "number of log entries per job task and level", + }, []string{"zrepl_job", "level"}) + prometheus.MustRegister(prom.taskLogEntries) +} + +func (j *prometheusJob) Name() string { return jobNamePrometheus } + +func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} } + +func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {} + +func (j *prometheusJob) Run(ctx context.Context) { + + if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil { + panic(err) + } + + log := job.GetLogger(ctx) + + l, err := net.Listen("tcp", j.listen) + if err != nil { + log.WithError(err).Error("cannot listen") + } + go func() { + select { + case <-ctx.Done(): + l.Close() + } + }() + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + err = http.Serve(l, mux) + if err != nil { + log.WithError(err).Error("error while serving") + } + +} + +type prometheusJobOutlet struct { + jobName string +} + +var _ logger.Outlet = prometheusJobOutlet{} + +func newPrometheusLogOutlet(jobName string) prometheusJobOutlet { + return prometheusJobOutlet{jobName} +} + +func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error { + prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc() + return nil +} + diff --git a/daemon/pruner/pruner.go b/daemon/pruner/pruner.go new file mode 100644 index 0000000..cddd687 --- /dev/null +++ b/daemon/pruner/pruner.go @@ -0,0 +1,523 @@ +package pruner + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/pruning" + "github.com/zrepl/zrepl/replication/pdu" + "net" + "sort" + "sync" + "time" +) + +// Try to keep it compatible with gitub.com/zrepl/zrepl/replication.Endpoint +type History interface { + ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) +} + +type Target interface { + ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) + ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) // fix depS + DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) +} + +type Logger = logger.Logger + +type contextKey int + +const contextKeyLogger contextKey = 0 + +func WithLogger(ctx context.Context, log Logger) context.Context { + return context.WithValue(ctx, contextKeyLogger, log) +} + +func GetLogger(ctx context.Context) Logger { + if l, ok := ctx.Value(contextKeyLogger).(Logger); ok { + return l + } + return logger.NewNullLogger() +} + +type args struct { + ctx context.Context + target Target + receiver History + rules []pruning.KeepRule + retryWait time.Duration + considerSnapAtCursorReplicated bool + promPruneSecs prometheus.Observer +} + +type Pruner struct { + args args + + mtx sync.RWMutex + + state State + + // State ErrWait|ErrPerm + sleepUntil time.Time + err error + + // State Exec + prunePending []*fs + pruneCompleted []*fs +} + +type PrunerFactory struct { + senderRules []pruning.KeepRule + receiverRules []pruning.KeepRule + retryWait time.Duration + considerSnapAtCursorReplicated bool + promPruneSecs *prometheus.HistogramVec +} + +func checkContainsKeep1(rules []pruning.KeepRule) error { + if len(rules) == 0 { + return nil //No keep rules means keep all - ok + } + for _, e := range rules { + switch e.(type) { + case *pruning.KeepLastN: + return nil + } + } + return errors.New("sender keep rules must contain last_n or be empty so that the last snapshot is definitely kept") +} + +func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus.HistogramVec) (*PrunerFactory, error) { + keepRulesReceiver, err := pruning.RulesFromConfig(in.KeepReceiver) + if err != nil { + return nil, errors.Wrap(err, "cannot build receiver pruning rules") + } + + keepRulesSender, err := pruning.RulesFromConfig(in.KeepSender) + if err != nil { + return nil, errors.Wrap(err, "cannot build sender pruning rules") + } + + considerSnapAtCursorReplicated := false + for _, r := range in.KeepSender { + knr, ok := r.Ret.(*config.PruneKeepNotReplicated) + if !ok { + continue + } + considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor + } + f := &PrunerFactory{ + keepRulesSender, + keepRulesReceiver, + 10 * time.Second, //FIXME constant + considerSnapAtCursorReplicated, + promPruneSecs, + } + return f, nil +} + +func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner { + p := &Pruner{ + args: args{ + WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")), + target, + receiver, + f.senderRules, + f.retryWait, + f.considerSnapAtCursorReplicated, + f.promPruneSecs.WithLabelValues("sender"), + }, + state: Plan, + } + return p +} + +func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner { + p := &Pruner{ + args: args{ + WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")), + target, + receiver, + f.receiverRules, + f.retryWait, + false, // senseless here anyways + f.promPruneSecs.WithLabelValues("receiver"), + }, + state: Plan, + } + return p +} + +//go:generate enumer -type=State +type State int + +const ( + Plan State = 1 << iota + PlanWait + Exec + ExecWait + ErrPerm + Done +) + +func (s State) statefunc() state { + var statemap = map[State]state{ + Plan: statePlan, + PlanWait: statePlanWait, + Exec: stateExec, + ExecWait: stateExecWait, + ErrPerm: nil, + Done: nil, + } + return statemap[s] +} + +type updater func(func(*Pruner)) State +type state func(args *args, u updater) state + +func (p *Pruner) Prune() { + p.prune(p.args) +} + +func (p *Pruner) prune(args args) { + s := p.state.statefunc() + for s != nil { + pre := p.state + s = s(&args, func(f func(*Pruner)) State { + p.mtx.Lock() + defer p.mtx.Unlock() + f(p) + return p.state + }) + post := p.state + GetLogger(args.ctx). + WithField("transition", fmt.Sprintf("%s=>%s", pre, post)). + Debug("state transition") + } +} + +type Report struct { + State string + SleepUntil time.Time + Error string + Pending, Completed []FSReport +} + +type FSReport struct { + Filesystem string + SnapshotList, DestroyList []SnapshotReport + Error string +} + +type SnapshotReport struct { + Name string + Replicated bool + Date time.Time +} + +func (p *Pruner) Report() *Report { + p.mtx.Lock() + defer p.mtx.Unlock() + + r := Report{State: p.state.String()} + + if p.state & PlanWait|ExecWait != 0 { + r.SleepUntil = p.sleepUntil + } + if p.state & PlanWait|ExecWait|ErrPerm != 0 { + if p.err != nil { + r.Error = p.err.Error() + } + } + + if p.state & Plan|PlanWait == 0 { + return &r + } + + r.Pending = make([]FSReport, len(p.prunePending)) + for i, fs := range p.prunePending{ + r.Pending[i] = fs.Report() + } + r.Completed = make([]FSReport, len(p.pruneCompleted)) + for i, fs := range p.pruneCompleted{ + r.Completed[i] = fs.Report() + } + + return &r +} + +type fs struct { + path string + + // snapshots presented by target + // (type snapshot) + snaps []pruning.Snapshot + // destroy list returned by pruning.PruneSnapshots(snaps) + // (type snapshot) + destroyList []pruning.Snapshot + + mtx sync.RWMutex + // for Plan + err error +} + +func (f *fs) Update(err error) { + f.mtx.Lock() + defer f.mtx.Unlock() + f.err = err +} + +func (f *fs) Report() FSReport { + f.mtx.Lock() + defer f.mtx.Unlock() + + r := FSReport{} + r.Filesystem = f.path + if f.err != nil { + r.Error = f.err.Error() + } + + r.SnapshotList = make([]SnapshotReport, len(f.snaps)) + for i, snap := range f.snaps { + r.SnapshotList[i] = snap.(snapshot).Report() + } + + r.DestroyList = make([]SnapshotReport, len(f.destroyList)) + for i, snap := range f.destroyList{ + r.DestroyList[i] = snap.(snapshot).Report() + } + + return r +} + +type snapshot struct { + replicated bool + date time.Time + fsv *pdu.FilesystemVersion +} + +func (s snapshot) Report() SnapshotReport { + return SnapshotReport{ + Name: s.Name(), + Replicated: s.Replicated(), + Date: s.Date(), + } +} + +var _ pruning.Snapshot = snapshot{} + +func (s snapshot) Name() string { return s.fsv.Name } + +func (s snapshot) Replicated() bool { return s.replicated } + +func (s snapshot) Date() time.Time { return s.date } + +func shouldRetry(e error) bool { + switch e.(type) { + case nil: + return true + case net.Error: + return true + } + return false +} + +func onErr(u updater, e error) state { + return u(func(p *Pruner) { + p.err = e + if !shouldRetry(e) { + p.state = ErrPerm + return + } + switch p.state { + case Plan: + p.state = PlanWait + case Exec: + p.state = ExecWait + default: + panic(p.state) + } + }).statefunc() +} + +func statePlan(a *args, u updater) state { + + ctx, target, receiver := a.ctx, a.target, a.receiver + + tfss, err := target.ListFilesystems(ctx) + if err != nil { + return onErr(u, err) + } + + pfss := make([]*fs, len(tfss)) +fsloop: + for i, tfs := range tfss { + + l := GetLogger(ctx).WithField("fs", tfs.Path) + l.Debug("plan filesystem") + + + pfs := &fs{ + path: tfs.Path, + } + pfss[i] = pfs + + tfsvs, err := target.ListFilesystemVersions(ctx, tfs.Path) + if err != nil { + l.WithError(err).Error("cannot list filesystem versions") + if shouldRetry(err) { + return onErr(u, err) + } + pfs.err = err + continue fsloop + } + pfs.snaps = make([]pruning.Snapshot, 0, len(tfsvs)) + + rcReq := &pdu.ReplicationCursorReq{ + Filesystem: tfs.Path, + Op: &pdu.ReplicationCursorReq_Get{ + Get: &pdu.ReplicationCursorReq_GetOp{}, + }, + } + rc, err := receiver.ReplicationCursor(ctx, rcReq) + if err != nil { + l.WithError(err).Error("cannot get replication cursor") + if shouldRetry(err) { + return onErr(u, err) + } + pfs.err = err + continue fsloop + } + if rc.GetError() != "" { + l.WithField("reqErr", rc.GetError()).Error("cannot get replication cursor") + pfs.err = fmt.Errorf("%s", rc.GetError()) + continue fsloop + } + + + // scan from older to newer, all snapshots older than cursor are interpreted as replicated + sort.Slice(tfsvs, func(i, j int) bool { + return tfsvs[i].CreateTXG < tfsvs[j].CreateTXG + }) + + haveCursorSnapshot := false + for _, tfsv := range tfsvs { + if tfsv.Type != pdu.FilesystemVersion_Snapshot { + continue + } + if tfsv.Guid == rc.GetGuid() { + haveCursorSnapshot = true + } + } + preCursor := haveCursorSnapshot + for _, tfsv := range tfsvs { + if tfsv.Type != pdu.FilesystemVersion_Snapshot { + continue + } + creation, err := tfsv.CreationAsTime() + if err != nil { + pfs.err = fmt.Errorf("%s%s has invalid creation date: %s", tfs, tfsv.RelName(), err) + l.WithError(pfs.err).Error("") + continue fsloop + } + // note that we cannot use CreateTXG because target and receiver could be on different pools + atCursor := tfsv.Guid == rc.GetGuid() + preCursor = preCursor && !atCursor + pfs.snaps = append(pfs.snaps, snapshot{ + replicated: preCursor || (a.considerSnapAtCursorReplicated && atCursor), + date: creation, + fsv: tfsv, + }) + } + if preCursor { + pfs.err = fmt.Errorf("replication cursor not found in prune target filesystem versions") + l.WithError(pfs.err).Error("") + continue fsloop + } + + // Apply prune rules + pfs.destroyList = pruning.PruneSnapshots(pfs.snaps, a.rules) + + } + + return u(func(pruner *Pruner) { + for _, pfs := range pfss { + if pfs.err != nil { + pruner.pruneCompleted = append(pruner.pruneCompleted, pfs) + } else { + pruner.prunePending = append(pruner.prunePending, pfs) + } + } + pruner.state = Exec + }).statefunc() +} + +func stateExec(a *args, u updater) state { + + var pfs *fs + state := u(func(pruner *Pruner) { + if len(pruner.prunePending) == 0 { + nextState := Done + for _, pfs := range pruner.pruneCompleted { + if pfs.err != nil { + nextState = ErrPerm + } + } + pruner.state = nextState + return + } + pfs = pruner.prunePending[0] + }) + if state != Exec { + return state.statefunc() + } + + destroyList := make([]*pdu.FilesystemVersion, len(pfs.destroyList)) + for i := range destroyList { + destroyList[i] = pfs.destroyList[i].(snapshot).fsv + GetLogger(a.ctx). + WithField("fs", pfs.path). + WithField("destroy_snap", destroyList[i].Name). + Debug("policy destroys snapshot") + } + pfs.Update(nil) + req := pdu.DestroySnapshotsReq{ + Filesystem: pfs.path, + Snapshots: destroyList, + } + _, err := a.target.DestroySnapshots(a.ctx, &req) + pfs.Update(err) + if err != nil && shouldRetry(err) { + return onErr(u, err) + } + // if it's not retryable, treat is like as being done + + return u(func(pruner *Pruner) { + pruner.pruneCompleted = append(pruner.pruneCompleted, pfs) + pruner.prunePending = pruner.prunePending[1:] + }).statefunc() +} + +func stateExecWait(a *args, u updater) state { + return doWait(Exec, a, u) +} + +func statePlanWait(a *args, u updater) state { + return doWait(Plan, a, u) +} + +func doWait(goback State, a *args, u updater) state { + timer := time.NewTimer(a.retryWait) + defer timer.Stop() + select { + case <-timer.C: + return u(func(pruner *Pruner) { + pruner.state = goback + }).statefunc() + case <-a.ctx.Done(): + return onErr(u, a.ctx.Err()) + } +} diff --git a/daemon/pruner/pruner_test.go b/daemon/pruner/pruner_test.go new file mode 100644 index 0000000..a3e1656 --- /dev/null +++ b/daemon/pruner/pruner_test.go @@ -0,0 +1,211 @@ +package pruner + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/pruning" + "github.com/zrepl/zrepl/replication/pdu" + "net" + "testing" + "time" +) + +type mockFS struct { + path string + snaps []string +} + +func (m *mockFS) Filesystem() *pdu.Filesystem { + return &pdu.Filesystem{ + Path: m.path, + } +} + +func (m *mockFS) FilesystemVersions() []*pdu.FilesystemVersion { + versions := make([]*pdu.FilesystemVersion, len(m.snaps)) + for i, v := range m.snaps { + versions[i] = &pdu.FilesystemVersion{ + Type: pdu.FilesystemVersion_Snapshot, + Name: v, + Creation: pdu.FilesystemVersionCreation(time.Unix(0, 0)), + Guid: uint64(i), + } + } + return versions +} + +type mockTarget struct { + fss []mockFS + destroyed map[string][]string + listVersionsErrs map[string][]error + listFilesystemsErr []error + destroyErrs map[string][]error +} + +func (t *mockTarget) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) { + if len(t.listFilesystemsErr) > 0 { + e := t.listFilesystemsErr[0] + t.listFilesystemsErr = t.listFilesystemsErr[1:] + return nil, e + } + fss := make([]*pdu.Filesystem, len(t.fss)) + for i := range fss { + fss[i] = t.fss[i].Filesystem() + } + return fss, nil +} + +func (t *mockTarget) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) { + if len(t.listVersionsErrs[fs]) != 0 { + e := t.listVersionsErrs[fs][0] + t.listVersionsErrs[fs] = t.listVersionsErrs[fs][1:] + return nil, e + } + + for _, mfs := range t.fss { + if mfs.path != fs { + continue + } + return mfs.FilesystemVersions(), nil + } + return nil, fmt.Errorf("filesystem %s does not exist", fs) +} + +func (t *mockTarget) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) { + fs, snaps := req.Filesystem, req.Snapshots + if len(t.destroyErrs[fs]) != 0 { + e := t.destroyErrs[fs][0] + t.destroyErrs[fs] = t.destroyErrs[fs][1:] + return nil, e + } + destroyed := t.destroyed[fs] + res := make([]*pdu.DestroySnapshotRes, len(snaps)) + for i, s := range snaps { + destroyed = append(destroyed, s.Name) + res[i] = &pdu.DestroySnapshotRes{Error: "", Snapshot: s} + } + t.destroyed[fs] = destroyed + return &pdu.DestroySnapshotsRes{Results: res}, nil +} + +type mockCursor struct { + snapname string + guid uint64 +} +type mockHistory struct { + errs map[string][]error + cursors map[string]*mockCursor +} + +func (r *mockHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) { + fs := req.Filesystem + if len(r.errs[fs]) > 0 { + e := r.errs[fs][0] + r.errs[fs] = r.errs[fs][1:] + return nil, e + } + return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: 0}}, nil +} + +type stubNetErr struct { + msg string + temporary, timeout bool +} + +var _ net.Error = stubNetErr{} + +func (e stubNetErr) Error() string { + return e.msg +} + +func (e stubNetErr) Temporary() bool { return e.temporary } + +func (e stubNetErr) Timeout() bool { return e.timeout } + +func TestPruner_Prune(t *testing.T) { + + var _ net.Error = &net.OpError{} // we use it below + target := &mockTarget{ + listFilesystemsErr: []error{ + stubNetErr{msg: "fakerror0"}, + }, + listVersionsErrs: map[string][]error{ + "zroot/foo": { + stubNetErr{msg: "fakeerror1"}, // should be classified as temporaty + stubNetErr{msg: "fakeerror2"}, + }, + }, + destroyErrs: map[string][]error{ + "zroot/foo": { + fmt.Errorf("permanent error"), + }, + "zroot/bar": { + stubNetErr{msg: "fakeerror3"}, + }, + }, + destroyed: make(map[string][]string), + fss: []mockFS{ + { + path: "zroot/foo", + snaps: []string{ + "keep_a", + "keep_b", + "drop_c", + "keep_d", + }, + }, + { + path: "zroot/bar", + snaps: []string{ + "keep_e", + "keep_f", + "drop_g", + }, + }, + { + path: "zroot/baz", + snaps: []string{ + "keep_h", + "drop_i", + }, + }, + }, + } + history := &mockHistory{ + errs: map[string][]error{ + "zroot/foo": { + stubNetErr{msg: "fakeerror4"}, + }, + "zroot/baz": { + fmt.Errorf("permanent error2"), + }, + }, + } + + keepRules := []pruning.KeepRule{pruning.MustKeepRegex("^keep")} + + p := Pruner{ + args: args{ + ctx: WithLogger(context.Background(), logger.NewTestLogger(t)), + target: target, + receiver: history, + rules: keepRules, + retryWait: 10*time.Millisecond, + }, + state: Plan, + } + p.Prune() + + exp := map[string][]string{ + "zroot/bar": {"drop_g"}, + // drop_c is prohibited by failing destroy + // drop_i is prohibiteed by failing ReplicationCursor call + } + + assert.Equal(t, exp, target.destroyed) + + //assert.Equal(t, map[string][]error{}, target.listVersionsErrs, "retried") + +} diff --git a/daemon/pruner/state_enumer.go b/daemon/pruner/state_enumer.go new file mode 100644 index 0000000..8c396ab --- /dev/null +++ b/daemon/pruner/state_enumer.go @@ -0,0 +1,76 @@ +// Code generated by "enumer -type=State"; DO NOT EDIT. + +package pruner + +import ( + "fmt" +) + +const ( + _StateName_0 = "PlanPlanWait" + _StateName_1 = "Exec" + _StateName_2 = "ExecWait" + _StateName_3 = "ErrPerm" + _StateName_4 = "Done" +) + +var ( + _StateIndex_0 = [...]uint8{0, 4, 12} + _StateIndex_1 = [...]uint8{0, 4} + _StateIndex_2 = [...]uint8{0, 8} + _StateIndex_3 = [...]uint8{0, 7} + _StateIndex_4 = [...]uint8{0, 4} +) + +func (i State) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _StateName_0[_StateIndex_0[i]:_StateIndex_0[i+1]] + case i == 4: + return _StateName_1 + case i == 8: + return _StateName_2 + case i == 16: + return _StateName_3 + case i == 32: + return _StateName_4 + default: + return fmt.Sprintf("State(%d)", i) + } +} + +var _StateValues = []State{1, 2, 4, 8, 16, 32} + +var _StateNameToValueMap = map[string]State{ + _StateName_0[0:4]: 1, + _StateName_0[4:12]: 2, + _StateName_1[0:4]: 4, + _StateName_2[0:8]: 8, + _StateName_3[0:7]: 16, + _StateName_4[0:4]: 32, +} + +// StateString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func StateString(s string) (State, error) { + if val, ok := _StateNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to State values", s) +} + +// StateValues returns all values of the enum +func StateValues() []State { + return _StateValues +} + +// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise +func (i State) IsAState() bool { + for _, v := range _StateValues { + if i == v { + return true + } + } + return false +} diff --git a/daemon/snapper/snapper.go b/daemon/snapper/snapper.go new file mode 100644 index 0000000..6cd5b98 --- /dev/null +++ b/daemon/snapper/snapper.go @@ -0,0 +1,369 @@ +package snapper + +import ( + "github.com/zrepl/zrepl/config" + "github.com/pkg/errors" + "time" + "context" + "github.com/zrepl/zrepl/daemon/filters" + "fmt" + "github.com/zrepl/zrepl/zfs" + "sort" + "github.com/zrepl/zrepl/logger" + "sync" +) + + +//go:generate stringer -type=SnapState +type SnapState uint + +const ( + SnapPending SnapState = 1 << iota + SnapStarted + SnapDone + SnapError +) + +type snapProgress struct { + state SnapState + + // SnapStarted, SnapDone, SnapError + name string + startAt time.Time + + // SnapDone + doneAt time.Time + + // SnapErr + err error +} + +type args struct { + ctx context.Context + log Logger + prefix string + interval time.Duration + fsf *filters.DatasetMapFilter + snapshotsTaken chan<-struct{} +} + +type Snapper struct { + args args + + mtx sync.Mutex + state State + + // set in state Plan, used in Waiting + lastInvocation time.Time + + // valid for state Snapshotting + plan map[*zfs.DatasetPath]snapProgress + + // valid for state SyncUp and Waiting + sleepUntil time.Time + + // valid for state Err + err error +} + +//go:generate stringer -type=State +type State uint + +const ( + SyncUp State = 1<%s", pre, post)). + Debug("state transition") + + } + +} + +func onErr(err error, u updater) state { + return u(func(s *Snapper) { + s.err = err + s.state = ErrorWait + }).sf() +} + +func onMainCtxDone(ctx context.Context, u updater) state { + return u(func(s *Snapper) { + s.err = ctx.Err() + s.state = Stopped + }).sf() +} + +func syncUp(a args, u updater) state { + fss, err := listFSes(a.fsf) + if err != nil { + return onErr(err, u) + } + syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval) + if err != nil { + return onErr(err, u) + } + u(func(s *Snapper){ + s.sleepUntil = syncPoint + }) + t := time.NewTimer(syncPoint.Sub(time.Now())) + defer t.Stop() + select { + case <-t.C: + return u(func(s *Snapper) { + s.state = Planning + }).sf() + case <-a.ctx.Done(): + return onMainCtxDone(a.ctx, u) + } +} + +func plan(a args, u updater) state { + u(func(snapper *Snapper) { + snapper.lastInvocation = time.Now() + }) + fss, err := listFSes(a.fsf) + if err != nil { + return onErr(err, u) + } + + plan := make(map[*zfs.DatasetPath]snapProgress, len(fss)) + for _, fs := range fss { + plan[fs] = snapProgress{state: SnapPending} + } + return u(func(s *Snapper) { + s.state = Snapshotting + s.plan = plan + }).sf() +} + +func snapshot(a args, u updater) state { + + var plan map[*zfs.DatasetPath]snapProgress + u(func(snapper *Snapper) { + plan = snapper.plan + }) + + hadErr := false + // TODO channel programs -> allow a little jitter? + for fs, progress := range plan { + suffix := time.Now().In(time.UTC).Format("20060102_150405_000") + snapname := fmt.Sprintf("%s%s", a.prefix, suffix) + + l := a.log. + WithField("fs", fs.ToString()). + WithField("snap", snapname) + + u(func(snapper *Snapper) { + progress.name = snapname + progress.startAt = time.Now() + progress.state = SnapStarted + }) + + l.Debug("create snapshot") + err := zfs.ZFSSnapshot(fs, snapname, false) + if err != nil { + hadErr = true + l.WithError(err).Error("cannot create snapshot") + } + doneAt := time.Now() + + u(func(snapper *Snapper) { + progress.doneAt = doneAt + progress.state = SnapDone + if err != nil { + progress.state = SnapError + progress.err = err + } + }) + } + + select { + case a.snapshotsTaken <- struct{}{}: + default: + if a.snapshotsTaken != nil { + a.log.Warn("callback channel is full, discarding snapshot update event") + } + } + + return u(func(snapper *Snapper) { + if hadErr { + snapper.state = ErrorWait + snapper.err = errors.New("one or more snapshots could not be created, check logs for details") + } else { + snapper.state = Waiting + } + }).sf() +} + +func wait(a args, u updater) state { + var sleepUntil time.Time + u(func(snapper *Snapper) { + lastTick := snapper.lastInvocation + snapper.sleepUntil = lastTick.Add(a.interval) + sleepUntil = snapper.sleepUntil + }) + + t := time.NewTimer(sleepUntil.Sub(time.Now())) + defer t.Stop() + + select { + case <-t.C: + return u(func(snapper *Snapper) { + snapper.state = Planning + }).sf() + case <-a.ctx.Done(): + return onMainCtxDone(a.ctx, u) + } +} + +func listFSes(mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) { + return zfs.ZFSListMapping(mf) +} + +func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) { + type snapTime struct { + ds *zfs.DatasetPath + time time.Time + } + + if len(fss) == 0 { + return time.Now(), nil + } + + snaptimes := make([]snapTime, 0, len(fss)) + + now := time.Now() + + log.Debug("examine filesystem state") + for _, d := range fss { + + l := log.WithField("fs", d.ToString()) + + fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot)) + if err != nil { + l.WithError(err).Error("cannot list filesystem versions") + continue + } + if len(fsvs) <= 0 { + l.WithField("prefix", prefix).Debug("no filesystem versions with prefix") + continue + } + + // Sort versions by creation + sort.SliceStable(fsvs, func(i, j int) bool { + return fsvs[i].CreateTXG < fsvs[j].CreateTXG + }) + + latest := fsvs[len(fsvs)-1] + l.WithField("creation", latest.Creation). + Debug("found latest snapshot") + + since := now.Sub(latest.Creation) + if since < 0 { + l.WithField("snapshot", latest.Name). + WithField("creation", latest.Creation). + Error("snapshot is from the future") + continue + } + next := now + if since < interval { + next = latest.Creation.Add(interval) + } + snaptimes = append(snaptimes, snapTime{d, next}) + } + + if len(snaptimes) == 0 { + snaptimes = append(snaptimes, snapTime{nil, now}) + } + + sort.Slice(snaptimes, func(i, j int) bool { + return snaptimes[i].time.Before(snaptimes[j].time) + }) + + return snaptimes[0].time, nil + +} + diff --git a/daemon/snapper/snapper_all.go b/daemon/snapper/snapper_all.go new file mode 100644 index 0000000..22281dc --- /dev/null +++ b/daemon/snapper/snapper_all.go @@ -0,0 +1,39 @@ +package snapper + +import ( + "context" + "fmt" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/filters" +) + +// FIXME: properly abstract snapshotting: +// - split up things that trigger snapshotting from the mechanism +// - timer-based trigger (periodic) +// - call from control socket (manual) +// - mixed modes? +// - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual +type PeriodicOrManual struct { + s *Snapper +} + +func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan <- struct{}) { + if s.s != nil { + s.s.Run(ctx, wakeUpCommon) + } +} + +func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) { + switch v := in.Ret.(type) { + case *config.SnapshottingPeriodic: + snapper, err := PeriodicFromConfig(g, fsf, v) + if err != nil { + return nil, err + } + return &PeriodicOrManual{snapper}, nil + case *config.SnapshottingManual: + return &PeriodicOrManual{}, nil + default: + return nil, fmt.Errorf("unknown snapshotting type %T", v) + } +} diff --git a/daemon/snapper/snapstate_string.go b/daemon/snapper/snapstate_string.go new file mode 100644 index 0000000..faa7347 --- /dev/null +++ b/daemon/snapper/snapstate_string.go @@ -0,0 +1,29 @@ +// Code generated by "stringer -type=SnapState"; DO NOT EDIT. + +package snapper + +import "strconv" + +const ( + _SnapState_name_0 = "SnapPendingSnapStarted" + _SnapState_name_1 = "SnapDone" + _SnapState_name_2 = "SnapError" +) + +var ( + _SnapState_index_0 = [...]uint8{0, 11, 22} +) + +func (i SnapState) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _SnapState_name_0[_SnapState_index_0[i]:_SnapState_index_0[i+1]] + case i == 4: + return _SnapState_name_1 + case i == 8: + return _SnapState_name_2 + default: + return "SnapState(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/daemon/snapper/state_string.go b/daemon/snapper/state_string.go new file mode 100644 index 0000000..53796b0 --- /dev/null +++ b/daemon/snapper/state_string.go @@ -0,0 +1,35 @@ +// Code generated by "stringer -type=State"; DO NOT EDIT. + +package snapper + +import "strconv" + +const ( + _State_name_0 = "SyncUpPlanning" + _State_name_1 = "Snapshotting" + _State_name_2 = "Waiting" + _State_name_3 = "ErrorWait" + _State_name_4 = "Stopped" +) + +var ( + _State_index_0 = [...]uint8{0, 6, 14} +) + +func (i State) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _State_name_0[_State_index_0[i]:_State_index_0[i+1]] + case i == 4: + return _State_name_1 + case i == 8: + return _State_name_2 + case i == 16: + return _State_name_3 + case i == 32: + return _State_name_4 + default: + return "State(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/daemon/streamrpcconfig/streamrpcconfig.go b/daemon/streamrpcconfig/streamrpcconfig.go new file mode 100644 index 0000000..da28d5d --- /dev/null +++ b/daemon/streamrpcconfig/streamrpcconfig.go @@ -0,0 +1,25 @@ +package streamrpcconfig + +import ( + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/config" +) + +func FromDaemonConfig(g *config.Global, in *config.RPCConfig) (*streamrpc.ConnConfig, error) { + conf := in + if conf == nil { + conf = g.RPC + } + srpcConf := &streamrpc.ConnConfig{ + RxHeaderMaxLen: conf.RxHeaderMaxLen, + RxStructuredMaxLen: conf.RxStructuredMaxLen, + RxStreamMaxChunkSize: conf.RxStreamChunkMaxLen, + TxChunkSize: conf.TxChunkSize, + Timeout: conf.Timeout, + SendHeartbeatInterval: conf.SendHeartbeatInterval, + } + if err := srpcConf.Validate(); err != nil { + return nil, err + } + return srpcConf, nil +} diff --git a/daemon/transport/connecter/connect_local.go b/daemon/transport/connecter/connect_local.go new file mode 100644 index 0000000..45c3d68 --- /dev/null +++ b/daemon/transport/connecter/connect_local.go @@ -0,0 +1,30 @@ +package connecter + +import ( + "context" + "fmt" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/transport/serve" + "net" +) + +type LocalConnecter struct { + listenerName string + clientIdentity string +} + +func LocalConnecterFromConfig(in *config.LocalConnect) (*LocalConnecter, error) { + if in.ClientIdentity == "" { + return nil, fmt.Errorf("ClientIdentity must not be empty") + } + if in.ListenerName == "" { + return nil, fmt.Errorf("ListenerName must not be empty") + } + return &LocalConnecter{listenerName: in.ListenerName, clientIdentity: in.ClientIdentity}, nil +} + +func (c *LocalConnecter) Connect(dialCtx context.Context) (conn net.Conn, err error) { + l := serve.GetLocalListener(c.listenerName) + return l.Connect(dialCtx, c.clientIdentity) +} + diff --git a/daemon/transport/connecter/connect_ssh.go b/daemon/transport/connecter/connect_ssh.go new file mode 100644 index 0000000..7efeec5 --- /dev/null +++ b/daemon/transport/connecter/connect_ssh.go @@ -0,0 +1,66 @@ +package connecter + +import ( + "context" + "github.com/jinzhu/copier" + "github.com/pkg/errors" + "github.com/problame/go-netssh" + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/config" + "net" + "time" +) + +type SSHStdinserverConnecter struct { + Host string + User string + Port uint16 + IdentityFile string + TransportOpenCommand []string + SSHCommand string + Options []string + dialTimeout time.Duration +} + +var _ streamrpc.Connecter = &SSHStdinserverConnecter{} + +func SSHStdinserverConnecterFromConfig(in *config.SSHStdinserverConnect) (c *SSHStdinserverConnecter, err error) { + + c = &SSHStdinserverConnecter{ + Host: in.Host, + User: in.User, + Port: in.Port, + IdentityFile: in.IdentityFile, + SSHCommand: in.SSHCommand, + Options: in.Options, + dialTimeout: in.DialTimeout, + } + return + +} + +type netsshConnToConn struct{ *netssh.SSHConn } + +var _ net.Conn = netsshConnToConn{} + +func (netsshConnToConn) SetDeadline(dl time.Time) error { return nil } +func (netsshConnToConn) SetReadDeadline(dl time.Time) error { return nil } +func (netsshConnToConn) SetWriteDeadline(dl time.Time) error { return nil } + +func (c *SSHStdinserverConnecter) Connect(dialCtx context.Context) (net.Conn, error) { + + var endpoint netssh.Endpoint + if err := copier.Copy(&endpoint, c); err != nil { + return nil, errors.WithStack(err) + } + dialCtx, dialCancel := context.WithTimeout(dialCtx, c.dialTimeout) // context.TODO tied to error handling below + defer dialCancel() + nconn, err := netssh.Dial(dialCtx, endpoint) + if err != nil { + if err == context.DeadlineExceeded { + err = errors.Errorf("dial_timeout of %s exceeded", c.dialTimeout) + } + return nil, err + } + return netsshConnToConn{nconn}, nil +} diff --git a/daemon/transport/connecter/connect_tcp.go b/daemon/transport/connecter/connect_tcp.go new file mode 100644 index 0000000..3d8b77e --- /dev/null +++ b/daemon/transport/connecter/connect_tcp.go @@ -0,0 +1,24 @@ +package connecter + +import ( + "context" + "github.com/zrepl/zrepl/config" + "net" +) + +type TCPConnecter struct { + Address string + dialer net.Dialer +} + +func TCPConnecterFromConfig(in *config.TCPConnect) (*TCPConnecter, error) { + dialer := net.Dialer{ + Timeout: in.DialTimeout, + } + + return &TCPConnecter{in.Address, dialer}, nil +} + +func (c *TCPConnecter) Connect(dialCtx context.Context) (conn net.Conn, err error) { + return c.dialer.DialContext(dialCtx, "tcp", c.Address) +} diff --git a/daemon/transport/connecter/connect_tls.go b/daemon/transport/connecter/connect_tls.go new file mode 100644 index 0000000..28f3440 --- /dev/null +++ b/daemon/transport/connecter/connect_tls.go @@ -0,0 +1,43 @@ +package connecter + +import ( + "context" + "crypto/tls" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/tlsconf" + "net" +) + +type TLSConnecter struct { + Address string + dialer net.Dialer + tlsConfig *tls.Config +} + +func TLSConnecterFromConfig(in *config.TLSConnect) (*TLSConnecter, error) { + dialer := net.Dialer{ + Timeout: in.DialTimeout, + } + + ca, err := tlsconf.ParseCAFile(in.Ca) + if err != nil { + return nil, errors.Wrap(err, "cannot parse ca file") + } + + cert, err := tls.LoadX509KeyPair(in.Cert, in.Key) + if err != nil { + return nil, errors.Wrap(err, "cannot parse cert/key pair") + } + + tlsConfig, err := tlsconf.ClientAuthClient(in.ServerCN, ca, cert) + if err != nil { + return nil, errors.Wrap(err, "cannot build tls config") + } + + return &TLSConnecter{in.Address, dialer, tlsConfig}, nil +} + +func (c *TLSConnecter) Connect(dialCtx context.Context) (conn net.Conn, err error) { + return tls.DialWithDialer(&c.dialer, "tcp", c.Address, c.tlsConfig) +} diff --git a/daemon/transport/connecter/connecter.go b/daemon/transport/connecter/connecter.go new file mode 100644 index 0000000..fa772a7 --- /dev/null +++ b/daemon/transport/connecter/connecter.go @@ -0,0 +1,84 @@ +package connecter + +import ( + "context" + "fmt" + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/streamrpcconfig" + "github.com/zrepl/zrepl/daemon/transport" + "net" + "time" +) + + +type HandshakeConnecter struct { + connecter streamrpc.Connecter +} + +func (c HandshakeConnecter) Connect(ctx context.Context) (net.Conn, error) { + conn, err := c.connecter.Connect(ctx) + if err != nil { + return nil, err + } + dl, ok := ctx.Deadline() + if !ok { + dl = time.Now().Add(10 * time.Second) // FIXME constant + } + if err := transport.DoHandshakeCurrentVersion(conn, dl); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + + + +func FromConfig(g *config.Global, in config.ConnectEnum) (*ClientFactory, error) { + var ( + connecter streamrpc.Connecter + errConnecter, errRPC error + connConf *streamrpc.ConnConfig + ) + switch v := in.Ret.(type) { + case *config.SSHStdinserverConnect: + connecter, errConnecter = SSHStdinserverConnecterFromConfig(v) + connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.TCPConnect: + connecter, errConnecter = TCPConnecterFromConfig(v) + connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.TLSConnect: + connecter, errConnecter = TLSConnecterFromConfig(v) + connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.LocalConnect: + connecter, errConnecter = LocalConnecterFromConfig(v) + connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC) + default: + panic(fmt.Sprintf("implementation error: unknown connecter type %T", v)) + } + + if errConnecter != nil { + return nil, errConnecter + } + if errRPC != nil { + return nil, errRPC + } + + config := streamrpc.ClientConfig{ConnConfig: connConf} + if err := config.Validate(); err != nil { + return nil, err + } + + connecter = HandshakeConnecter{connecter} + + return &ClientFactory{connecter: connecter, config: &config}, nil +} + +type ClientFactory struct { + connecter streamrpc.Connecter + config *streamrpc.ClientConfig +} + +func (f ClientFactory) NewClient() (*streamrpc.Client, error) { + return streamrpc.NewClient(f.connecter, f.config) +} diff --git a/daemon/transport/handshake.go b/daemon/transport/handshake.go new file mode 100644 index 0000000..ecfd495 --- /dev/null +++ b/daemon/transport/handshake.go @@ -0,0 +1,136 @@ +package transport + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "time" + "unicode/utf8" +) + +type HandshakeMessage struct { + ProtocolVersion int + Extensions []string +} + +func (m *HandshakeMessage) Encode() ([]byte, error) { + if m.ProtocolVersion <= 0 || m.ProtocolVersion > 9999 { + return nil, fmt.Errorf("protocol version must be in [1, 9999]") + } + if len(m.Extensions) >= 9999 { + return nil, fmt.Errorf("protocol only supports [0, 9999] extensions") + } + // EXTENSIONS is a count of subsequent \n separated lines that contain protocol extensions + var extensions strings.Builder + for i, ext := range m.Extensions { + if strings.ContainsAny(ext, "\n") { + return nil, fmt.Errorf("Extension #%d contains forbidden newline character", i) + } + if !utf8.ValidString(ext) { + return nil, fmt.Errorf("Extension #%d is not valid UTF-8", i) + } + extensions.WriteString(ext) + extensions.WriteString("\n") + } + withoutLen := fmt.Sprintf("ZREPL_ZFS_REPLICATION PROTOVERSION=%04d EXTENSIONS=%04d\n%s", + m.ProtocolVersion, len(m.Extensions), extensions.String()) + withLen := fmt.Sprintf("%010d %s", len(withoutLen), withoutLen) + return []byte(withLen), nil +} + +func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { + var lenAndSpace [11]byte + if _, err := io.ReadFull(r, lenAndSpace[:]); err != nil { + return err + } + if !utf8.Valid(lenAndSpace[:]) { + return fmt.Errorf("invalid start of handshake message: not valid UTF-8") + } + var followLen int + n, err := fmt.Sscanf(string(lenAndSpace[:]), "%010d ", &followLen) + if n != 1 || err != nil { + return fmt.Errorf("could not parse handshake message length") + } + if followLen > maxLen { + return fmt.Errorf("handshake message length exceeds max length (%d vs %d)", + followLen, maxLen) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, io.LimitReader(r, int64(followLen))) + if err != nil { + return err + } + + var ( + protoVersion, extensionCount int + ) + n, err = fmt.Fscanf(&buf, "ZREPL_ZFS_REPLICATION PROTOVERSION=%04d EXTENSIONS=%4d\n", + &protoVersion, &extensionCount) + if n != 2 || err != nil { + return fmt.Errorf("could not parse handshake message: %s", err) + } + if protoVersion < 1 { + return fmt.Errorf("invalid protocol version %q", protoVersion) + } + m.ProtocolVersion = protoVersion + + if extensionCount < 0 { + return fmt.Errorf("invalid extension count %q", extensionCount) + } + if extensionCount == 0 { + if buf.Len() != 0 { + return fmt.Errorf("unexpected data trailing after header") + } + m.Extensions = nil + return nil + } + s := buf.String() + if strings.Count(s, "\n") != extensionCount { + return fmt.Errorf("inconsistent extension count: found %d, header says %d", len(m.Extensions), extensionCount) + } + exts := strings.Split(s, "\n") + if exts[len(exts)-1] != "" { + return fmt.Errorf("unexpected data trailing after last extension newline") + } + m.Extensions = exts[0:len(exts)-1] + + return nil +} + +func DoHandshakeCurrentVersion(conn net.Conn, deadline time.Time) error { + // current protocol version is hardcoded here + return DoHandshakeVersion(conn, deadline, 1) +} + +func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) error { + ours := HandshakeMessage{ + ProtocolVersion: version, + Extensions: nil, + } + hsb, err := ours.Encode() + if err != nil { + return fmt.Errorf("could not encode protocol banner: %s", err) + } + + conn.SetDeadline(deadline) + _, err = io.Copy(conn, bytes.NewBuffer(hsb)) + if err != nil { + return fmt.Errorf("could not send protocol banner: %s", err) + } + + theirs := HandshakeMessage{} + if err := theirs.DecodeReader(conn, 16 * 4096); err != nil { // FIXME constant + return fmt.Errorf("could not decode protocol banner: %s", err) + } + + if theirs.ProtocolVersion != ours.ProtocolVersion { + return fmt.Errorf("protocol versions do not match: ours is %d, theirs is %d", + ours.ProtocolVersion, theirs.ProtocolVersion) + } + // ignore extensions, we don't use them + + return nil +} diff --git a/daemon/transport/handshake_test.go b/daemon/transport/handshake_test.go new file mode 100644 index 0000000..d1c72b4 --- /dev/null +++ b/daemon/transport/handshake_test.go @@ -0,0 +1,119 @@ +package transport + +import ( + "bytes" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zrepl/zrepl/util/socketpair" + "io" + "strings" + "testing" + "time" +) + +func TestHandshakeMessage_Encode(t *testing.T) { + + msg := HandshakeMessage{ + ProtocolVersion: 2342, + } + + encB, err := msg.Encode() + require.NoError(t, err) + enc := string(encB) + t.Logf("enc: %s", enc) + + + + assert.False(t, strings.ContainsAny(enc[0:10], " ")) + assert.True(t, enc[10] == ' ') + + var ( + headerlen, protoversion, extensionCount int + ) + n, err := fmt.Sscanf(enc, "%010d ZREPL_ZFS_REPLICATION PROTOVERSION=%04d EXTENSIONS=%04d\n", + &headerlen, &protoversion, &extensionCount) + if n != 3 || (err != nil && err != io.EOF) { + t.Fatalf("%v %v", n, err) + } + + assert.Equal(t, 2342, protoversion) + assert.Equal(t, 0, extensionCount) + assert.Equal(t, len(enc)-11, headerlen) + +} + +func TestHandshakeMessage_Encode_InvalidProtocolVersion(t *testing.T) { + + for _, pv := range []int{-1, 0, 10000, 10001} { + t.Logf("testing invalid protocol version = %v", pv) + msg := HandshakeMessage{ + ProtocolVersion: pv, + } + b, err := msg.Encode() + assert.Error(t, err) + assert.Nil(t, b) + } + +} + +func TestHandshakeMessage_DecodeReader(t *testing.T) { + + in := HandshakeMessage{ + 2342, + []string{"foo", "bar 2342"}, + } + + enc, err := in.Encode() + require.NoError(t, err) + + out := HandshakeMessage{} + err = out.DecodeReader(bytes.NewReader([]byte(enc)), 4 * 4096) + assert.NoError(t, err) + assert.Equal(t, 2342, out.ProtocolVersion) + assert.Equal(t, 2, len(out.Extensions)) + assert.Equal(t, "foo", out.Extensions[0]) + assert.Equal(t, "bar 2342", out.Extensions[1]) + +} + +func TestDoHandshakeVersion_ErrorOnDifferentVersions(t *testing.T) { + srv, client, err := socketpair.SocketPair() + if err != nil { + t.Fatal(err) + } + defer srv.Close() + defer client.Close() + + srvErrCh := make(chan error) + go func() { + srvErrCh <- DoHandshakeVersion(srv, time.Now().Add(2*time.Second), 1) + }() + err = DoHandshakeVersion(client, time.Now().Add(2*time.Second), 2) + t.Log(err) + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "version")) + + srvErr := <-srvErrCh + t.Log(srvErr) + assert.Error(t, srvErr) + assert.True(t, strings.Contains(srvErr.Error(), "version")) +} + +func TestDoHandshakeCurrentVersion(t *testing.T) { + srv, client, err := socketpair.SocketPair() + if err != nil { + t.Fatal(err) + } + defer srv.Close() + defer client.Close() + + srvErrCh := make(chan error) + go func() { + srvErrCh <- DoHandshakeVersion(srv, time.Now().Add(2*time.Second), 1) + }() + err = DoHandshakeVersion(client, time.Now().Add(2*time.Second), 1) + assert.Nil(t, err) + assert.Nil(t, <-srvErrCh) + +} diff --git a/daemon/transport/serve/serve.go b/daemon/transport/serve/serve.go new file mode 100644 index 0000000..c1b3bb1 --- /dev/null +++ b/daemon/transport/serve/serve.go @@ -0,0 +1,147 @@ +package serve + +import ( + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/transport" + "net" + "github.com/zrepl/zrepl/daemon/streamrpcconfig" + "github.com/problame/go-streamrpc" + "context" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/zfs" + "time" +) + +type contextKey int + +const contextKeyLog contextKey = 0 + +type Logger = logger.Logger + +func WithLogger(ctx context.Context, log Logger) context.Context { + return context.WithValue(ctx, contextKeyLog, log) +} + +func getLogger(ctx context.Context) Logger { + if log, ok := ctx.Value(contextKeyLog).(Logger); ok { + return log + } + return logger.NewNullLogger() +} + +type AuthenticatedConn interface { + net.Conn + // ClientIdentity must be a string that satisfies ValidateClientIdentity + ClientIdentity() string +} + +// A client identity must be a single component in a ZFS filesystem path +func ValidateClientIdentity(in string) (err error) { + path, err := zfs.NewDatasetPath(in) + if err != nil { + return err + } + if path.Length() != 1 { + return errors.New("client identity must be a single path comonent (not empty, no '/')") + } + return nil +} + +type authConn struct { + net.Conn + clientIdentity string +} + +var _ AuthenticatedConn = authConn{} + +func (c authConn) ClientIdentity() string { + if err := ValidateClientIdentity(c.clientIdentity); err != nil { + panic(err) + } + return c.clientIdentity +} + +// like net.Listener, but with an AuthenticatedConn instead of net.Conn +type AuthenticatedListener interface { + Addr() (net.Addr) + Accept(ctx context.Context) (AuthenticatedConn, error) + Close() error +} + +type ListenerFactory interface { + Listen() (AuthenticatedListener, error) +} + +type HandshakeListenerFactory struct { + lf ListenerFactory +} + +func (lf HandshakeListenerFactory) Listen() (AuthenticatedListener, error) { + l, err := lf.lf.Listen() + if err != nil { + return nil, err + } + return HandshakeListener{l}, nil +} + +type HandshakeListener struct { + l AuthenticatedListener +} + +func (l HandshakeListener) Addr() (net.Addr) { return l.l.Addr() } + +func (l HandshakeListener) Close() error { return l.l.Close() } + +func (l HandshakeListener) Accept(ctx context.Context) (AuthenticatedConn, error) { + conn, err := l.l.Accept(ctx) + if err != nil { + return nil, err + } + dl, ok := ctx.Deadline() + if !ok { + dl = time.Now().Add(10*time.Second) // FIXME constant + } + if err := transport.DoHandshakeCurrentVersion(conn, dl); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +func FromConfig(g *config.Global, in config.ServeEnum) (lf ListenerFactory, conf *streamrpc.ConnConfig, _ error) { + + var ( + lfError, rpcErr error + ) + switch v := in.Ret.(type) { + case *config.TCPServe: + lf, lfError = TCPListenerFactoryFromConfig(g, v) + conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.TLSServe: + lf, lfError = TLSListenerFactoryFromConfig(g, v) + conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.StdinserverServer: + lf, lfError = MultiStdinserverListenerFactoryFromConfig(g, v) + conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC) + case *config.LocalServe: + lf, lfError = LocalListenerFactoryFromConfig(g, v) + conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC) + default: + return nil, nil, errors.Errorf("internal error: unknown serve type %T", v) + } + + if lfError != nil { + return nil, nil, lfError + } + if rpcErr != nil { + return nil, nil, rpcErr + } + + lf = HandshakeListenerFactory{lf} + + return lf, conf, nil + +} + + diff --git a/daemon/transport/serve/serve_local.go b/daemon/transport/serve/serve_local.go new file mode 100644 index 0000000..f71ba70 --- /dev/null +++ b/daemon/transport/serve/serve_local.go @@ -0,0 +1,187 @@ +package serve + +import ( + "context" + "fmt" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/util/socketpair" + "net" + "sync" +) + +var localListeners struct { + m map[string]*LocalListener // listenerName -> listener + init sync.Once + mtx sync.Mutex +} + +func GetLocalListener(listenerName string) (*LocalListener) { + + localListeners.init.Do(func() { + localListeners.m = make(map[string]*LocalListener) + }) + + localListeners.mtx.Lock() + defer localListeners.mtx.Unlock() + + l, ok := localListeners.m[listenerName] + if !ok { + l = newLocalListener() + localListeners.m[listenerName] = l + } + return l + +} + +type connectRequest struct { + clientIdentity string + callback chan connectResult +} + +type connectResult struct { + conn net.Conn + err error +} + +type LocalListener struct { + connects chan connectRequest +} + +func newLocalListener() *LocalListener { + return &LocalListener{ + connects: make(chan connectRequest), + } +} + +// Connect to the LocalListener from a client with identity clientIdentity +func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string) (conn net.Conn, err error) { + + // place request + req := connectRequest{ + clientIdentity: clientIdentity, + callback: make(chan connectResult), + } + select { + case l.connects <- req: + case <-dialCtx.Done(): + return nil, dialCtx.Err() + } + + // wait for listener response + select { + case connRes := <- req.callback: + conn, err = connRes.conn, connRes.err + case <-dialCtx.Done(): + close(req.callback) // sending to the channel afterwards will panic, the listener has to catch this + conn, err = nil, dialCtx.Err() + } + + return conn, err +} + +type localAddr struct { + S string +} + +func (localAddr) Network() string { return "local" } + +func (a localAddr) String() string { return a.S } + +func (l *LocalListener) Addr() (net.Addr) { return localAddr{""} } + +type localConn struct { + net.Conn + clientIdentity string +} + +func (l localConn) ClientIdentity() string { return l.clientIdentity } + +func (l *LocalListener) Accept(ctx context.Context) (AuthenticatedConn, error) { + respondToRequest := func(req connectRequest, res connectResult) (err error) { + getLogger(ctx). + WithField("res.conn", res.conn).WithField("res.err", res.err). + Debug("responding to client request") + defer func() { + errv := recover() + getLogger(ctx).WithField("recover_err", errv). + Debug("panic on send to client callback, likely a legitimate client-side timeout") + }() + select { + case req.callback <- res: + err = nil + default: + err = fmt.Errorf("client-provided callback did block on send") + } + close(req.callback) + return err + } + + getLogger(ctx).Debug("waiting for local client connect requests") + var req connectRequest + select { + case req = <-l.connects: + case <-ctx.Done(): + return nil, ctx.Err() + } + + getLogger(ctx).WithField("client_identity", req.clientIdentity).Debug("got connect request") + if req.clientIdentity == "" { + res := connectResult{nil, fmt.Errorf("client identity must not be empty")} + if err := respondToRequest(req, res); err != nil { + return nil, err + } + return nil, fmt.Errorf("client connected with empty client identity") + } + + getLogger(ctx).Debug("creating socketpair") + left, right, err := socketpair.SocketPair() + if err != nil { + res := connectResult{nil, fmt.Errorf("server error: %s", err)} + if respErr := respondToRequest(req, res); respErr != nil { + // returning the socketpair error properly is more important than the error sent to the client + getLogger(ctx).WithError(respErr).Error("error responding to client") + } + return nil, err + } + + getLogger(ctx).Debug("responding with left side of socketpair") + res := connectResult{left, nil} + if err := respondToRequest(req, res); err != nil { + getLogger(ctx).WithError(err).Error("error responding to client") + if err := left.Close(); err != nil { + getLogger(ctx).WithError(err).Error("cannot close left side of socketpair") + } + if err := right.Close(); err != nil { + getLogger(ctx).WithError(err).Error("cannot close right side of socketpair") + } + return nil, err + } + + return localConn{right, req.clientIdentity}, nil +} + +func (l *LocalListener) Close() error { + // FIXME: make sure concurrent Accepts return with error, and further Accepts return that error, too + // Example impl: for each accept, do context.WithCancel, and store the cancel in a list + // When closing, set a member variable to state=closed, make sure accept will exit early + // and then call all cancels in the list + // The code path from Accept entry over check if state=closed to list entry must be protected by a mutex. + return nil +} + +type LocalListenerFactory struct { + listenerName string +} + +func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (f *LocalListenerFactory, err error) { + if in.ListenerName == "" { + return nil, fmt.Errorf("ListenerName must not be empty") + } + return &LocalListenerFactory{listenerName: in.ListenerName}, nil +} + + +func (lf *LocalListenerFactory) Listen() (AuthenticatedListener, error) { + return GetLocalListener(lf.listenerName), nil +} + diff --git a/daemon/transport/serve/serve_stdinserver.go b/daemon/transport/serve/serve_stdinserver.go new file mode 100644 index 0000000..f02bf20 --- /dev/null +++ b/daemon/transport/serve/serve_stdinserver.go @@ -0,0 +1,158 @@ +package serve + +import ( + "github.com/problame/go-netssh" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/nethelpers" + "io" + "net" + "path" + "time" + "context" + "github.com/pkg/errors" + "sync/atomic" +) + +type StdinserverListenerFactory struct { + ClientIdentities []string + Sockdir string +} + +func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (f *multiStdinserverListenerFactory, err error) { + + for _, ci := range in.ClientIdentities { + if err := ValidateClientIdentity(ci); err != nil { + return nil, errors.Wrapf(err, "invalid client identity %q", ci) + } + } + + f = &multiStdinserverListenerFactory{ + ClientIdentities: in.ClientIdentities, + Sockdir: g.Serve.StdinServer.SockDir, + } + + return +} + +type multiStdinserverListenerFactory struct { + ClientIdentities []string + Sockdir string +} + +func (f *multiStdinserverListenerFactory) Listen() (AuthenticatedListener, error) { + return multiStdinserverListenerFromClientIdentities(f.Sockdir, f.ClientIdentities) +} + +type multiStdinserverAcceptRes struct { + conn AuthenticatedConn + err error +} + +type MultiStdinserverListener struct { + listeners []*stdinserverListener + accepts chan multiStdinserverAcceptRes + closed int32 +} + +// client identities must be validated +func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string) (*MultiStdinserverListener, error) { + listeners := make([]*stdinserverListener, 0, len(cis)) + var err error + for _, ci := range cis { + sockpath := path.Join(sockdir, ci) + l := &stdinserverListener{clientIdentity: ci} + if err = nethelpers.PreparePrivateSockpath(sockpath); err != nil { + break + } + if l.l, err = netssh.Listen(sockpath); err != nil { + break + } + listeners = append(listeners, l) + } + if err != nil { + for _, l := range listeners { + l.Close() // FIXME error reporting? + } + return nil, err + } + return &MultiStdinserverListener{listeners: listeners}, nil +} + +func (m *MultiStdinserverListener) Accept(ctx context.Context) (AuthenticatedConn, error){ + + if m.accepts == nil { + m.accepts = make(chan multiStdinserverAcceptRes, len(m.listeners)) + for i := range m.listeners { + go func(i int) { + for atomic.LoadInt32(&m.closed) == 0 { + conn, err := m.listeners[i].Accept(context.TODO()) + m.accepts <- multiStdinserverAcceptRes{conn, err} + } + }(i) + } + } + + res := <- m.accepts + return res.conn, res.err + +} + +func (m *MultiStdinserverListener) Addr() (net.Addr) { + return netsshAddr{} +} + +func (m *MultiStdinserverListener) Close() error { + atomic.StoreInt32(&m.closed, 1) + var oneErr error + for _, l := range m.listeners { + if err := l.Close(); err != nil && oneErr == nil { + oneErr = err + } + } + return oneErr +} + +// a single stdinserverListener (part of multiStinserverListener) +type stdinserverListener struct { + l *netssh.Listener + clientIdentity string +} + +func (l stdinserverListener) Addr() net.Addr { + return netsshAddr{} +} + +func (l stdinserverListener) Accept(ctx context.Context) (AuthenticatedConn, error) { + c, err := l.l.Accept() + if err != nil { + return nil, err + } + return netsshConnToNetConnAdatper{c, l.clientIdentity}, nil +} + +func (l stdinserverListener) Close() (err error) { + return l.l.Close() +} + +type netsshAddr struct{} + +func (netsshAddr) Network() string { return "netssh" } +func (netsshAddr) String() string { return "???" } + +type netsshConnToNetConnAdatper struct { + io.ReadWriteCloser // works for both netssh.SSHConn and netssh.ServeConn + clientIdentity string +} + +func (a netsshConnToNetConnAdatper) ClientIdentity() string { return a.clientIdentity } + +func (netsshConnToNetConnAdatper) LocalAddr() net.Addr { return netsshAddr{} } + +func (netsshConnToNetConnAdatper) RemoteAddr() net.Addr { return netsshAddr{} } + +// FIXME log warning once! +func (netsshConnToNetConnAdatper) SetDeadline(t time.Time) error { return nil } + +func (netsshConnToNetConnAdatper) SetReadDeadline(t time.Time) error { return nil } + +func (netsshConnToNetConnAdatper) SetWriteDeadline(t time.Time) error { return nil } diff --git a/daemon/transport/serve/serve_tcp.go b/daemon/transport/serve/serve_tcp.go new file mode 100644 index 0000000..957d3b9 --- /dev/null +++ b/daemon/transport/serve/serve_tcp.go @@ -0,0 +1,91 @@ +package serve + +import ( + "github.com/zrepl/zrepl/config" + "net" + "github.com/pkg/errors" + "context" +) + +type TCPListenerFactory struct { + address *net.TCPAddr + clientMap *ipMap +} + +type ipMapEntry struct { + ip net.IP + ident string +} + +type ipMap struct { + entries []ipMapEntry +} + +func ipMapFromConfig(clients map[string]string) (*ipMap, error) { + entries := make([]ipMapEntry, 0, len(clients)) + for clientIPString, clientIdent := range clients { + clientIP := net.ParseIP(clientIPString) + if clientIP == nil { + return nil, errors.Errorf("cannot parse client IP %q", clientIPString) + } + if err := ValidateClientIdentity(clientIdent); err != nil { + return nil, errors.Wrapf(err,"invalid client identity for IP %q", clientIPString) + } + entries = append(entries, ipMapEntry{clientIP, clientIdent}) + } + return &ipMap{entries: entries}, nil +} + +func (m *ipMap) Get(ip net.IP) (string, error) { + for _, e := range m.entries { + if e.ip.Equal(ip) { + return e.ident, nil + } + } + return "", errors.Errorf("no identity mapping for client IP %s", ip) +} + +func TCPListenerFactoryFromConfig(c *config.Global, in *config.TCPServe) (*TCPListenerFactory, error) { + addr, err := net.ResolveTCPAddr("tcp", in.Listen) + if err != nil { + return nil, errors.Wrap(err, "cannot parse listen address") + } + clientMap, err := ipMapFromConfig(in.Clients) + if err != nil { + return nil, errors.Wrap(err, "cannot parse client IP map") + } + lf := &TCPListenerFactory{ + address: addr, + clientMap: clientMap, + } + return lf, nil +} + +func (f *TCPListenerFactory) Listen() (AuthenticatedListener, error) { + l, err := net.ListenTCP("tcp", f.address) + if err != nil { + return nil, err + } + return &TCPAuthListener{l, f.clientMap}, nil +} + +type TCPAuthListener struct { + *net.TCPListener + clientMap *ipMap +} + +func (f *TCPAuthListener) Accept(ctx context.Context) (AuthenticatedConn, error) { + nc, err := f.TCPListener.Accept() + if err != nil { + return nil, err + } + clientIP := nc.RemoteAddr().(*net.TCPAddr).IP + clientIdent, err := f.clientMap.Get(clientIP) + if err != nil { + getLogger(ctx).WithField("ip", clientIP).Error("client IP not in client map") + nc.Close() + return nil, err + } + return authConn{nc, clientIdent}, nil +} + diff --git a/daemon/transport/serve/serve_tls.go b/daemon/transport/serve/serve_tls.go new file mode 100644 index 0000000..bc95e41 --- /dev/null +++ b/daemon/transport/serve/serve_tls.go @@ -0,0 +1,83 @@ +package serve + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/tlsconf" + "net" + "time" + "context" +) + +type TLSListenerFactory struct { + address string + clientCA *x509.CertPool + serverCert tls.Certificate + handshakeTimeout time.Duration + clientCNs map[string]struct{} +} + +func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (lf *TLSListenerFactory, err error) { + lf = &TLSListenerFactory{ + address: in.Listen, + handshakeTimeout: in.HandshakeTimeout, + } + + if in.Ca == "" || in.Cert == "" || in.Key == "" { + return nil, errors.New("fields 'ca', 'cert' and 'key'must be specified") + } + + lf.clientCA, err = tlsconf.ParseCAFile(in.Ca) + if err != nil { + return nil, errors.Wrap(err, "cannot parse ca file") + } + + lf.serverCert, err = tls.LoadX509KeyPair(in.Cert, in.Key) + if err != nil { + return nil, errors.Wrap(err, "cannot parse cer/key pair") + } + + lf.clientCNs = make(map[string]struct{}, len(in.ClientCNs)) + for i, cn := range in.ClientCNs { + if err := ValidateClientIdentity(cn); err != nil { + return nil, errors.Wrapf(err, "unsuitable client_cn #%d %q", i, cn) + } + // dupes are ok fr now + lf.clientCNs[cn] = struct{}{} + } + + return lf, nil +} + +func (f *TLSListenerFactory) Listen() (AuthenticatedListener, error) { + l, err := net.Listen("tcp", f.address) + if err != nil { + return nil, err + } + tl := tlsconf.NewClientAuthListener(l, f.clientCA, f.serverCert, f.handshakeTimeout) + return tlsAuthListener{tl, f.clientCNs}, nil +} + +type tlsAuthListener struct { + *tlsconf.ClientAuthListener + clientCNs map[string]struct{} +} + +func (l tlsAuthListener) Accept(ctx context.Context) (AuthenticatedConn, error) { + c, cn, err := l.ClientAuthListener.Accept() + if err != nil { + return nil, err + } + if _, ok := l.clientCNs[cn]; !ok { + if err := c.Close(); err != nil { + getLogger(ctx).WithError(err).Error("error closing connection with unauthorized common name") + } + return nil, fmt.Errorf("unauthorized client common name %q from %s", cn, c.RemoteAddr()) + } + return authConn{c, cn}, nil +} + + diff --git a/docs/changelog.rst b/docs/changelog.rst index db22ed1..eb2909a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,82 +1,103 @@ -.. |break_config| replace:: **[BREAK]** +.. |break_config| replace:: **[CONFIG]** .. |break| replace:: **[BREAK]** .. |bugfix| replace:: [BUG] .. |docs| replace:: [DOCS] .. |feature| replace:: [FEATURE] +.. _changelog: + Changelog ========= -The changelog summarized bugfixes that are deemed relevant for users. +The changelog summarizes bugfixes that are deemed relevant for users and package maintainers. Developers should consult the git commit log or GitHub issue tracker. -0.0.4 (unreleased) +We use the following annotations for classifying changes: + +* |break_config| Change that breaks the config. + As a package maintainer, make sure to warn your users about config breakage somehow. +* |break| Change that breaks interoperability or persistent state representation with previous releases. + As a package maintainer, make sure to warn your users about config breakage somehow. + Note that even updating the package on both sides might not be sufficient, e.g. if persistent state needs to be migrated to a new format. +* |feature| Change that introduces new functionality. +* |bugfix| Change that fixes a bug, no regressions or incompatibilities expected. +* |docs| Change to the documentation. + +0.1 (unreleased) ------------------ +This release is a milestone for zrepl and required significant refactoring if not rewrites of substantial parts of the application. +It breaks both configuration and transport format, and thus requires manual intervention and updates on both sides of a replication setup. + +Notes to Package Maintainers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* If the daemon crashes, the stack trace produced by the Go runtime and possibly diagnostic output of zrepl will be written to stderr. + This behavior is independent from the ``stdout`` outlet type. + Please make sure the stderr output of the daemon is captured to a file. + Rotation should not be necessary because stderr is not written to under normal circumstances. + To conserve precious stack traces, make sure that multiple service restarts do not directly discard previous stderr output. +* Make it obvious for users how to set the ``GOTRACEBACK`` environment variable to ``GOTRACEBACK=crash``. + This functionality will cause SIGABRT on panics and can be used to capture a coredump of the panicking process. + To that extend, make sure that your package build system, your OS's coredump collection and the Go delve debugger work together. + Use your build system to package the Go program in `this tutorial on Go coredumps and the delve debugger `_ , and make sure the symbol resolution etc. work on coredumps captured from the binary produced by your build system. (Special focus on symbol stripping, etc.) + +Changes +~~~~~~~ + +* |feature| :issue:`55` : Push replication (see :ref:`push job ` and :ref:`sink job `) +* |feature| :ref:`TCP Transport ` +* |feature| :ref:`TCP + TLS client authentication transport ` +* |feature| :issue:`78` TODO MERGE COMMIT Replication protocol rewrite + + * Uses ``github.com/problame/go-streamrpc`` for RPC layer + * |break| zrepl 0.1 and restart on both sides of a replication setup is required + * |feature| :issue:`83`: Improved error handling of network-level errors (zrepl retries instead of failing the entire job) + * |bugfix| :issue:`75` :issue:`81`: use connection timeouts and protocol-level heartbeats + * |break| |break_config|: mappings are no longer supported + + * Receiving sides (``pull`` and ``sink`` job) specify a single ``root_fs``. + Received filesystems are then stored *per client* in ``${root_fs}/${client_identity}``. + +* |feature| |break| |break_config| Manual snapshotting + triggering of replication + + * |feature| :issue:`69`: include manually created snapshots in replication + * |break_config| ``manual`` and ``periodic`` :ref:`snapshotting types ` + * |feature| ``zrepl wakeup JOB`` subcommand to trigger *just* replication + +* |feature| |break| |break_config| New pruning system + + * The active side of a replication (pull or push) decides what to prune for both sender and receiver. + The RPC protocol is used to execute the destroy operations on the remote side. + * New pruning policies (see :ref:`configuration documentation ` ) + + * The decision what snapshots shall be pruned is now made based on *keep rules* + * |feature| :issue:`68`: keep rule ``not_replicated`` prevents divergence of sender and receiver + + * |feature| |break| Bookmark pruning is no longer necessary + + * Per filesystem, zrepl creates a single bookmark (``#zrepl_replication_cursor``) and moves it forward with the most recent successfully replicated snapshot on the receiving side. + * Old bookmarks created prior to zrepl 0.1 (named like their corresponding snapshot) must be deleted manually. + * |break_config| ``keep_bookmarks`` parameter of the ``grid`` keep rule has been removed + +* |feature| ``zrepl status`` for live-updating replication progress (it's really cool!) * |feature| :issue:`67`: Expose `Prometheus `_ metrics via HTTP (:ref:`config docs `) + +* |break_config| Logging outlet types must be specified using the ``type`` instead of ``outlet`` key +* |break| :issue:`53`: CLI: ``zrepl control *`` subcommands have been made direct subcommands of ``zrepl *`` * |bugfix| Goroutine leak on ssh transport connection timeouts +* |bugfix| :issue:`81` :issue:`77` : handle failed accepts correctly (``source`` job) -0.0.3 ------ +.. |lastrelease| replace:: 0.0.3 -* |break_config| |feature| :issue:`34`: automatic bookmarking of snapshots +Previous Releases +----------------- - * Snapshots are automatically bookmarked and pruning of bookmarks **must** be configured. - * This breaks existing configuration: ``grid`` :ref:`prune policy ` specifications require the new ``keep_bookmarks`` parameter. - * Make sure to understand the meaning bookmarks have for :ref:`maximum replication downtime `. - * Example: :sampleconf:`pullbackup/productionhost.yml` - -* |break| :commit:`ccd062e`: ``ssh+stdinserver`` transport: changed protocol requires daemon restart on both sides - - * The delicate procedure of talking to the serving-side zrepl daemon via the stdinserver proxy command now has better error handling. - * This includes handshakes between client+proxy and client + remote daemo, which is not implemented in previous versions of zrepl. - * The connecting side will therefore time out, with the message ``dial_timeout of 10s exceeded``. - * Both sides of a replication setup must be updated and restarted. Otherwise the connecting side will hang and not time out. - -* |break_config| :commit:`2bfcfa5`: first outlet in ``global.logging`` is now used for logging meta-errors, for example problems encountered when writing to other outlets. -* |feature| :issue:`10`: ``zrepl control status`` subcommand - - * Allows inspection of job activity per task and their log output at runtime. - * Supports ``--format raw`` option for JSON output, usable for monitoring from scripts. - -* |feature| :commit:`d7f3fb9`: subcommand bash completions - - * Package maintainers should install this as appropriate. - -* |bugfix| :issue:`61`: fix excessive memory usage -* |bugfix| :issue:`8` and :issue:`56`: ``ssh+stdinserver`` transport properly reaps SSH child processes -* |bugfix| :commit:`cef63ac`: ``human`` format now prints non-string values correctly -* |bugfix| :issue:`26`: slow TCP outlets no longer block the daemon -* |docs| :issue:`64`: tutorial: document ``known_host`` file entry - -0.0.2 ------ - -* |break_config| :commit:`b95260f`: ``global.logging`` is no longer a dictionary but a list - -* |break_config| :commit:`3e647c1`: ``source`` job field ``datasets`` renamed to ``filesystems`` - - * **NOTE**: zrepl will parse missing ``filesystems`` field as an empty filter, - i.e. no filesystems are presented to the other side. - -* |bugfix| :commit:`72d2885` fix aliasing bug with root `<` subtree wildcard - - * Filesystems paths with final match at blank `s` subtree wildcard are now appended to the target path - * Non-root subtree wildcards, e.g. `zroot/foo/bar<` still map directrly onto the target path - -* Support days (``d``) and weeks (``w``) in durations - -* Docs - - * Ditch Hugo, move to Python Sphinx - * Improve & simplify tutorial (single SSH key per installation) - * Document pruning policies - * Document job types - * Document logging - * Start updating implementation overview +.. NOTE:: + Due to limitations in our documentation system, we only show the changelog since the last release and the time this documentation is built. + For the changelog of previous releases, use the version selection in the hosted version of these docs at `zrepl.github.io `_. + + -0.0.1 ------ - -* Initial release +W diff --git a/docs/configuration.rst b/docs/configuration.rst index 98b7174..ffba719 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -10,7 +10,7 @@ Configuration configuration/preface configuration/jobs configuration/transports - configuration/map_filter_syntax + configuration/filter_syntax configuration/prune configuration/logging configuration/monitoring diff --git a/docs/configuration/filter_syntax.rst b/docs/configuration/filter_syntax.rst new file mode 100644 index 0000000..6657998 --- /dev/null +++ b/docs/configuration/filter_syntax.rst @@ -0,0 +1,67 @@ +.. include:: ../global.rst.inc + +.. _pattern-filter: + +Filter Syntax +============= + +For :ref:`source jobs ` and :ref:`push jobs `, a filesystem filter must be defined (field ``filesystems``). +A filter takes a filesystem path (in the ZFS filesystem hierarchy) as parameter and returns ``true`` (pass) or ``false`` (block). + +A filter is specified as a **YAML dictionary** with patterns as keys and booleans as values. +The following rules determine which result is chosen for a given filesystem path: + +* More specific path patterns win over less specific ones +* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern) +* If the path in question does not match any pattern, the result is ``false``. + +The **subtree wildcard** ``<`` means "the dataset left of ``<`` and all its children". + +.. TIP:: + You can try out patterns for a configured job using the ``zrepl test filesystems`` subcommand for push and source jobs. + +Examples +-------- + +Full Access +~~~~~~~~~~~ + +The following configuration will allow access to all filesystems. + +:: + + jobs: + - type: source + filesystems: { + "<": true, + } + ... + + +Fine-grained +~~~~~~~~~~~~ + +The following configuration demonstrates all rules presented above. + +:: + + jobs: + - type: source + filesystems: { + "tank<": true, # rule 1 + "tank/foo<": false, # rule 2 + "tank/foo/bar": true, # rule 3 + } + ... + + +Which rule applies to given path, and what is the result? + +:: + + tank/foo/bar/loo => 2 false + tank/bar => 1 true + tank/foo/bar => 3 true + zroot => NONE false + tank/var/log => 1 true + diff --git a/docs/configuration/jobs.rst b/docs/configuration/jobs.rst index b921e79..cdbafba 100644 --- a/docs/configuration/jobs.rst +++ b/docs/configuration/jobs.rst @@ -1,20 +1,106 @@ .. include:: ../global.rst.inc -.. |patient| replace:: :ref:`patient ` -.. |serve-transport| replace:: :ref:`serve transport` -.. |connect-transport| replace:: :ref:`connect transport` -.. |mapping| replace:: :ref:`mapping ` -.. |filter| replace:: :ref:`filter ` -.. |prune| replace:: :ref:`prune ` +.. |serve-transport| replace:: :ref:`serve specification` +.. |connect-transport| replace:: :ref:`connect specification` +.. |snapshotting-spec| replace:: :ref:`snapshotting specification ` +.. |pruning-spec| replace:: :ref:`pruning specification ` +.. |filter-spec| replace:: :ref:`filter specification` .. _job: -Job Types -========= +Job Types & Replication +======================= + +Overview & Terminology +---------------------- A *job* is the unit of activity tracked by the zrepl daemon and configured in the |mainconfig|. Every job has a unique ``name``, a ``type`` and type-dependent fields which are documented on this page. -Check out the :ref:`tutorial` and :sampleconf:`/` for examples on how job types are actually used. + +Replication always happens between a pair of jobs: one is the **active side**, and one the **passive side**. +The active side executes the replication logic whereas the passive side responds to requests after checking the active side's permissions. +For communication, the active side connects to the passive side using a :ref:`transport ` and starts issuing remote procedure calls (RPCs). + +The following table shows how different job types can be combined to achieve both push and pull mode setups: + ++-----------------------+--------------+----------------------------------+-----------------------------------------------+ +| Setup name | active side | passive side | use case | ++=======================+==============+==================================+===============================================+ +| Push mode | ``push`` | ``sink`` | * Laptop backup | +| | | | * NAS behind NAT to offsite | ++-----------------------+--------------+----------------------------------+-----------------------------------------------+ +| Pull mode | ``pull`` | ``source`` | * Central backup-server for many nodes | +| | | | * Remote server to NAS behind NAT | ++-----------------------+--------------+----------------------------------+-----------------------------------------------+ +| Local replication | | ``push`` + ``sink`` in one config | * Backup FreeBSD boot pool | +| | | with :ref:`local transport ` | | ++-----------------------+--------------+----------------------------------+-----------------------------------------------+ + +How the Active Side Works +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The active side (:ref:`push ` and :ref:`pull ` job) executes the replication and pruning logic: + +* Wakeup because of finished snapshotting (``push`` job) or pull interval ticker (``pull`` job). +* Connect to the corresponding passive side using a :ref:`transport ` and instantiate an RPC client. +* Replicate data from the sending to the receiving side. +* Prune on sender & receiver. + +.. TIP:: + The progress of the active side can be watched live using the ``zrepl status`` subcommand. + +How the Passive Side Works +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The passive side (:ref:`sink ` and :ref:`source `) waits for connections from the corresponding active side, +using the transport listener type specified in the ``serve`` field of the job configuration. +Each transport listener provides a client's identity to the passive side job. +It uses the client identity for access control: + +* The ``sink`` job only allows pushes to those ZFS filesystems to the active side that are located below ``root_fs/${client_identity}``. +* The ``source`` job has a whitelist of client identities that are allowed pull access. + +.. TIP:: + The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components. + +How Replication Works +~~~~~~~~~~~~~~~~~~~~~ + +One of the major design goals of the replication module is to avoid any duplication of the nontrivial logic. +As such, the code works on abstract senders and receiver **endpoints**, where typically one will be implemented by a local program object and the other is an RPC client instance. +Regardless of push- or pull-style setup, the logic executes on the active side, i.e. in the ``push`` or ``pull`` job. + +The following steps take place during replication and can be monitored using the ``zrepl status`` subcommand: + +* Plan the replication: + + * Compare sender and receiver filesystem snapshots + * Build the **replication plan** + + * Per filesystem, compute a diff between sender and receiver snapshots + * Build a list of replication steps + + * If possible, use incremental sends (``zfs send -i``) + * Otherwise, use full send of most recent snapshot on sender + * Give up on filesystems that cannot be replicated without data loss + + * Retry on errors that are likely temporary (i.e. network failures). + * Give up on filesystems where a permanent error was received over RPC. + +* Execute the plan + + * Perform replication steps in the following order: + Among all filesystems with pending replication steps, pick the filesystem whose next replication step's snapshot is the oldest. + * After a successful replication step, update the replication cursor bookmark (see below) + +The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated. + +.. _replication-cursor-bookmark: + +The **replication cursor bookmark** ``#zrepl_replication_cursor`` is kept per filesystem on the sending side of a replication setup: +It is a bookmark of the most recent successfully replicated snapshot to the receiving side. +It is is used by the :ref:`not_replicated ` keep rule to identify all snapshots that have not yet been replicated to the receiving side. +Regardless of whether that keep rule is used, the bookmark ensures that replication can always continue incrementally. .. ATTENTION:: @@ -22,12 +108,133 @@ Check out the :ref:`tutorial` and :sampleconf:`/` for examples on how job types Whe receiving a filesystem, it is never mounted (`-u` flag) and `mountpoint=none` is set. This is temporary and being worked on :issue:`24`. + +.. _job-snapshotting-spec: + +Taking Snaphots +--------------- + +The ``push`` and ``source`` jobs can automatically take periodic snapshots of the filesystems matched by the ``filesystems`` filter field. +The snapshot names are composed of a user-defined prefix followed by a UTC date formatted like ``20060102_150405_000``. +We use UTC because it will avoid name conflicts when switching time zones or between summer and winter time. + +For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted. + +:: + + jobs: + - type: push + filesystems: { + "<": true, + "tmp": false + } + snapshotting: + type: periodic + prefix: zrepl_ + interval: 10m + ... + +There is also a ``manual`` snapshotting type, which covers the following use cases: + +* Existing infrastructure for automatic snapshots: you only want to use zrepl for replication. +* Run scripts before and after taking snapshots (like locking database tables). + We are working on better integration for this use case: see :issue:`74`. + +Note that you will have to trigger replication manually using the ``zrepl wakeup JOB`` subcommand in that case. + +:: + + jobs: + - type: push + filesystems: { + "<": true, + "tmp": false + } + snapshotting: + type: manual + ... + + +.. _job-push: + +Job Type ``push`` +----------------- + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Parameter + - Comment + * - ``type`` + - = ``push`` + * - ``name`` + - unique name of the job + * - ``connect`` + - |connect-transport| + * - ``filesystems`` + - |filter-spec| for filesystems to be snapshotted and pushed to the sink + * - ``snapshotting`` + - |snapshotting-spec| + * - ``pruning`` + - |pruning-spec| + +Example config: :sampleconf:`/push.yml` + +.. _job-sink: + +Job Type ``sink`` +----------------- + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Parameter + - Comment + * - ``type`` + - = ``sink`` + * - ``name`` + - unique name of the job + * - ``serve`` + - |serve-transport| + * - ``root_fs`` + - ZFS dataset path are received to + ``$root_fs/$client_identity`` + +Example config: :sampleconf:`/sink.yml` + +.. _job-pull: + +Job Type ``pull`` +----------------- + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Parameter + - Comment + * - ``type`` + - = ``pull`` + * - ``name`` + - unique name of the job + * - ``connect`` + - |connect-transport| + * - ``root_fs`` + - ZFS dataset path are received to + ``$root_fs/$client_identity`` + * - ``interval`` + - Interval at which to pull from the source job + * - ``pruning`` + - |pruning-spec| + +Example config: :sampleconf:`/pull.yml` + .. _job-source: -Source Job ----------- - -Example: :sampleconf:`pullbackup/productionhost.yml`. +Job Type ``source`` +------------------- .. list-table:: :widths: 20 80 @@ -42,146 +249,19 @@ Example: :sampleconf:`pullbackup/productionhost.yml`. * - ``serve`` - |serve-transport| * - ``filesystems`` - - |filter| for filesystems to expose to client - * - ``snapshot_prefix`` - - prefix for ZFS snapshots taken by this job - * - ``interval`` - - snapshotting interval - * - ``prune`` - - |prune| for versions of filesytems in ``filesystems``, versions prefixed with ``snapshot_prefix`` + - |filter-spec| for filesystems to be snapshotted and exposed to connecting clients + * - ``snapshotting`` + - |snapshotting-spec| + +Example config: :sampleconf:`/source.yml` + +.. _replication-local: + +Local replication +----------------- + +If you have the need for local replication (most likely between two local storage pools), you can use the :ref:`local transport type ` to connect a local push job to a local sink job. + +Example config: :sampleconf:`/local.yml`. -- Snapshotting Task (every ``interval``, |patient|) - - - A snapshot of filesystems matched by ``filesystems`` is taken every ``interval`` with prefix ``snapshot_prefix``. - - A bookmark of that snapshot is created with the same name. - - The ``prune`` policy is evaluated for versions of filesystems matched by ``filesystems``, versions prefixed with ``snapshot_prefix``. - -- Serve Task - - - Wait for connections from pull job using ``serve``. - -A source job is the counterpart to a :ref:`job-pull`. - -Make sure you read the |prune| policy documentation. - -.. _job-pull: - -Pull Job --------- - -Example: :sampleconf:`pullbackup/backuphost.yml` - -.. list-table:: - :widths: 20 80 - :header-rows: 1 - - * - Parameter - - Comment - * - ``type`` - - = ``pull`` - * - ``name`` - - unqiue name of the job - * - ``connect`` - - |connect-transport| - * - ``interval`` - - Interval between pull attempts - * - ``mapping`` - - |mapping| for remote to local filesystems - * - ``initial_repl_policy`` - - default = ``most_recent``, initial replication policy - * - ``snapshot_prefix`` - - prefix snapshots must match to be considered for replication & pruning - * - ``prune`` - - |prune| policy for versions of filesystems of local filesystems reachable by ``mapping``, versions prefixed with ``snapshot_prefix`` - -* Main Task (every ``interval``, |patient|) - - #. A connection to the remote source job is established using the strategy in ``connect`` - #. ``mapping`` maps filesystems presented by the remote side to local *target filesystems* - #. Those remote filesystems with a local *target filesystem* are replicated - - #. Only snapshots with prefix ``snapshot_prefix`` are replicated. - #. If possible, incremental replication takes place. - #. If the local target filesystem does not exist, ``initial_repl_policy`` is used. - #. On conflicts, an error is logged but replication of other filesystems with mapping continues. - - #. The ``prune`` policy is evaluated for all *target filesystems* - -A pull job is the counterpart to a :ref:`job-source`. - -Make sure you read the |prune| policy documentation. - -.. _job-local: - -Local Job ---------- - -Example: :sampleconf:`localbackup/host1.yml` - -.. list-table:: - :widths: 20 80 - :header-rows: 1 - - * - Parameter - - Comment - * - ``type`` - - = ``local`` - * - ``name`` - - unqiue name of the job - * - ``mapping`` - - |mapping| from source to target filesystem (both local) - * - ``snapshot_prefix`` - - prefix for ZFS snapshots taken by this job - * - ``interval`` - - snapshotting & replication interval - * - ``initial_repl_policy`` - - default = ``most_recent``, initial replication policy - * - ``prune_lhs`` - - pruning policy on left-hand-side (source) - * - ``prune_rhs`` - - pruning policy on right-hand-side (target) - -* Main Task (every ``interval``, |patient|) - - #. Evaluate ``mapping`` for local filesystems, those with a *target filesystem* are called *mapped filesystems*. - #. Snapshot *mapped filesystems* with ``snapshot_prefix``. - #. Bookmark the snapshot created above. - #. Replicate *mapped filesystems* to their respective *target filesystems*: - - #. Only snapshots with prefix ``snapshot_prefix`` are replicated. - #. If possible, incremental replication takes place. - #. If the *target filesystem* does not exist, ``initial_repl_policy`` is used. - #. On conflicts, an error is logged but replication of other *mapped filesystems* continues. - - #. The ``prune_lhs`` policy is triggered for all *mapped filesystems* - #. The ``prune_rhs`` policy is triggered for all *target filesystems* - -A local job is combination of source & pull job executed on the same machine. - -Terminology ------------ - -task - - A job consists of one or more tasks and a task consists of one or more steps. - Some tasks may be periodic while others wait for an event to occur. - - -patient task - - .. _job-term-patient: - - A patient task is supposed to execute some task every `interval`. - We call the start of the task an *invocation*. - - * If the task completes in less than `interval`, the task is restarted at `last_invocation + interval`. - * Otherwise, a patient job - * logs a warning as soon as a task exceeds its configured `interval` - * waits for the last invocation to finish - * logs a warning with the effective task duration - * immediately starts a new invocation of the task - -filesystem version - - A snapshot or a bookmark. diff --git a/docs/configuration/logging.rst b/docs/configuration/logging.rst index 4be4991..f1a8466 100644 --- a/docs/configuration/logging.rst +++ b/docs/configuration/logging.rst @@ -8,18 +8,17 @@ Logging zrepl uses structured logging to provide users with easily processable log messages. Logging outlets are configured in the ``global`` section of the |mainconfig|. -Check out :sampleconf:`random/logging_and_monitoring.yml` for an example on how to configure multiple outlets: :: global: logging: - - outlet: OUTLET_TYPE + - type: OUTLET_TYPE level: MINIMUM_LEVEL format: FORMAT - - outlet: OUTLET_TYPE + - type: OUTLET_TYPE level: MINIMUM_LEVEL format: FORMAT @@ -45,7 +44,7 @@ By default, the following logging configuration is used global: logging: - - outlet: "stdout" + - type: "stdout" level: "warn" format: "human" @@ -93,8 +92,8 @@ Formats * - Format - Description * - ``human`` - - emphasizes context by putting job, task, step and other context variables into brackets - before the actual message, followed by remaining fields in logfmt style| + - prints job and subsystem into brackets before the actual message, + followed by remaining fields in logfmt style * - ``logfmt`` - `logfmt `_ output. zrepl uses `this Go package `_. * - ``json`` @@ -118,7 +117,7 @@ Outlets are the destination for log entries. * - Parameter - Comment - * - ``outlet`` + * - ``type`` - ``stdout`` * - ``level`` - minimum :ref:`log level ` @@ -126,9 +125,11 @@ Outlets are the destination for log entries. - output :ref:`format ` * - ``time`` - always include time in output (``true`` or ``false``) + * - ``color`` + - colorize output according to log level (``true`` or ``false``) Writes all log entries with minimum level ``level`` formatted by ``format`` to stdout. -If stdout is a tty, interactive usage is assumed and the current time is included in the output. +If stdout is a tty, interactive usage is assumed and both ``time`` and ``color`` are set to ``true``. Can only be specified once. @@ -140,7 +141,7 @@ Can only be specified once. * - Parameter - Comment - * - ``outlet`` + * - ``type`` - ``syslog`` * - ``level`` - minimum :ref:`log level ` @@ -163,7 +164,7 @@ Can only be specified once. * - Parameter - Comment - * - ``outlet`` + * - ``type`` - ``tcp`` * - ``level`` - minimum :ref:`log level ` @@ -179,11 +180,9 @@ Can only be specified once. - TLS config (see below) Establishes a TCP connection to ``address`` and sends log messages with minimum level ``level`` formatted by ``format``. - If ``tls`` is not specified, an unencrypted connection is established. - If ``tls`` is specified, the TCP connection is secured with TLS + Client Authentication. -This is particularly useful in combination with log aggregation services that run on an other machine. +The latter is particularly useful in combination with log aggregation services. .. list-table:: :widths: 10 90 diff --git a/docs/configuration/map_filter_syntax.rst b/docs/configuration/map_filter_syntax.rst deleted file mode 100644 index 3f6f44a..0000000 --- a/docs/configuration/map_filter_syntax.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. include:: ../global.rst.inc - -Mapping & Filter Syntax -======================= - -For various job types, a filesystem ``mapping`` or ``filter`` needs to be -specified. - -Both have in common that they take a filesystem path (in the ZFS filesystem hierarchy)as parameters and return something. -Mappings return a *target filesystem* and filters return a *filter result*. - -The pattern syntax is the same for mappings and filters and is documented in the following section. - -Common Pattern Syntax ---------------------- - -A mapping / filter is specified as a **YAML dictionary** with patterns as keys and -results as values. -The following rules determine which result is chosen for a given filesystem path: - -* More specific path patterns win over less specific ones -* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern) - -The **subtree wildcard** ``<`` means "the dataset left of ``<`` and all its children". - -Example -~~~~~~~ - -:: - - # Rule number and its pattern - 1: tank< # tank and all its children - 2: tank/foo/bar # full path pattern (no wildcard) - 3: tank/foo< # tank/foo and all its children - - # Which rule applies to given path? - tank/foo/bar/loo => 3 - tank/bar => 1 - tank/foo/bar => 2 - zroot => NO MATCH - tank/var/log => 1 - -.. _pattern-mapping: - -Mappings --------- - -Mappings map a *source filesystem path* to a *target filesystem path*. -Per pattern, either a target filesystem path or ``"!"`` is specified as a result. - -* If no pattern matches, there exists no target filesystem (``NO MATCH``). -* If the result is a ``"!"``, there exists no target filesystem (``NO MATCH``). -* If the pattern is a non-wildcard pattern, the source path is mapped to the target path on the right. -* If the pattern ends with a *subtree wildcard* (``<``), the source path is **prefix-trimmed** with the path specified left of ``<``. - - * Note: this means that only for *wildcard-only* patterns (pattern= ``<`` ) is the source path simply appended to the target path. - -The example is from the :sampleconf:`localbackup/host1.yml` example config. - -:: - - jobs: - - name: mirror_local - type: local - mapping: { - "zroot/var/db<": "storage/backups/local/zroot/var/db", - "zroot/usr/home<": "storage/backups/local/zroot/usr/home", - "zroot/usr/home/paranoid": "!", #don't backup paranoid user - "zroot/poudriere/ports<": "!", #don't backup the ports trees - } - ... - - -:: - - zroot/var/db => storage/backups/local/zroot/var/db - zroot/var/db/a/child => storage/backups/local/zroot/var/db/a/child - zroot/usr/home => storage/backups/local/zroot/usr/home - zroot/usr/home/paranoid => NOT MAPPED - zroot/usr/home/bob => storage/backups/local/zroot/usr/home/bob - zroot/usr/src => NOT MAPPED - zroot/poudriere/ports/2017Q3 => NOT MAPPED - zroot/poudriere/ports/HEAD => NOT MAPPED - -.. TIP:: - - You can try out patterns for a configured job using the ``zrepl test`` subcommand. - -.. _pattern-filter: - -Filters -------- - -Valid filter results: ``ok`` or ``!``. - -The example below show the source job from the :ref:`tutorial `: -The corresponding pull job is allowed access to ``zroot/var/db``, ``zroot/usr/home`` + children except ``zroot/usr/home/paranoid``:: - - jobs: - - name: pull_backup - type: source - ... - filesystems: { - "zroot/var/db": "ok", - "zroot/usr/home<": "ok", - "zroot/usr/home/paranoid": "!", - } - ... diff --git a/docs/configuration/monitoring.rst b/docs/configuration/monitoring.rst index ac213e8..8974ee3 100644 --- a/docs/configuration/monitoring.rst +++ b/docs/configuration/monitoring.rst @@ -6,7 +6,6 @@ Monitoring ========== Monitoring endpoints are configured in the ``global.monitoring`` section of the |mainconfig|. -Check out :sampleconf:`random/logging_and_monitoring.yml` for examples. .. _monitoring-prometheus: @@ -17,7 +16,7 @@ zrepl can expose `Prometheus metrics `_ string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``. The Prometheues monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**. -There is no stability guarantee on the exported metrics. +At the time of writing, there is no stability guarantee on the exported metrics. :: diff --git a/docs/configuration/preface.rst b/docs/configuration/preface.rst index 989f720..b184d44 100644 --- a/docs/configuration/preface.rst +++ b/docs/configuration/preface.rst @@ -15,7 +15,7 @@ zrepl searches for its main configuration file in the following locations (in th * ``/etc/zrepl/zrepl.yml`` * ``/usr/local/etc/zrepl/zrepl.yml`` -The examples in the :ref:`tutorial` or the ``cmd/sampleconf`` directory should provide a good starting point. +The examples in the :ref:`tutorial` or the :sampleconf:`/` directory should provide a good starting point. ------------------- Runtime Directories @@ -35,5 +35,5 @@ Alternatively, for default settings, the following should to the trick. Validating ---------- -The config can be validated using the ``zrepl test`` subcommand. +The config can be validated using the ``zrepl configcheck`` subcommand. diff --git a/docs/configuration/prune.rst b/docs/configuration/prune.rst index 0d5c181..7aaacad 100644 --- a/docs/configuration/prune.rst +++ b/docs/configuration/prune.rst @@ -3,46 +3,93 @@ Pruning Policies ================ -In zrepl, *pruning* means *destroying filesystem versions by some policy* where filesystem versions are bookmarks and snapshots. +In zrepl, *pruning* means *destroying snapshots*. +Pruning must happen on both sides of a replication or the systems would inevitable run out of disk space at some point. -A *pruning policy* takes a list of filesystem versions and decides for each whether it should be kept or destroyed. +Typically, the requirements to temporal resolution and maximum retention time differ per side. +For example, when using zrepl to back up a busy database server, you will want high temporal resolution (snapshots every 10 min) for the last 24h in case of administrative disasters, but cannot afford to store them for much longer because you might have high turnover volume in the database. +On the receiving side, you may have more disk space available, or need to comply with other backup retention policies. -The job context defines which snapshots are even considered for pruning, for example through the ``snapshot_prefix`` variable. -Check the respective :ref:`job definition ` for details. +zrepl uses a set of **keep rules** to determine which snapshots shall be kept per filesystem. +**A snapshot that is not kept by any rule is destroyed.** +The keep rules are **evaluated on the active side** (:ref:`push ` or :ref:`pull job `) of the replication setup, for both active and passive side, after replication completed or was determined to have failed permanently. -Currently, the :ref:`prune-retention-grid` is the only supported pruning policy. +Example Configuration: -.. TIP:: +:: - You can perform a dry-run of a job's pruning policy using the ``zrepl test`` subcommand. + jobs: + - type: push + name: ... + connect: ... + filesystems: { + "<": true, + "tmp": false + } + snapshotting: + type: periodic + prefix: zrepl_ + interval: 10m + pruning: + keep_sender: + - type: not_replicated + # make sure manually created snapshots by the administrator are kept + - type: regex + regex: "^manual_.*" + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + regex: "^zrepl_.*" + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + regex: "^zrepl_.*" + # manually created snapshots will be kept forever on receiver -.. _prune-retention-grid: -Retention Grid --------------- +.. ATTENTION:: + + It is currently not possible to define pruning on a source job. + The source job creates snapshots, which means that extended replication downtime will fill up the source's zpool with snapshots, since pruning is directed by the corresponding active side (pull job). + If this is a potential risk for you, consider using :ref:`push mode `. + + +.. _prune-keep-not-replicated: + +Policy ``not_replicated`` +------------------------- + +:: + + jobs: + - type: push + pruning: + keep_sender: + - type: not_replicated + ... + +``not_replicated`` keeps all snapshots that have not been replicated to the receiving side. +It only makes sense to specify this rule on a sender (source or push job). +The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark ` on the sending side. + +.. _prune-keep-retention-grid: + +Policy ``grid`` +--------------- :: jobs: - - name: pull_app-srv - type: pull - ... - prune: - policy: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d - │ │ + - type: pull + pruning: + keep_receiver: + - type: grid + regex: "^zrepl_.*" + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + │ │ └─ one hour interval - │ - └─ 24 adjacent one-hour intervals - - - name: pull_backup - type: source - interval: 10m - prune: - policy: grid - grid: 1x1d(keep=all) - keep_bookmarks: 144 - + │ + └─ 24 adjacent one-hour intervals + ... The retention grid can be thought of as a time-based sieve: The ``grid`` field specifies a list of adjacent time intervals: @@ -51,16 +98,13 @@ All intervals to its right describe time intervals further in the past. Each interval carries a maximum number of snapshots to keep. It is secified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer. -The default value is **1**. - -Bookmarks are not affected by the above. -Instead, the ``keep_bookmarks`` field specifies the number of bookmarks to be kept per filesystem. -You only need to specify ``keep_bookmarks`` at the source-side of a replication setup since the destination side does not receive bookmarks. -You can specify ``all`` as a value to keep all bookmarks, but be warned that you should install some other way to prune unneeded ones then (see below). +The default value is **keep=1**. The following procedure happens during pruning: -#. The list of snapshots eligible for pruning is sorted by ``creation`` +#. The list of snapshots is filtered by the regular expression in ``regex``. + Only snapshots names that match the regex are considered for this rule, all others are not affected. +#. The filtered list of snapshots is sorted by ``creation`` #. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot #. A list of buckets is created, one for each interval #. The list of snapshots is split up into the buckets. @@ -69,16 +113,42 @@ The following procedure happens during pruning: #. the contained snapshot list is sorted by creation. #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached. #. all remaining snapshots on the list are kept. -#. The list of bookmarks eligible for pruning is sorted by ``createtxg`` and the most recent ``keep_bookmarks`` bookmarks are kept. -.. _replication-downtime: -.. ATTENTION:: +.. _prune-keep-last-n: + +Policy ``last_n`` +----------------- + +:: + + jobs: + - type: push + pruning: + keep_receiver: + - type: last_n + count: 10 + ... + +``last_n`` keeps the last ``count`` snapshots (last = youngest = most recent creation date). + +.. _prune-keep-regex: + +Policy ``regex`` +---------------- + +:: + + jobs: + - type: push + pruning: + keep_receiver: + - type: regex + regex: "^(zrepl|manual)_.*" + ... + +``regex`` keeps all snapshots whose names are matched by the regular expressionin ``regex``. +Like all other regular expression fields in prune policies, zrepl uses Go's `regexp.Regexp `_ Perl-compatible regular expressions (`Syntax `_). - Be aware that ``keep_bookmarks x interval`` (interval of the job level) controls the **maximum allowable replication downtime** between source and destination. - If replication does not work for whatever reason, source and destination will eventually run out of sync because the source will continue pruning snapshots. - The only recovery in that case is full replication, which may not always be viable due to disk space or traffic constraints. - Further note that while bookmarks consume a constant amount of disk space, listing them requires temporary dynamic **kernel memory** proportional to the number of bookmarks. - Thus, do not use ``all`` or an inappropriately high value without good reason. diff --git a/docs/configuration/transports.rst b/docs/configuration/transports.rst index e1c512a..05e4222 100644 --- a/docs/configuration/transports.rst +++ b/docs/configuration/transports.rst @@ -5,42 +5,156 @@ Transports ========== -A transport provides an authenticated `io.ReadWriteCloser `_ to the RPC layer. -(An ``io.ReadWriteCloser`` is essentially a bidirectional reliable communication channel.) +The zrepl RPC layer uses **transports** to establish a single, bidirectional data stream between an active and passive job. +On the passive (serving) side, the transport also provides the **client identity** to the upper layers: +this string is used for access control and separation of filesystem sub-trees in :ref:`sink jobs `. +Transports are specified in the ``connect`` or ``serve`` section of a job definition. -Currently, only the ``ssh+stdinserver`` transport is supported. +.. ATTENTION:: + + The **client identities must be valid ZFS dataset path components** + because the :ref:`sink job ` uses ``${root_fs}/${client_identity}`` to determine the client's subtree. + +.. _transport-tcp: + +``tcp`` Transport +----------------- + +The ``tcp`` transport uses plain TCP, which means that the data is **not encrypted** on the wire. +Clients are identified by their IPv4 or IPv6 addresses, and the client identity is established through a mapping on the server. + +This transport may also be used in conjunction with network-layer encryption and/or VPN tunnels to provide encryption on the wire. +To make the IP-based client authentication effective, such solutions should provide authenticated IP addresses. +Some options to consider: + +* `WireGuard `_: Linux-focussed, in-kernel TLS +* `OpenVPN `_: Cross-platform VPN, uses tun on \*nix +* `IPSec `_: Properly standardized, in-kernel network-layer VPN +* `spiped `_: think of it as an encrypted pipe between two servers +* SSH + + * `sshuttle `_: VPN-like solution, but using SSH + * `SSH port forwarding `_: Systemd user unit & make it start before the zrepl service. + +Serve +~~~~~ + +:: + + jobs: + - type: sink + serve: + type: tcp + listen: ":8888" + clients: { + "192.168.122.123" : "mysql01" + "192.168.122.123" : "mx01" + } + ... + +Connect +~~~~~~~ + +:: + + jobs: + - type: push + connect: + type: tcp + address: "10.23.42.23:8888" + dial_timeout: # optional, default 10s + ... + +.. _transport-tcp+tlsclientauth: + +``tls`` Transport +----------------- + +The ``tls`` transport uses TCP + TLS with client authentication using client certificates. +The client identity is the common name (CN) presented in the client certificate. +It is recommended to set up a dedicated CA infrastructure for this transport, e.g. using OpenVPN's `EasyRSA `_. + +The implementation uses `Go's TLS library `_. +Since Go binaries are statically linked, you or your distribution need to recompile zrepl when vulnerabilities in that library are disclosed. + +All file paths are resolved relative to the zrepl daemon's working directory. +Specify absolute paths if you are unsure what directory that is (or find out from your init system). + +Serve +~~~~~ + +:: + + jobs: + - type: sink + root_fs: "pool2/backup_laptops" + serve: + type: tls + listen: ":8888" + ca: /etc/zrepl/ca.crt + cert: /etc/zrepl/prod.crt + key: /etc/zrepl/prod.key + client_cns: + - "laptop1" + - "homeserver" + +The ``ca`` field specified the certificate authority used to validate client certificates. +The ``client_cns`` list specifies a list of accepted client common names (which are also the client identities for this transport). + +Connect +~~~~~~~ + +:: + + jobs: + - type: pull + connect: + type: tls + address: "server1.foo.bar:8888" + ca: /etc/zrepl/ca.crt + cert: /etc/zrepl/backupserver.crt + key: /etc/zrepl/backupserver.key + server_cn: "server1" + dial_timeout: # optional, default 10s + +The ``ca`` field specifies the CA which signed the server's certificate (``serve.cert``). +The ``server_cn`` specifies the expected common name (CN) of the server's certificate. +It overrides the hostname specified in ``address``. +The connection fails if either do not match. .. _transport-ssh+stdinserver: ``ssh+stdinserver`` Transport ----------------------------- -The way the ``ssh+stdinserver`` transport works is inspired by `git shell `_ and `Borg Backup `_. +``ssh+stdinserver`` is inspired by `git shell `_ and `Borg Backup `_. It is provided by the Go package ``github.com/problame/go-netssh``. -The config excerpts are taken from the :ref:`tutorial` which you should complete before reading further. .. _transport-ssh+stdinserver-serve: -Serve Mode -~~~~~~~~~~ +Serve +~~~~~ :: jobs: - - name: pull_backup - type: source + - type: source serve: type: stdinserver - client_identity: backup-srv.example.com + client_identities: + - "client1" + - "client2" ... -The serving job opens a UNIX socket named after ``client_identity`` in the runtime directory, e.g. ``/var/run/zrepl/stdinserver/backup-srv.example.com``. +First of all, note that ``type=stdinserver`` in this case: +Currently, only ``connect.type=ssh+stdinserver`` can connect to a ``serve.type=stdinserver``, but we want to keep that option open for future extensions. -On the same machine, the ``zrepl stdinserver $client_identity`` command connects to that socket. -For example, ``zrepl stdinserver backup-srv.example.com`` connects to the UNIX socket ``/var/run/zrepl/stdinserver/backup-srv.example.com``. +The serving job opens a UNIX socket named after ``client_identity`` in the runtime directory. +In our example above, that is ``/var/run/zrepl/stdinserver/client1`` and ``/var/run/zrepl/stdinserver/client2``. +On the same machine, the ``zrepl stdinserver $client_identity`` command connects to ``/var/run/zrepl/stdinserver/$client_identity``. It then passes its stdin and stdout file descriptors to the zrepl daemon via *cmsg(3)*. -zrepl daemon in turn combines them into an ``io.ReadWriteCloser``: +zrepl daemon in turn combines them into an object implementing ``net.Conn``: a ``Write()`` turns into a write to stdout, a ``Read()`` turns into a read from stdin. Interactive use of the ``stdinserver`` subcommand does not make much sense. @@ -54,8 +168,8 @@ This can be achieved with an entry in the ``authorized_keys`` file of the servin # for older OpenSSH versions command="zrepl stdinserver CLIENT_IDENTITY",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc CLIENT_SSH_KEY -* CLIENT_IDENTITY is substituted with ``backup-srv.example.com`` in our example -* CLIENT_SSH_KEY is substituted with the public part of the SSH keypair specified in the ``connect`` directive on the connecting host. +* CLIENT_IDENTITY is substituted with an entry from ``client_identities`` in our example +* CLIENT_SSH_KEY is substituted with the public part of the SSH keypair specified in the ``connect.identity_file`` directive on the connecting host. .. NOTE:: @@ -64,24 +178,24 @@ This can be achieved with an entry in the ``authorized_keys`` file of the servin To recap, this is of how client authentication works with the ``ssh+stdinserver`` transport: -* Connections to the ``client_identity`` UNIX socket are blindly trusted by zrepl daemon. -* Thus, the runtime directory must be private to the zrepl user (checked by zrepl daemon) +* Connections to the ``/var/run/zrepl/stdinserver/${client_identity}`` UNIX socket are blindly trusted by zrepl daemon. + The connection client identity is the name of the socket, i.e. ``${client_identity}``. +* Thus, the runtime directory must be private to the zrepl user (this is checked by zrepl daemon) * The admin of the host with the serving zrepl daemon controls the ``authorized_keys`` file. * Thus, the administrator controls the mapping ``PUBKEY -> CLIENT_IDENTITY``. .. _transport-ssh+stdinserver-connect: -Connect Mode -~~~~~~~~~~~~ +Connect +~~~~~~~ :: jobs: - - name: pull_app-srv - type: pull + - type: pull connect: type: ssh+stdinserver - host: app-srv.example.com + host: prod.example.com user: root port: 22 identity_file: /etc/zrepl/ssh/identity @@ -102,15 +216,46 @@ The connecting zrepl daemon #. The remote user, host and port correspond to those configured. #. Further options can be specified using the ``options`` field, which appends each entry in the list to the command line using ``-o $entry``. -#. Wraps the pipe ends in an ``io.ReadWriteCloser`` and uses it for RPC. - +#. Wraps the pipe ends in a ``net.Conn`` and returns it to the RPC layer. As discussed in the section above, the connecting zrepl daemon expects that ``zrepl stdinserver $client_identity`` is executed automatically via an ``authorized_keys`` file entry. -The ``known_hosts`` file used by the ssh command must contain an entry for the serving host, e.g., ``app-srv.example.com`` in the example above. +The ``known_hosts`` file used by the ssh command must contain an entry for ``connect.host`` prior to starting zrepl. +Thus, run the following on the pulling host's command line (substituting ``connect.host``): + +:: + + ssh -i /etc/zrepl/ssh/identity root@prod.example.com .. NOTE:: The environment variables of the underlying SSH process are cleared. ``$SSH_AUTH_SOCK`` will not be available. It is suggested to create a separate, unencrypted SSH key solely for that purpose. + +.. _transport-local: + +``local`` Transport +------------------- + +The local transport can be used to implement :ref:`local replication `, i.e., push replication between a push and sink job defined in the same configuration file. + +The ``listener_name`` is analogous to a hostname and must match between ``serve`` and ``connect``. +The ``client_identity`` is used by the sink as documented above. + +:: + + jobs: + - type: sink + serve: + type: local + listener_name: localsink + ... + + - type: push + connect: + type: local + listener_name: localsink + client_identity: local_backup + ... + diff --git a/docs/implementation.rst b/docs/implementation.rst index bc9b1f9..4e514d2 100644 --- a/docs/implementation.rst +++ b/docs/implementation.rst @@ -5,9 +5,12 @@ Implementation Overview .. WARNING:: - Incomplete / under construction + Incomplete and possibly outdated. + Check out the :ref:`talks about zrepl ` at various conferences for up-to-date material. + Alternatively, have a `look at the source code `_ ;) The following design aspects may convince you that ``zrepl`` is superior to a hacked-together shell script solution. +Also check out the :ref:`talks about zrepl ` at various conferences. Testability & Performance ------------------------- @@ -28,7 +31,7 @@ While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv` * The snapshot streams need to be compatible. * Communication is still unidirectional. Thus, you will most likely - * either not take advantage of features such as *compressed send & recv* + * either not take advantage of advanced replication features such as *compressed send & recv* * or issue additional ``ssh`` commands in advance to figure out what features are supported on the other side. * Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain. @@ -36,25 +39,21 @@ While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv` zrepl takes a different approach: * Define an RPC protocol. -* Establish an encrypted, authenticated, bidirectional communication channel... -* ... with zrepl running at both ends of it. +* Establish an encrypted, authenticated, bidirectional communication channel. +* Run daemons on both sides of the setup and let them talk to each other. This has several obvious benefits: * No blank root shell access is given to the other side. -* Instead, an *authenticated* peer can *request* filesystem lists, snapshot streams, etc. -* Requests are then checked against job-specific ACLs, limiting a client to the filesystems it is actually allowed to replicate. -* The :ref:`transport mechanism ` is decoupled from the remaining logic, keeping it extensible. +* An *authenticated* peer *requests* filesystem lists, snapshot streams, etc. +* The server decides which filesystems it exposes to which peers. +* The :ref:`transport mechanism ` is decoupled from the remaining logic, which allows us to painlessly offer multiple transport mechanisms. Protocol Implementation ~~~~~~~~~~~~~~~~~~~~~~~ -zrepl implements its own RPC protocol. -This is mostly due to the fact that existing solutions do not provide efficient means to transport large amounts of data. - -Package `github.com/zrepl/zrepl/rpc `_ builds a special-case handling around returning an ``io.Reader`` as part of a unary RPC call. - -Measurements show only a single memory-to-memory copy of a snapshot stream is made using ``github.com/zrepl/zrepl/rpc``, and there is still potential for further optimizations. +zrepl uses a custom RPC protol because, at the time of writing, existing solutions like gRPC do not provide efficient means to transport large amounts of data, whose size is unknown at send time (= zfs send streams). +The package used is `github.com/problame/go-streamrpc `_. Logging & Transparency ---------------------- diff --git a/docs/index.rst b/docs/index.rst index d7de850..6fe0e7e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,10 +8,15 @@ zrepl - ZFS replication ----------------------- -.. ATTENTION:: - zrepl as well as this documentation is still under active development. - It is neither feature complete nor is there a stability guarantee on the configuration format. - Use & test at your own risk ;) +**zrepl** is a one-stop, integrated solution for ZFS replication. + +.. raw:: html + +
+ + + +
Getting started ~~~~~~~~~~~~~~~ @@ -21,42 +26,47 @@ The :ref:`10 minutes tutorial setup ` gives you a first impression. Main Features ~~~~~~~~~~~~~ -* Filesystem Replication +* **Filesystem replication** - * [x] Local & Remote - * [x] Pull mode - * [ ] Push mode - * [x] Access control checks when pulling datasets - * [x] :ref:`Flexible mapping ` rules - * [x] Bookmarks support - * [ ] Feature-negotiation for + * [x] Pull & Push mode + * [x] Multiple transport :ref:`transports `: TCP, TCP + TLS client auth, SSH - * Resumable `send & receive` - * Compressed `send & receive` - * Raw encrypted `send & receive` (as soon as it is available) + * Advanced replication features -* Automatic snapshot creation + * [ ] Resumable send & receive + * [ ] Compressed send & receive + * [ ] Raw encrypted send & receive - * [x] Ensure fixed time interval between snapshots +* **Automatic snapshot management** -* Automatic snapshot :ref:`pruning ` + * [x] Periodic filesystem snapshots + * [x] Flexible :ref:`pruning rule system ` - * [x] Age-based fading (grandfathering scheme) + * [x] Age-based fading (grandfathering scheme) + * [x] Bookmarks to avoid divergence between sender and receiver -* Logging \& Monitoring +* **Sophisticated Monitoring & Logging** - * Detailed & structured :ref:`logging ` + * [x] Live progress reporting via `zrepl status` :ref:`subcommand ` + * [x] Comprehensive, structured :ref:`logging ` * ``human``, ``logfmt`` and ``json`` formatting * stdout, syslog and TCP (+TLS client auth) outlets - * Prometheus :ref:`monitoring ` endpoint + * [x] Prometheus :ref:`monitoring ` endpoint -* Maintainable implementation in Go +* **Maintainable implementation in Go** * [x] Cross platform * [x] Type safe & testable code + +.. ATTENTION:: + zrepl as well as this documentation is still under active development. + There is no stability guarantee on the RPC protocol or configuration format, + but we do our best to document breaking changes in the :ref:`changelog`. + + Contributing ~~~~~~~~~~~~ @@ -88,6 +98,7 @@ Table of Contents configuration usage implementation + pr changelog GitHub Repository & Issue Tracker - pr + diff --git a/docs/pr.rst b/docs/pr.rst index 8d40bec..fcb64d7 100644 --- a/docs/pr.rst +++ b/docs/pr.rst @@ -1,8 +1,17 @@ + +.. _pr-talks: + Talks & Presentations ===================== -* Talk at EuroBSDCon2017 FreeBSD DevSummit ( - `Slides `_, - `Event `_ +* Talk at OpenZFS Developer Summit 2018 of pre-release 0.1 ( + `25min Recording `__ , + `Slides `__ , + `Event `__ + ) +* Talk at EuroBSDCon2017 FreeBSD DevSummit with live demo of zrepl 0.0.3 ( + `55min Recording `__, + `Slides `__, + `Event `__ ) diff --git a/docs/sphinxconf/conf.py b/docs/sphinxconf/conf.py index 6d13b08..b270434 100644 --- a/docs/sphinxconf/conf.py +++ b/docs/sphinxconf/conf.py @@ -101,6 +101,17 @@ html_static_path = ['../_static'] html_logo = '../_static/zrepl.svg' +html_context = { + # https://github.com/rtfd/sphinx_rtd_theme/issues/205 + # Add 'Edit on Github' link instead of 'View page source' + "display_github": True, + "github_user": "zrepl", + "github_repo": "zrepl", + "github_version": "master", + "conf_py_path": "/docs/", + "source_suffix": source_suffix, +} + # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. @@ -162,7 +173,7 @@ texinfo_documents = [ # http://www.sphinx-doc.org/en/stable/ext/extlinks.html extlinks = { 'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'), - 'sampleconf':('https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/%s', 'cmd/sampleconf/'), + 'sampleconf':('https://github.com/zrepl/zrepl/blob/master/config/samples%s', 'config/samples'), 'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit '), } diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 40192d8..db83e22 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -9,165 +9,120 @@ Tutorial This tutorial shows how zrepl can be used to implement a ZFS-based pull backup. We assume the following scenario: -* Production server ``app-srv`` with filesystems to back up: +* Production server ``prod`` with filesystems to back up: * ``zroot/var/db`` * ``zroot/usr/home`` and all its child filesystems * **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves -* Backup server ``backup-srv`` with +* Backup server ``backups`` with - * Filesystem ``storage/zrepl/pull/app-srv`` + children dedicated to backups of ``app-srv`` + * Filesystem ``storage/zrepl/pull/prod`` + children dedicated to backups of ``prod`` Our backup solution should fulfill the following requirements: -* Periodically snapshot the filesystems on ``app-srv`` *every 10 minutes* -* Incrementally replicate these snapshots to ``storage/zrepl/pull/app-srv/*`` on ``backup-srv`` -* Keep only very few snapshots on ``app-srv`` to save disk space -* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backup-srv`` +* Periodically snapshot the filesystems on ``prod`` *every 10 minutes* +* Incrementally replicate these snapshots to ``storage/zrepl/pull/prod/*`` on ``backups`` +* Keep only very few snapshots on ``prod`` to save disk space +* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backups`` Analysis -------- We can model this situation as two jobs: -* A **source job** on ``app-srv`` +* A **source job** on ``prod`` * Creates the snapshots - * Keeps a short history of snapshots to enable incremental replication to ``backup-srv`` - * Accepts connections from ``backup-srv`` + * Keeps a short history of snapshots to enable incremental replication to ``backups`` + * Accepts connections from ``backups`` -* A **pull job** on ``backup-srv`` +* A **pull job** on ``backups`` - * Connects to the ``zrepl daemon`` process on ``app-srv`` - * Pulls the snapshots to ``storage/zrepl/pull/app-srv/*`` - * Fades out snapshots in ``storage/zrepl/pull/app-srv/*`` as they age + * Connects to the ``zrepl daemon`` process on ``prod`` + * Pulls the snapshots to ``storage/zrepl/pull/prod/*`` + * Fades out snapshots in ``storage/zrepl/pull/prod/*`` as they age Why doesn't the **pull job** create the snapshots before pulling? -As is the case with all distributed systems, the link between ``app-srv`` and ``backup-srv`` might be down for an hour or two. +As is the case with all distributed systems, the link between ``prod`` and ``backups`` might be down for an hour or two. We do not want to sacrifice our required backup resolution of 10 minute intervals for a temporary connection outage. -When the link comes up again, ``backup-srv`` will happily catch up the 12 snapshots taken by ``app-srv`` in the meantime, without -a gap in our backup history. +When the link comes up again, ``backups`` will catch up with the snapshots taken by ``prod`` in the meantime, without a gap in our backup history. Install zrepl ------------- Follow the :ref:`OS-specific installation instructions ` and come back here. -Configure ``backup-srv`` ------------------------- +Configure server ``backups`` +---------------------------- -We define a **pull job** named ``pull_app-srv`` in the |mainconfig| on host ``backup-srv``: :: +We define a **pull job** named ``pull_prod`` in ``/etc/zrepl/zrepl.yml`` or ``/usr/local/etc/zrepl/zrepl.yml`` on host ``backups`` : :: jobs: - - name: pull_app-srv + - name: pull_prod type: pull connect: - type: ssh+stdinserver - host: app-srv.example.com - user: root - port: 22 - identity_file: /etc/zrepl/ssh/identity + type: tcp + address: "192.168.2.20:2342" + root_fs: "storage/zrepl/pull/prod" interval: 10m - mapping: { - "<":"storage/zrepl/pull/app-srv" - } - initial_repl_policy: most_recent - snapshot_prefix: zrepl_pull_backup_ - prune: - policy: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d + regex: "^zrepl_" + interval: 10m -The ``connect`` section instructs the zrepl daemon to use the ``stdinserver`` transport: -``backup-srv`` will connect to the specified SSH server and expect ``zrepl stdinserver CLIENT_IDENTITY`` instead of the shell on the other side. +The ``connect`` section instructs the zrepl daemon to use plain TCP transport. +Check out the :ref:`transports ` section for alternatives that support encryption. -It uses the private key specified at ``connect.identity_file`` which we still need to create: :: +.. _tutorial-configure-prod: - cd /etc/zrepl - mkdir -p ssh - chmod 0700 ssh - ssh-keygen -t ed25519 -N '' -f /etc/zrepl/ssh/identity +Configure server ``prod`` +------------------------- -Note that most use cases do not benefit from separate keypairs per remote endpoint. -Thus, it is sufficient to create one keypair and use it for all ``connect`` directives on one host. - -zrepl uses ssh's default ``known_hosts`` file, which must contain a host identification entry for ``app-srv.example.com``. -If that entry does not already exist, we need to generate it. -Run the following command, compare the host fingerprints, and confirm with yes if they match. -You will not be able to get a shell with the identity file we just generated, which is fine. :: - - ssh -i /etc/zrepl/ssh/identity root@app-srv.example.com - -Learn more about :ref:`transport-ssh+stdinserver` transport and the :ref:`pull job ` format. - -.. _tutorial-configure-app-srv: - -Configure ``app-srv`` ---------------------- - -We define a corresponding **source job** named ``pull_backup`` in the |mainconfig| on host ``app-srv``: :: +We define a corresponding **source job** named ``source_backups`` in ``/etc/zrepl/zrepl.yml`` or ``/usr/local/etc/zrepl/zrepl.yml`` on host ``prod`` : :: jobs: - - name: pull_backup + - name: source_backups type: source serve: - type: stdinserver - client_identity: backup-srv.example.com + type: tcp + listen: ":2342" + clients: { + "192.168.2.10" : "backups" + } filesystems: { - "zroot/var/db": "ok", - "zroot/usr/home<": "ok", - "zroot/usr/home/paranoid": "!", + "zroot/var/db:": true, + "zroot/usr/home<": true, + "zroot/usr/home/paranoid": false } - snapshot_prefix: zrepl_pull_backup_ - interval: 10m - prune: - policy: grid - grid: 1x1d(keep=all) - keep_bookmarks: 144 + snapshotting: + type: periodic + prefix: zrepl_ + interval: 10m -The ``serve`` section corresponds to the ``connect`` section in the configuration of ``backup-srv``. - -We now want to authenticate ``backup-srv`` before allowing it to pull data. -This is done by limiting SSH connections from ``backup-srv`` to execute the ``stdinserver`` subcommand. - -Open ``/root/.ssh/authorized_keys`` and add either of the the following lines.:: - - # for OpenSSH >= 7.2 - command="zrepl stdinserver backup-srv.example.com",restrict CLIENT_SSH_KEY - # for older OpenSSH versions - command="zrepl stdinserver backup-srv.example.com",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc CLIENT_SSH_KEY - -.. ATTENTION:: - - Replace CLIENT_SSH_KEY with the contents of ``/etc/zrepl/ssh/identity.pub`` from ``app-srv``. - Mind the trailing ``.pub`` in the filename. - The entries **must** be on a single line, including the replaced CLIENT_SSH_KEY. - - -.. HINT:: - - You may need to adjust the ``PermitRootLogin`` option in ``/etc/ssh/sshd_config`` to ``forced-commands-only`` or higher for this to work. - Refer to sshd_config(5) for details. - -The argument ``backup-srv.example.com`` is the client identity of ``backup-srv`` as defined in ``jobs.serve.client_identity``. - -Again, both :ref:`transport-ssh+stdinserver` transport and the :ref:`job-source` format are documented. +The ``serve`` section whitelists ``backups``'s IP address ``192.168.2.10`` and assigns it the client identity ``backups`` which will show up in the logs. +Again, check the :ref:`docs for encrypted transports `. Apply Configuration Changes --------------------------- -We need to restart the zrepl daemon on **both** ``app-srv`` and ``backup-srv``. +We need to restart the zrepl daemon on **both** ``prod`` and ``backups``. This is :ref:`OS-specific `. Watch it Work ------------- -Run ``zrepl control status`` to view the current activity of the configured jobs. -If a job encountered problems since it last left idle state, the output contains useful debug log. +Run ``zrepl status`` on ``prod`` to monitor the replication and pruning activity. Additionally, you can check the detailed structured logs of the `zrepl daemon` process and use GNU *watch* to view the snapshots present on both machines. @@ -176,7 +131,7 @@ If you like tmux, here is a handy script that works on FreeBSD: :: pkg install gnu-watch tmux tmux new-window tmux split-window "tail -f /var/log/zrepl.log" - tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'" + tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_'" tmux select-layout tiled The Linux equivalent might look like this: :: @@ -184,7 +139,7 @@ The Linux equivalent might look like this: :: # make sure tmux is installed & let's assume you use systemd + journald tmux new-window tmux split-window "journalctl -f -u zrepl.service" - tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'" + tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_'" tmux select-layout tiled Summary diff --git a/docs/usage.rst b/docs/usage.rst index 3beb7ba..3bddfab 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -10,8 +10,8 @@ CLI Overview .. NOTE:: - To avoid duplication, the zrepl binary is self-documenting: - invoke any subcommand at any level with the ``--help`` flag to get information on the subcommand, available flags, etc. + The zrepl binary is self-documenting: + run ``zrepl help`` for an overview of the available subcommands or ``zrepl SUBCOMMAND --help`` for information on available flags, etc. .. list-table:: :widths: 30 70 @@ -19,16 +19,18 @@ CLI Overview * - Subcommand - Description + * - ``zrepl help`` + - show subcommand overview * - ``zrepl daemon`` - run the daemon, required for all zrepl functionality - * - ``zrepl control`` - - control / query the daemon - * - ``zrepl control status`` - - show job activity / monitoring (``--format raw``) - * - ``zrepl test`` - - test configuration, try pattern syntax, dry run pruning policy, etc. + * - ``zrepl status`` + - show job activity, or with ``--raw`` for JSON output * - ``zrepl stdinserver`` - see :ref:`transport-ssh+stdinserver` + * - ``zrepl wakeup JOB`` + - manually trigger replication + pruning + * - ``zrepl configcheck`` + - check if config can be parsed without errors .. _usage-zrepl-daemon: diff --git a/endpoint/context.go b/endpoint/context.go new file mode 100644 index 0000000..09f9032 --- /dev/null +++ b/endpoint/context.go @@ -0,0 +1,25 @@ +package endpoint + +import ( + "context" + "github.com/zrepl/zrepl/logger" +) + +type contextKey int + +const ( + contextKeyLogger contextKey = iota +) + +type Logger = logger.Logger + +func WithLogger(ctx context.Context, log Logger) context.Context { + return context.WithValue(ctx, contextKeyLogger, log) +} + +func getLogger(ctx context.Context) Logger { + if l, ok := ctx.Value(contextKeyLogger).(Logger); ok { + return l + } + return logger.NewNullLogger() +} diff --git a/endpoint/endpoint.go b/endpoint/endpoint.go new file mode 100644 index 0000000..fb1347a --- /dev/null +++ b/endpoint/endpoint.go @@ -0,0 +1,613 @@ +// Package endpoint implements replication endpoints for use with package replication. +package endpoint + +import ( + "bytes" + "context" + "fmt" + "github.com/golang/protobuf/proto" + "github.com/pkg/errors" + "github.com/problame/go-streamrpc" + "github.com/zrepl/zrepl/replication" + "github.com/zrepl/zrepl/replication/pdu" + "github.com/zrepl/zrepl/zfs" + "io" +) + +// Sender implements replication.ReplicationEndpoint for a sending side +type Sender struct { + FSFilter zfs.DatasetFilter +} + +func NewSender(fsf zfs.DatasetFilter) *Sender { + return &Sender{FSFilter: fsf} +} + +func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) { + dp, err := zfs.NewDatasetPath(fs) + if err != nil { + return nil, err + } + if dp.Length() == 0 { + return nil, errors.New("empty filesystem not allowed") + } + pass, err := s.FSFilter.Filter(dp) + if err != nil { + return nil, err + } + if !pass { + return nil, replication.NewFilteredError(fs) + } + return dp, nil +} + +func (p *Sender) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) { + fss, err := zfs.ZFSListMapping(p.FSFilter) + if err != nil { + return nil, err + } + rfss := make([]*pdu.Filesystem, len(fss)) + for i := range fss { + rfss[i] = &pdu.Filesystem{ + Path: fss[i].ToString(), + // FIXME: not supporting ResumeToken yet + } + } + return rfss, nil +} + +func (p *Sender) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) { + lp, err := p.filterCheckFS(fs) + if err != nil { + return nil, err + } + fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil) + if err != nil { + return nil, err + } + rfsvs := make([]*pdu.FilesystemVersion, len(fsvs)) + for i := range fsvs { + rfsvs[i] = pdu.FilesystemVersionFromZFS(&fsvs[i]) + } + return rfsvs, nil +} + +func (p *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) { + _, err := p.filterCheckFS(r.Filesystem) + if err != nil { + return nil, nil, err + } + + if r.DryRun { + size, err := zfs.ZFSSendDry(r.Filesystem, r.From, r.To) + if err == zfs.BookmarkSizeEstimationNotSupported { + return &pdu.SendRes{ExpectedSize: 0}, nil, nil + } + if err != nil { + return nil, nil, err + } + return &pdu.SendRes{ExpectedSize: size}, nil, nil + } else { + stream, err := zfs.ZFSSend(r.Filesystem, r.From, r.To) + if err != nil { + return nil, nil, err + } + return &pdu.SendRes{}, stream, nil + } +} + +func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) { + dp, err := p.filterCheckFS(req.Filesystem) + if err != nil { + return nil, err + } + return doDestroySnapshots(ctx, dp, req.Snapshots) +} + +func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) { + dp, err := p.filterCheckFS(req.Filesystem) + if err != nil { + return nil, err + } + + switch op := req.Op.(type) { + case *pdu.ReplicationCursorReq_Get: + cursor, err := zfs.ZFSGetReplicationCursor(dp) + if err != nil { + return nil, err + } + if cursor == nil { + return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Error{Error: "cursor does not exist"}}, nil + } + return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil + case *pdu.ReplicationCursorReq_Set: + guid, err := zfs.ZFSSetReplicationCursor(dp, op.Set.Snapshot) + if err != nil { + return nil, err + } + return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: guid}}, nil + default: + return nil, errors.Errorf("unknown op %T", op) + } +} + +type FSFilter interface { // FIXME unused + Filter(path *zfs.DatasetPath) (pass bool, err error) +} + +// FIXME: can we get away without error types here? +type FSMap interface { // FIXME unused + FSFilter + Map(path *zfs.DatasetPath) (*zfs.DatasetPath, error) + Invert() (FSMap, error) + AsFilter() FSFilter +} + +// Receiver implements replication.ReplicationEndpoint for a receiving side +type Receiver struct { + root *zfs.DatasetPath +} + +func NewReceiver(rootDataset *zfs.DatasetPath) (*Receiver, error) { + if rootDataset.Length() <= 0 { + return nil, errors.New("root dataset must not be an empty path") + } + return &Receiver{root: rootDataset.Copy()}, nil +} + +type subroot struct { + localRoot *zfs.DatasetPath +} + +var _ zfs.DatasetFilter = subroot{} + +// Filters local p +func (f subroot) Filter(p *zfs.DatasetPath) (pass bool, err error) { + return p.HasPrefix(f.localRoot) && !p.Equal(f.localRoot), nil +} + +func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) { + p, err := zfs.NewDatasetPath(fs) + if err != nil { + return nil, err + } + if p.Length() == 0 { + return nil, errors.Errorf("cannot map empty filesystem") + } + c := f.localRoot.Copy() + c.Extend(p) + return c, nil +} + +func (e *Receiver) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) { + filtered, err := zfs.ZFSListMapping(subroot{e.root}) + if err != nil { + return nil, err + } + // present without prefix, and only those that are not placeholders + fss := make([]*pdu.Filesystem, 0, len(filtered)) + for _, a := range filtered { + ph, err := zfs.ZFSIsPlaceholderFilesystem(a) + if err != nil { + getLogger(ctx). + WithError(err). + WithField("fs", a). + Error("inconsistent placeholder property") + return nil, errors.New("server error, see logs") // don't leak path + } + if ph { + continue + } + a.TrimPrefix(e.root) + fss = append(fss, &pdu.Filesystem{Path: a.ToString()}) + } + return fss, nil +} + +func (e *Receiver) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) { + lp, err := subroot{e.root}.MapToLocal(fs) + if err != nil { + return nil, err + } + + fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil) + if err != nil { + return nil, err + } + + rfsvs := make([]*pdu.FilesystemVersion, len(fsvs)) + for i := range fsvs { + rfsvs[i] = pdu.FilesystemVersionFromZFS(&fsvs[i]) + } + + return rfsvs, nil +} + +func (e *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, sendStream io.ReadCloser) error { + defer sendStream.Close() + + lp, err := subroot{e.root}.MapToLocal(req.Filesystem) + if err != nil { + return err + } + + getLogger(ctx).Debug("incoming Receive") + + // create placeholder parent filesystems as appropriate + var visitErr error + f := zfs.NewDatasetPathForest() + f.Add(lp) + getLogger(ctx).Debug("begin tree-walk") + f.WalkTopDown(func(v zfs.DatasetPathVisit) (visitChildTree bool) { + if v.Path.Equal(lp) { + return false + } + _, err := zfs.ZFSGet(v.Path, []string{zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME}) + if err != nil { + // interpret this as an early exit of the zfs binary due to the fs not existing + if err := zfs.ZFSCreatePlaceholderFilesystem(v.Path); err != nil { + getLogger(ctx). + WithError(err). + WithField("placeholder_fs", v.Path). + Error("cannot create placeholder filesystem") + visitErr = err + return false + } + } + getLogger(ctx).WithField("filesystem", v.Path.ToString()).Debug("exists") + return true // leave this fs as is + }) + getLogger(ctx).WithField("visitErr", visitErr).Debug("complete tree-walk") + + if visitErr != nil { + return visitErr + } + + needForceRecv := false + props, err := zfs.ZFSGet(lp, []string{zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME}) + if err == nil { + if isPlaceholder, _ := zfs.IsPlaceholder(lp, props.Get(zfs.ZREPL_PLACEHOLDER_PROPERTY_NAME)); isPlaceholder { + needForceRecv = true + } + } + + args := make([]string, 0, 1) + if needForceRecv { + args = append(args, "-F") + } + + getLogger(ctx).Debug("start receive command") + + if err := zfs.ZFSRecv(lp.ToString(), sendStream, args...); err != nil { + getLogger(ctx). + WithError(err). + WithField("args", args). + Error("zfs receive failed") + sendStream.Close() + return err + } + return nil +} + +func (e *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) { + lp, err := subroot{e.root}.MapToLocal(req.Filesystem) + if err != nil { + return nil, err + } + return doDestroySnapshots(ctx, lp, req.Snapshots) +} + +func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.FilesystemVersion) (*pdu.DestroySnapshotsRes, error) { + fsvs := make([]*zfs.FilesystemVersion, len(snaps)) + for i, fsv := range snaps { + if fsv.Type != pdu.FilesystemVersion_Snapshot { + return nil, fmt.Errorf("version %q is not a snapshot", fsv.Name) + } + var err error + fsvs[i], err = fsv.ZFSFilesystemVersion() + if err != nil { + return nil, err + } + } + res := &pdu.DestroySnapshotsRes{ + Results: make([]*pdu.DestroySnapshotRes, len(fsvs)), + } + for i, fsv := range fsvs { + err := zfs.ZFSDestroyFilesystemVersion(lp, fsv) + errMsg := "" + if err != nil { + errMsg = err.Error() + } + res.Results[i] = &pdu.DestroySnapshotRes{ + Snapshot: pdu.FilesystemVersionFromZFS(fsv), + Error: errMsg, + } + } + return res, nil +} + +// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +// RPC STUBS +// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + +const ( + RPCListFilesystems = "ListFilesystems" + RPCListFilesystemVersions = "ListFilesystemVersions" + RPCReceive = "Receive" + RPCSend = "Send" + RPCSDestroySnapshots = "DestroySnapshots" + RPCReplicationCursor = "ReplicationCursor" +) + +// Remote implements an endpoint stub that uses streamrpc as a transport. +type Remote struct { + c *streamrpc.Client +} + +func NewRemote(c *streamrpc.Client) Remote { + return Remote{c} +} + +func (s Remote) ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) { + req := pdu.ListFilesystemReq{} + b, err := proto.Marshal(&req) + if err != nil { + return nil, err + } + rb, rs, err := s.c.RequestReply(ctx, RPCListFilesystems, bytes.NewBuffer(b), nil) + if err != nil { + return nil, err + } + if rs != nil { + rs.Close() + return nil, errors.New("response contains unexpected stream") + } + var res pdu.ListFilesystemRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + return nil, err + } + return res.Filesystems, nil +} + +func (s Remote) ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) { + req := pdu.ListFilesystemVersionsReq{ + Filesystem: fs, + } + b, err := proto.Marshal(&req) + if err != nil { + return nil, err + } + rb, rs, err := s.c.RequestReply(ctx, RPCListFilesystemVersions, bytes.NewBuffer(b), nil) + if err != nil { + return nil, err + } + if rs != nil { + rs.Close() + return nil, errors.New("response contains unexpected stream") + } + var res pdu.ListFilesystemVersionsRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + return nil, err + } + return res.Versions, nil +} + +func (s Remote) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) { + b, err := proto.Marshal(r) + if err != nil { + return nil, nil, err + } + rb, rs, err := s.c.RequestReply(ctx, RPCSend, bytes.NewBuffer(b), nil) + if err != nil { + return nil, nil, err + } + if !r.DryRun && rs == nil { + return nil, nil, errors.New("response does not contain a stream") + } + if r.DryRun && rs != nil { + rs.Close() + return nil, nil, errors.New("response contains unexpected stream (was dry run)") + } + var res pdu.SendRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + rs.Close() + return nil, nil, err + } + return &res, rs, nil +} + +func (s Remote) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error { + defer sendStream.Close() + b, err := proto.Marshal(r) + if err != nil { + return err + } + rb, rs, err := s.c.RequestReply(ctx, RPCReceive, bytes.NewBuffer(b), sendStream) + getLogger(ctx).WithField("err", err).Debug("Remote.Receive RequestReplyReturned") + if err != nil { + return err + } + if rs != nil { + rs.Close() + return errors.New("response contains unexpected stream") + } + var res pdu.ReceiveRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + return err + } + return nil +} + +func (s Remote) DestroySnapshots(ctx context.Context, r *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) { + b, err := proto.Marshal(r) + if err != nil { + return nil, err + } + rb, rs, err := s.c.RequestReply(ctx, RPCSDestroySnapshots, bytes.NewBuffer(b), nil) + if err != nil { + return nil, err + } + if rs != nil { + rs.Close() + return nil, errors.New("response contains unexpected stream") + } + var res pdu.DestroySnapshotsRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + return nil, err + } + return &res, nil +} + +func (s Remote) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) { + b, err := proto.Marshal(req) + if err != nil { + return nil, err + } + rb, rs, err := s.c.RequestReply(ctx, RPCReplicationCursor, bytes.NewBuffer(b), nil) + if err != nil { + return nil, err + } + if rs != nil { + rs.Close() + return nil, errors.New("response contains unexpected stream") + } + var res pdu.ReplicationCursorRes + if err := proto.Unmarshal(rb.Bytes(), &res); err != nil { + return nil, err + } + return &res, nil +} + +// Handler implements the server-side streamrpc.HandlerFunc for a Remote endpoint stub. +type Handler struct { + ep replication.Endpoint +} + +func NewHandler(ep replication.Endpoint) Handler { + return Handler{ep} +} + +func (a *Handler) Handle(ctx context.Context, endpoint string, reqStructured *bytes.Buffer, reqStream io.ReadCloser) (resStructured *bytes.Buffer, resStream io.ReadCloser, err error) { + + switch endpoint { + case RPCListFilesystems: + var req pdu.ListFilesystemReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + fsses, err := a.ep.ListFilesystems(ctx) + if err != nil { + return nil, nil, err + } + res := &pdu.ListFilesystemRes{ + Filesystems: fsses, + } + b, err := proto.Marshal(res) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), nil, nil + + case RPCListFilesystemVersions: + + var req pdu.ListFilesystemVersionsReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + fsvs, err := a.ep.ListFilesystemVersions(ctx, req.Filesystem) + if err != nil { + return nil, nil, err + } + res := &pdu.ListFilesystemVersionsRes{ + Versions: fsvs, + } + b, err := proto.Marshal(res) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), nil, nil + + case RPCSend: + + sender, ok := a.ep.(replication.Sender) + if !ok { + goto Err + } + + var req pdu.SendReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + res, sendStream, err := sender.Send(ctx, &req) + if err != nil { + return nil, nil, err + } + b, err := proto.Marshal(res) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), sendStream, err + + case RPCReceive: + + receiver, ok := a.ep.(replication.Receiver) + if !ok { + goto Err + } + + var req pdu.ReceiveReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + err := receiver.Receive(ctx, &req, reqStream) + if err != nil { + return nil, nil, err + } + b, err := proto.Marshal(&pdu.ReceiveRes{}) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), nil, err + + case RPCSDestroySnapshots: + + var req pdu.DestroySnapshotsReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + + res, err := a.ep.DestroySnapshots(ctx, &req) + if err != nil { + return nil, nil, err + } + b, err := proto.Marshal(res) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), nil, nil + + case RPCReplicationCursor: + + sender, ok := a.ep.(replication.Sender) + if !ok { + goto Err + } + + var req pdu.ReplicationCursorReq + if err := proto.Unmarshal(reqStructured.Bytes(), &req); err != nil { + return nil, nil, err + } + res, err := sender.ReplicationCursor(ctx, &req) + if err != nil { + return nil, nil, err + } + b, err := proto.Marshal(res) + if err != nil { + return nil, nil, err + } + return bytes.NewBuffer(b), nil, nil + + } +Err: + return nil, nil, errors.New("no handler for given endpoint") +} diff --git a/lazy.sh b/lazy.sh index 8d785cb..434f66b 100755 --- a/lazy.sh +++ b/lazy.sh @@ -29,7 +29,9 @@ builddep() { step "Install build depdencies using 'go get' to \$GOPATH/bin" go get -u golang.org/x/tools/cmd/stringer go get -u github.com/golang/dep/cmd/dep - if ! type stringer || ! type dep; then + go get -u github.com/golang/protobuf/protoc-gen-go + go get -u github.com/alvaroloes/enumer + if ! type stringer || ! type dep || ! type protoc-gen-go; || ! type enumer then echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2 exit 1 fi diff --git a/logger/datastructures.go b/logger/datastructures.go index d6460ee..782c8e9 100644 --- a/logger/datastructures.go +++ b/logger/datastructures.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/fatih/color" "github.com/pkg/errors" "sync" "time" @@ -105,6 +106,21 @@ type Entry struct { Fields Fields } +func (e Entry) Color() *color.Color { + c := color.New() + switch e.Level { + case Debug: + c.Add(color.FgHiBlue) + case Info: + c.Add(color.FgHiGreen) + case Warn: + c.Add(color.FgHiYellow) + case Error: + c.Add(color.FgHiRed) + } + return c +} + // An outlet receives log entries produced by the Logger and writes them to some destination. type Outlet interface { // Write the entry to the destination. diff --git a/logger/logger.go b/logger/logger.go index fdfe103..b007267 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -14,7 +14,20 @@ const ( const DefaultUserFieldCapacity = 5 -type Logger struct { +type Logger interface { + WithOutlet(outlet Outlet, level Level) Logger + ReplaceField(field string, val interface{}) Logger + WithField(field string, val interface{}) Logger + WithFields(fields Fields) Logger + WithError(err error) Logger + Debug(msg string) + Info(msg string) + Warn(msg string) + Error(msg string) + Printf(format string, args ...interface{}) +} + +type loggerImpl struct { fields Fields outlets *Outlets outletTimeout time.Duration @@ -22,8 +35,10 @@ type Logger struct { mtx *sync.Mutex } -func NewLogger(outlets *Outlets, outletTimeout time.Duration) *Logger { - return &Logger{ +var _ Logger = &loggerImpl{} + +func NewLogger(outlets *Outlets, outletTimeout time.Duration) Logger { + return &loggerImpl{ make(Fields, DefaultUserFieldCapacity), outlets, outletTimeout, @@ -36,7 +51,7 @@ type outletResult struct { Error error } -func (l *Logger) logInternalError(outlet Outlet, err string) { +func (l *loggerImpl) logInternalError(outlet Outlet, err string) { fields := Fields{} if outlet != nil { if _, ok := outlet.(fmt.Stringer); ok { @@ -54,7 +69,7 @@ func (l *Logger) logInternalError(outlet Outlet, err string) { l.outlets.GetLoggerErrorOutlet().WriteEntry(entry) } -func (l *Logger) log(level Level, msg string) { +func (l *loggerImpl) log(level Level, msg string) { l.mtx.Lock() defer l.mtx.Unlock() @@ -78,12 +93,12 @@ func (l *Logger) log(level Level, msg string) { } -func (l *Logger) WithOutlet(outlet Outlet, level Level) *Logger { +func (l *loggerImpl) WithOutlet(outlet Outlet, level Level) Logger { l.mtx.Lock() defer l.mtx.Unlock() newOutlets := l.outlets.DeepCopy() newOutlets.Add(outlet, level) - child := &Logger{ + child := &loggerImpl{ fields: l.fields, outlets: newOutlets, outletTimeout: l.outletTimeout, @@ -93,9 +108,9 @@ func (l *Logger) WithOutlet(outlet Outlet, level Level) *Logger { } // callers must hold l.mtx -func (l *Logger) forkLogger(field string, val interface{}) *Logger { +func (l *loggerImpl) forkLogger(field string, val interface{}) *loggerImpl { - child := &Logger{ + child := &loggerImpl{ fields: make(Fields, len(l.fields)+1), outlets: l.outlets, outletTimeout: l.outletTimeout, @@ -109,13 +124,13 @@ func (l *Logger) forkLogger(field string, val interface{}) *Logger { return child } -func (l *Logger) ReplaceField(field string, val interface{}) *Logger { +func (l *loggerImpl) ReplaceField(field string, val interface{}) Logger { l.mtx.Lock() defer l.mtx.Unlock() return l.forkLogger(field, val) } -func (l *Logger) WithField(field string, val interface{}) *Logger { +func (l *loggerImpl) WithField(field string, val interface{}) Logger { l.mtx.Lock() defer l.mtx.Unlock() if val, ok := l.fields[field]; ok && val != nil { @@ -125,16 +140,16 @@ func (l *Logger) WithField(field string, val interface{}) *Logger { return l.forkLogger(field, val) } -func (l *Logger) WithFields(fields Fields) (ret *Logger) { +func (l *loggerImpl) WithFields(fields Fields) Logger { // TODO optimize - ret = l + var ret Logger = l for field, value := range fields { ret = ret.WithField(field, value) } return ret } -func (l *Logger) WithError(err error) *Logger { +func (l *loggerImpl) WithError(err error) Logger { val := interface{}(nil) if err != nil { val = err.Error() @@ -142,22 +157,22 @@ func (l *Logger) WithError(err error) *Logger { return l.WithField(FieldError, val) } -func (l *Logger) Debug(msg string) { +func (l *loggerImpl) Debug(msg string) { l.log(Debug, msg) } -func (l *Logger) Info(msg string) { +func (l *loggerImpl) Info(msg string) { l.log(Info, msg) } -func (l *Logger) Warn(msg string) { +func (l *loggerImpl) Warn(msg string) { l.log(Warn, msg) } -func (l *Logger) Error(msg string) { +func (l *loggerImpl) Error(msg string) { l.log(Error, msg) } -func (l *Logger) Printf(format string, args ...interface{}) { +func (l *loggerImpl) Printf(format string, args ...interface{}) { l.log(Error, fmt.Sprintf(format, args...)) } diff --git a/logger/nulllogger.go b/logger/nulllogger.go new file mode 100644 index 0000000..ec954f7 --- /dev/null +++ b/logger/nulllogger.go @@ -0,0 +1,20 @@ +package logger + +type nullLogger struct{} + +var _ Logger = nullLogger{} + +func NewNullLogger() Logger { + return nullLogger{} +} + +func (n nullLogger) WithOutlet(outlet Outlet, level Level) Logger { return n } +func (n nullLogger) ReplaceField(field string, val interface{}) Logger { return n } +func (n nullLogger) WithField(field string, val interface{}) Logger { return n } +func (n nullLogger) WithFields(fields Fields) Logger { return n } +func (n nullLogger) WithError(err error) Logger { return n } +func (nullLogger) Debug(msg string) {} +func (nullLogger) Info(msg string) {} +func (nullLogger) Warn(msg string) {} +func (nullLogger) Error(msg string) {} +func (nullLogger) Printf(format string, args ...interface{}) {} diff --git a/logger/testlogger.go b/logger/testlogger.go new file mode 100644 index 0000000..c416109 --- /dev/null +++ b/logger/testlogger.go @@ -0,0 +1,28 @@ +package logger + +import ( + "testing" +) + +type testLogger struct { + Logger +} + +type testingLoggerOutlet struct { + t *testing.T +} + +func (o testingLoggerOutlet) WriteEntry(entry Entry) error { + o.t.Logf("%#v", entry) + return nil +} + +var _ Logger = testLogger{} + +func NewTestLogger(t *testing.T) Logger { + outlets := NewOutlets() + outlets.Add(&testingLoggerOutlet{t}, Debug) + return &testLogger{ + Logger: NewLogger(outlets, 0), + } +} diff --git a/main.go b/main.go index a7919b6..7d209b9 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,22 @@ package main import ( - "github.com/zrepl/zrepl/cmd" - "log" - "os" + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/client" + "github.com/zrepl/zrepl/daemon" ) -func main() { - if err := cmd.RootCmd.Execute(); err != nil { - log.Printf("error executing root command: %s", err) - os.Exit(1) - } +func init() { + cli.AddSubcommand(daemon.DaemonCmd) + cli.AddSubcommand(client.StatusCmd) + cli.AddSubcommand(client.SignalCmd) + cli.AddSubcommand(client.StdinserverCmd) + cli.AddSubcommand(client.ConfigcheckCmd) + cli.AddSubcommand(client.VersionCmd) + cli.AddSubcommand(client.PprofCmd) + cli.AddSubcommand(client.TestCmd) +} + +func main() { + cli.Run() } diff --git a/pruning/keep_grid.go b/pruning/keep_grid.go new file mode 100644 index 0000000..24da21a --- /dev/null +++ b/pruning/keep_grid.go @@ -0,0 +1,102 @@ +package pruning + +import ( + "fmt" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/pruning/retentiongrid" + "regexp" + "sort" + "time" +) + +// KeepGrid fits snapshots that match a given regex into a retentiongrid.Grid, +// uses the most recent snapshot among those that match the regex as 'now', +// and deletes all snapshots that do not fit the grid specification. +type KeepGrid struct { + retentionGrid *retentiongrid.Grid + re *regexp.Regexp +} + +func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) { + + if in.Regex == "" { + return nil, fmt.Errorf("Regex must not be empty") + } + re, err := regexp.Compile(in.Regex) + if err != nil { + return nil, errors.Wrap(err, "Regex is invalid") + } + + // Assert intervals are of increasing length (not necessarily required, but indicates config mistake) + lastDuration := time.Duration(0) + for i := range in.Grid { + + if in.Grid[i].Length() < lastDuration { + // If all intervals before were keep=all, this is ok + allPrevKeepCountAll := true + for j := i - 1; allPrevKeepCountAll && j >= 0; j-- { + allPrevKeepCountAll = in.Grid[j].KeepCount() == config.RetentionGridKeepCountAll + } + if allPrevKeepCountAll { + goto isMonotonicIncrease + } + err = errors.New("retention grid interval length must be monotonically increasing") + return + } + isMonotonicIncrease: + lastDuration = in.Grid[i].Length() + + } + + retentionIntervals := make([]retentiongrid.Interval, len(in.Grid)) + for i := range in.Grid { + retentionIntervals[i] = &in.Grid[i] + } + + return &KeepGrid{ + retentiongrid.NewGrid(retentionIntervals), + re, + }, nil +} + +type retentionGridAdaptor struct { + Snapshot +} + +func (a retentionGridAdaptor) LessThan(b retentiongrid.Entry) bool { + return a.Date().Before(b.Date()) +} + +// Prune filters snapshots with the retention grid. +func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { + + snaps = filterSnapList(snaps, func(snapshot Snapshot) bool { + return p.re.MatchString(snapshot.Name()) + }) + if len(snaps) == 0 { + return nil + } + + // Build adaptors for retention grid + adaptors := make([]retentiongrid.Entry, 0) + for i := range snaps { + adaptors = append(adaptors, retentionGridAdaptor{snaps[i]}) + } + + // determine 'now' edge + sort.SliceStable(adaptors, func(i, j int) bool { + return adaptors[i].LessThan(adaptors[j]) + }) + now := adaptors[len(adaptors)-1].Date() + + // Evaluate retention grid + _, removea := p.retentionGrid.FitEntries(now, adaptors) + + // Revert adaptors + destroyList = make([]Snapshot, len(removea)) + for i := range removea { + destroyList[i] = removea[i].(retentionGridAdaptor).Snapshot + } + return destroyList +} diff --git a/pruning/keep_helpers.go b/pruning/keep_helpers.go new file mode 100644 index 0000000..a756ce7 --- /dev/null +++ b/pruning/keep_helpers.go @@ -0,0 +1,17 @@ +package pruning + +func filterSnapList(snaps []Snapshot, predicate func(Snapshot) bool) []Snapshot { + r := make([]Snapshot, 0, len(snaps)) + for i := range snaps { + if predicate(snaps[i]) { + r = append(r, snaps[i]) + } + } + return r +} + +func shallowCopySnapList(snaps []Snapshot) []Snapshot { + c := make([]Snapshot, len(snaps)) + copy(c, snaps) + return c +} diff --git a/pruning/keep_helpers_test.go b/pruning/keep_helpers_test.go new file mode 100644 index 0000000..bf8b399 --- /dev/null +++ b/pruning/keep_helpers_test.go @@ -0,0 +1,22 @@ +package pruning + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestShallowCopySnapList(t *testing.T) { + + l1 := []Snapshot{ + stubSnap{name: "foo"}, + stubSnap{name: "bar"}, + } + l2 := shallowCopySnapList(l1) + + assert.Equal(t, l1, l2) + + l1[0] = stubSnap{name: "baz"} + assert.Equal(t, "baz", l1[0].Name()) + assert.Equal(t, "foo", l2[0].Name()) + +} diff --git a/pruning/keep_last_n.go b/pruning/keep_last_n.go new file mode 100644 index 0000000..a275154 --- /dev/null +++ b/pruning/keep_last_n.go @@ -0,0 +1,32 @@ +package pruning + +import ( + "github.com/pkg/errors" + "sort" +) + +type KeepLastN struct { + n int +} + +func NewKeepLastN(n int) (*KeepLastN, error) { + if n <= 0 { + return nil, errors.Errorf("must specify positive number as 'keep last count', got %d", n) + } + return &KeepLastN{n}, nil +} + +func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { + + if k.n > len(snaps) { + return []Snapshot{} + } + + res := shallowCopySnapList(snaps) + + sort.Slice(res, func(i, j int) bool { + return res[i].Date().After(res[j].Date()) + }) + + return res[k.n:] +} diff --git a/pruning/keep_last_n_test.go b/pruning/keep_last_n_test.go new file mode 100644 index 0000000..8ef0000 --- /dev/null +++ b/pruning/keep_last_n_test.go @@ -0,0 +1,72 @@ +package pruning + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestKeepLastN(t *testing.T) { + + o := func(minutes int) time.Time { + return time.Unix(123, 0).Add(time.Duration(minutes) * time.Minute) + } + + inputs := map[string][]Snapshot{ + "s1": []Snapshot{ + stubSnap{name: "1", date: o(10)}, + stubSnap{name: "2", date: o(20)}, + stubSnap{name: "3", date: o(15)}, + stubSnap{name: "4", date: o(30)}, + stubSnap{name: "5", date: o(30)}, + }, + "s2": []Snapshot{}, + } + + tcs := map[string]testCase{ + "keep2": { + inputs: inputs["s1"], + rules: []KeepRule{ + KeepLastN{2}, + }, + expDestroy: map[string]bool{ + "1": true, "2": true, "3": true, + }, + }, + "keep1OfTwoWithSameTime": { // Keep one of two with same time + inputs: inputs["s1"], + rules: []KeepRule{ + KeepLastN{1}, + }, + expDestroyAlternatives: []map[string]bool{ + {"1": true, "2": true, "3": true, "4": true}, + {"1": true, "2": true, "3": true, "5": true}, + }, + }, + "keepMany": { + inputs: inputs["s1"], + rules: []KeepRule{ + KeepLastN{100}, + }, + expDestroy: map[string]bool{}, + }, + "empty": { + inputs: inputs["s2"], + rules: []KeepRule{ + KeepLastN{100}, + }, + expDestroy: map[string]bool{}, + }, + } + + testTable(tcs, t) + + t.Run("mustBePositive", func(t *testing.T) { + var err error + _, err = NewKeepLastN(0) + assert.Error(t, err) + _, err = NewKeepLastN(-5) + assert.Error(t, err) + }) + +} diff --git a/pruning/keep_not_replicated.go b/pruning/keep_not_replicated.go new file mode 100644 index 0000000..955d9e6 --- /dev/null +++ b/pruning/keep_not_replicated.go @@ -0,0 +1,15 @@ +package pruning + +type KeepNotReplicated struct { + forceConstructor struct{} +} + +func (*KeepNotReplicated) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { + return filterSnapList(snaps, func(snapshot Snapshot) bool { + return snapshot.Replicated() + }) +} + +func NewKeepNotReplicated() *KeepNotReplicated { + return &KeepNotReplicated{} +} diff --git a/pruning/keep_not_replicated_test.go b/pruning/keep_not_replicated_test.go new file mode 100644 index 0000000..47aa37d --- /dev/null +++ b/pruning/keep_not_replicated_test.go @@ -0,0 +1,39 @@ +package pruning + +import ( + "testing" +) + +func TestNewKeepNotReplicated(t *testing.T) { + + inputs := map[string][]Snapshot{ + "s1": []Snapshot{ + stubSnap{name: "1", replicated: true}, + stubSnap{name: "2", replicated: false}, + stubSnap{name: "3", replicated: true}, + }, + "s2": []Snapshot{}, + } + + tcs := map[string]testCase{ + "destroysOnlyReplicated": { + inputs: inputs["s1"], + rules: []KeepRule{ + NewKeepNotReplicated(), + }, + expDestroy: map[string]bool{ + "1": true, "3": true, + }, + }, + "empty": { + inputs: inputs["s2"], + rules: []KeepRule{ + NewKeepNotReplicated(), + }, + expDestroy: map[string]bool{}, + }, + } + + testTable(tcs, t) + +} diff --git a/pruning/keep_regex.go b/pruning/keep_regex.go new file mode 100644 index 0000000..f8a5956 --- /dev/null +++ b/pruning/keep_regex.go @@ -0,0 +1,33 @@ +package pruning + +import ( + "regexp" +) + +type KeepRegex struct { + expr *regexp.Regexp +} + +var _ KeepRule = &KeepRegex{} + +func NewKeepRegex(expr string) (*KeepRegex, error) { + re, err := regexp.Compile(expr) + if err != nil { + return nil, err + } + return &KeepRegex{re}, nil +} + +func MustKeepRegex(expr string) *KeepRegex { + k, err := NewKeepRegex(expr) + if err != nil { + panic(err) + } + return k +} + +func (k *KeepRegex) KeepRule(snaps []Snapshot) []Snapshot { + return filterSnapList(snaps, func(s Snapshot) bool { + return k.expr.FindStringIndex(s.Name()) == nil + }) +} diff --git a/pruning/pruning.go b/pruning/pruning.go new file mode 100644 index 0000000..2c4dec9 --- /dev/null +++ b/pruning/pruning.go @@ -0,0 +1,69 @@ +package pruning + +import ( + "fmt" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" + "time" +) + +type KeepRule interface { + KeepRule(snaps []Snapshot) (destroyList []Snapshot) +} + +type Snapshot interface { + Name() string + Replicated() bool + Date() time.Time +} + +// The returned snapshot list is guaranteed to only contains elements of input parameter snaps +func PruneSnapshots(snaps []Snapshot, keepRules []KeepRule) []Snapshot { + + if keepRules == nil || len(keepRules) == 0 { + return []Snapshot{} + } + + remCount := make(map[Snapshot]int, len(snaps)) + for _, r := range keepRules { + ruleRems := r.KeepRule(snaps) + for _, ruleRem := range ruleRems { + remCount[ruleRem]++ + } + } + + remove := make([]Snapshot, 0, len(snaps)) + for snap, rc := range remCount { + if rc == len(keepRules) { + remove = append(remove, snap) + } + } + + return remove +} + +func RulesFromConfig(in []config.PruningEnum) (rules []KeepRule, err error) { + rules = make([]KeepRule, len(in)) + for i := range in { + rules[i], err = RuleFromConfig(in[i]) + if err != nil { + return nil, errors.Wrapf(err, "cannot build rule #%d", i) + } + } + return rules, nil +} + +func RuleFromConfig(in config.PruningEnum) (KeepRule, error) { + switch v := in.Ret.(type) { + case *config.PruneKeepNotReplicated: + return NewKeepNotReplicated(), nil + case *config.PruneKeepLastN: + return NewKeepLastN(v.Count) + case *config.PruneKeepRegex: + return NewKeepRegex(v.Regex) + case *config.PruneGrid: + return NewKeepGrid(v) + default: + return nil, fmt.Errorf("unknown keep rule type %T", v) + } +} diff --git a/pruning/pruning_test.go b/pruning/pruning_test.go new file mode 100644 index 0000000..678dc50 --- /dev/null +++ b/pruning/pruning_test.go @@ -0,0 +1,127 @@ +package pruning + +import ( + "testing" + "time" +) + +type stubSnap struct { + name string + replicated bool + date time.Time +} + +func (s stubSnap) Name() string { return s.name } + +func (s stubSnap) Replicated() bool { return s.replicated } + +func (s stubSnap) Date() time.Time { return s.date } + +type testCase struct { + inputs []Snapshot + rules []KeepRule + expDestroy, effDestroy map[string]bool + expDestroyAlternatives []map[string]bool +} + +func testTable(tcs map[string]testCase, t *testing.T) { + mapEqual := func(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true + } + + for name := range tcs { + t.Run(name, func(t *testing.T) { + tc := tcs[name] + remove := PruneSnapshots(tc.inputs, tc.rules) + tc.effDestroy = make(map[string]bool) + for _, s := range remove { + tc.effDestroy[s.Name()] = true + } + if tc.expDestroyAlternatives == nil { + if tc.expDestroy == nil { + panic("must specify either expDestroyAlternatives or expDestroy") + } + tc.expDestroyAlternatives = []map[string]bool{tc.expDestroy} + } + var okAlt map[string]bool = nil + for _, alt := range tc.expDestroyAlternatives { + t.Logf("testing possible result: %v", alt) + if mapEqual(alt, tc.effDestroy) { + okAlt = alt + } + } + if okAlt == nil { + t.Errorf("no alternatives matched result: %v", tc.effDestroy) + } + }) + } +} + +func TestPruneSnapshots(t *testing.T) { + + inputs := map[string][]Snapshot{ + "s1": []Snapshot{ + stubSnap{name: "foo_123"}, + stubSnap{name: "foo_456"}, + stubSnap{name: "bar_123"}, + }, + } + + tcs := map[string]testCase{ + "simple": { + inputs: inputs["s1"], + rules: []KeepRule{ + MustKeepRegex("foo_"), + }, + expDestroy: map[string]bool{ + "bar_123": true, + }, + }, + "multipleRules": { + inputs: inputs["s1"], + rules: []KeepRule{ + MustKeepRegex("foo_"), + MustKeepRegex("bar_"), + }, + expDestroy: map[string]bool{}, + }, + "onlyThoseRemovedByAllAreRemoved": { + inputs: inputs["s1"], + rules: []KeepRule{ + MustKeepRegex("notInS1"), // would remove all + MustKeepRegex("bar_"), // would remove all but bar_, i.e. foo_.* + }, + expDestroy: map[string]bool{ + "foo_123": true, + "foo_456": true, + }, + }, + "noRulesKeepsAll": { + inputs: inputs["s1"], + rules: []KeepRule{}, + expDestroy: map[string]bool{}, + }, + "nilRulesKeepsAll": { + inputs: inputs["s1"], + rules: nil, + expDestroy: map[string]bool{}, + }, + "noSnaps": { + inputs: []Snapshot{}, + rules: []KeepRule{ + MustKeepRegex("foo_"), + }, + expDestroy: map[string]bool{}, + }, + } + + testTable(tcs, t) +} diff --git a/util/retentiongrid.go b/pruning/retentiongrid/retentiongrid.go similarity index 59% rename from util/retentiongrid.go rename to pruning/retentiongrid/retentiongrid.go index 9e114b9..78f8767 100644 --- a/util/retentiongrid.go +++ b/pruning/retentiongrid/retentiongrid.go @@ -1,57 +1,58 @@ -package util +package retentiongrid import ( "sort" "time" ) -type RetentionInterval struct { - Length time.Duration - KeepCount int +type Interval interface { + Length() time.Duration + KeepCount() int } const RetentionGridKeepCountAll int = -1 -type RetentionGrid struct { - intervals []RetentionInterval +type Grid struct { + intervals []Interval } -type RetentionGridEntry interface { +//A point inside the grid, i.e. a thing the grid can decide to remove +type Entry interface { Date() time.Time - LessThan(b RetentionGridEntry) bool + LessThan(b Entry) bool } -func dateInInterval(date, startDateInterval time.Time, i RetentionInterval) bool { - return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length)) +func dateInInterval(date, startDateInterval time.Time, i Interval) bool { + return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length())) } -func NewRetentionGrid(l []RetentionInterval) *RetentionGrid { +func NewGrid(l []Interval) *Grid { // TODO Maybe check for ascending interval lengths here, although the algorithm // itself doesn't care about that. - return &RetentionGrid{l} + return &Grid{l} } -// Partition a list of RetentionGridEntries into the RetentionGrid, +// Partition a list of RetentionGridEntries into the Grid, // relative to a given start date `now`. // -// The `KeepCount` oldest entries per `RetentionInterval` are kept (`keep`), +// The `keepCount` oldest entries per `retentiongrid.Interval` are kept (`keep`), // the others are removed (`remove`). // // Entries that are younger than `now` are always kept. // Those that are older than the earliest beginning of an interval are removed. -func (g RetentionGrid) FitEntries(now time.Time, entries []RetentionGridEntry) (keep, remove []RetentionGridEntry) { +func (g Grid) FitEntries(now time.Time, entries []Entry) (keep, remove []Entry) { type bucket struct { - entries []RetentionGridEntry + entries []Entry } buckets := make([]bucket, len(g.intervals)) - keep = make([]RetentionGridEntry, 0) - remove = make([]RetentionGridEntry, 0) + keep = make([]Entry, 0) + remove = make([]Entry, 0) oldestIntervalStart := now for i := range g.intervals { - oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length) + oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length()) } for ei := 0; ei < len(entries); ei++ { @@ -69,7 +70,7 @@ func (g RetentionGrid) FitEntries(now time.Time, entries []RetentionGridEntry) ( iStartTime := now for i := 0; i < len(g.intervals); i++ { - iStartTime = iStartTime.Add(-g.intervals[i].Length) + iStartTime = iStartTime.Add(-g.intervals[i].Length()) if date == iStartTime || dateInInterval(date, iStartTime, g.intervals[i]) { buckets[i].entries = append(buckets[i].entries, e) } @@ -78,14 +79,14 @@ func (g RetentionGrid) FitEntries(now time.Time, entries []RetentionGridEntry) ( for bi, b := range buckets { - interval := &g.intervals[bi] + interval := g.intervals[bi] sort.SliceStable(b.entries, func(i, j int) bool { return b.entries[i].LessThan((b.entries[j])) }) i := 0 - for ; (interval.KeepCount == RetentionGridKeepCountAll || i < interval.KeepCount) && i < len(b.entries); i++ { + for ; (interval.KeepCount() == RetentionGridKeepCountAll || i < interval.KeepCount()) && i < len(b.entries); i++ { keep = append(keep, b.entries[i]) } for ; i < len(b.entries); i++ { diff --git a/util/retentiongrid_test.go b/pruning/retentiongrid/retentiongrid_test.go similarity index 80% rename from util/retentiongrid_test.go rename to pruning/retentiongrid/retentiongrid_test.go index b2fb0fb..3e5e52f 100644 --- a/util/retentiongrid_test.go +++ b/pruning/retentiongrid/retentiongrid_test.go @@ -1,4 +1,4 @@ -package util +package retentiongrid import ( "fmt" @@ -9,10 +9,23 @@ import ( "time" ) -func retentionGridFromString(gs string) (g *RetentionGrid) { +type retentionIntervalStub struct { + length time.Duration + keepCount int +} + +func (i *retentionIntervalStub) Length() time.Duration { + return i.length +} + +func (i *retentionIntervalStub) KeepCount() int { + return i.keepCount +} + +func gridFromString(gs string) (g *Grid) { intervals := strings.Split(gs, "|") - g = &RetentionGrid{ - intervals: make([]RetentionInterval, len(intervals)), + g = &Grid{ + intervals: make([]Interval, len(intervals)), } for idx, i := range intervals { comps := strings.SplitN(i, ",", 2) @@ -25,16 +38,16 @@ func retentionGridFromString(gs string) (g *RetentionGrid) { } var err error - var interval RetentionInterval + var interval retentionIntervalStub - if interval.KeepCount, err = strconv.Atoi(numSnapsStr); err != nil { + if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil { panic(err) } - if interval.Length, err = time.ParseDuration(durationStr); err != nil { + if interval.length, err = time.ParseDuration(durationStr); err != nil { panic(err) } - g.intervals[idx] = interval + g.intervals[idx] = &interval } return } @@ -49,11 +62,11 @@ func (ds dummySnap) Date() time.Time { return ds.date } -func (ds dummySnap) LessThan(b RetentionGridEntry) bool { +func (ds dummySnap) LessThan(b Entry) bool { return ds.date.Before(b.(dummySnap).date) // don't have a txg here } -func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []RetentionGridEntry) { +func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) { snapDescr := func(d dummySnap) string { return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now)) @@ -88,8 +101,8 @@ func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, r } func TestRetentionGridFitEntriesEmptyInput(t *testing.T) { - g := retentionGridFromString("10m|10m|10m|1h") - keep, remove := g.FitEntries(time.Now(), []RetentionGridEntry{}) + g := gridFromString("10m|10m|10m|1h") + keep, remove := g.FitEntries(time.Now(), []Entry{}) assert.Empty(t, keep) assert.Empty(t, remove) } @@ -98,13 +111,13 @@ func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) { // Intervals are (duration], i.e. 10min is in the first interval, not in the second - g := retentionGridFromString("10m|10m|10m") + g := gridFromString("10m|10m|10m") t.Logf("%#v\n", g) now := time.Unix(0, 0) - snaps := []RetentionGridEntry{ + snaps := []Entry{ dummySnap{"0", true, now.Add(1 * time.Minute)}, // before now dummySnap{"1", true, now}, // before now dummySnap{"2", true, now.Add(-10 * time.Minute)}, // 1st interval @@ -120,13 +133,13 @@ func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) { func TestRetentionGridFitEntries(t *testing.T) { - g := retentionGridFromString("10m,-1|10m|10m,2|1h") + g := gridFromString("10m,-1|10m|10m,2|1h") t.Logf("%#v\n", g) now := time.Unix(0, 0) - snaps := []RetentionGridEntry{ + snaps := []Entry{ dummySnap{"1", true, now.Add(3 * time.Minute)}, // pre-now must always be kept dummySnap{"b1", true, now.Add(-6 * time.Minute)}, // 1st interval allows unlimited entries dummySnap{"b3", true, now.Add(-8 * time.Minute)}, // 1st interval allows unlimited entries diff --git a/replication/context.go b/replication/context.go new file mode 100644 index 0000000..7e43981 --- /dev/null +++ b/replication/context.go @@ -0,0 +1,29 @@ +package replication + +import ( + "context" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/replication/fsrep" +) + +type contextKey int + +const ( + contextKeyLog contextKey = iota +) + +type Logger = logger.Logger + +func WithLogger(ctx context.Context, l Logger) context.Context { + ctx = context.WithValue(ctx, contextKeyLog, l) + ctx = fsrep.WithLogger(ctx, l) + return ctx +} + +func getLogger(ctx context.Context) Logger { + l, ok := ctx.Value(contextKeyLog).(Logger) + if !ok { + l = logger.NewNullLogger() + } + return l +} diff --git a/replication/fsrep/fsfsm.go b/replication/fsrep/fsfsm.go new file mode 100644 index 0000000..0297dc9 --- /dev/null +++ b/replication/fsrep/fsfsm.go @@ -0,0 +1,532 @@ +// Package fsrep implements replication of a single file system with existing versions +// from a sender to a receiver. +package fsrep + +import ( + "context" + "errors" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "io" + "net" + "sync" + "time" + + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/replication/pdu" + "github.com/zrepl/zrepl/util" +) + +type contextKey int + +const ( + contextKeyLogger contextKey = iota +) + +type Logger = logger.Logger + +func WithLogger(ctx context.Context, log Logger) context.Context { + return context.WithValue(ctx, contextKeyLogger, log) +} + +func getLogger(ctx context.Context) Logger { + l, ok := ctx.Value(contextKeyLogger).(Logger) + if !ok { + l = logger.NewNullLogger() + } + return l +} + +// A Sender is usually part of a github.com/zrepl/zrepl/replication.Endpoint. +type Sender interface { + // If a non-nil io.ReadCloser is returned, it is guaranteed to be closed before + // any next call to the parent github.com/zrepl/zrepl/replication.Endpoint. + // If the send request is for dry run the io.ReadCloser will be nil + Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) + ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) +} + +// A Sender is usually part of a github.com/zrepl/zrepl/replication.Endpoint. +type Receiver interface { + // Receive sends r and sendStream (the latter containing a ZFS send stream) + // to the parent github.com/zrepl/zrepl/replication.Endpoint. + // Implementors must guarantee that Close was called on sendStream before + // the call to Receive returns. + Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error +} + +type StepReport struct { + From, To string + Status StepState + Problem string + Bytes int64 + ExpectedBytes int64 // 0 means no size estimate possible +} + +type Report struct { + Filesystem string + Status string + Problem string + Completed, Pending []*StepReport +} + +//go:generate stringer -type=State +type State uint + +const ( + Ready State = 1 << iota + RetryWait + PermanentError + Completed +) + +func (s State) fsrsf() state { + m := map[State]state{ + Ready: stateReady, + RetryWait: stateRetryWait, + PermanentError: nil, + Completed: nil, + } + return m[s] +} + +type Replication struct { + promBytesReplicated prometheus.Counter + + // lock protects all fields below it in this struct, but not the data behind pointers + lock sync.Mutex + state State + fs string + err error + retryWaitUntil time.Time + completed, pending []*ReplicationStep +} + +func (f *Replication) State() State { + f.lock.Lock() + defer f.lock.Unlock() + return f.state +} + +func (f *Replication) UpdateSizeEsitmate(ctx context.Context, sender Sender) error { + f.lock.Lock() + defer f.lock.Unlock() + for _, e := range f.pending { + if err := e.updateSizeEstimate(ctx, sender); err != nil { + return err + } + } + return nil +} + +type ReplicationBuilder struct { + r *Replication +} + +func BuildReplication(fs string, promBytesReplicated prometheus.Counter) *ReplicationBuilder { + return &ReplicationBuilder{&Replication{fs: fs, promBytesReplicated: promBytesReplicated}} +} + +func (b *ReplicationBuilder) AddStep(from, to FilesystemVersion) *ReplicationBuilder { + step := &ReplicationStep{ + state: StepReplicationReady, + parent: b.r, + from: from, + to: to, + } + b.r.pending = append(b.r.pending, step) + return b +} + +func (b *ReplicationBuilder) Done() (r *Replication) { + if len(b.r.pending) > 0 { + b.r.state = Ready + } else { + b.r.state = Completed + } + r = b.r + b.r = nil + return r +} + +func NewReplicationWithPermanentError(fs string, err error) *Replication { + return &Replication{ + state: PermanentError, + fs: fs, + err: err, + } +} + +//go:generate stringer -type=StepState +type StepState uint + +const ( + StepReplicationReady StepState = 1 << iota + StepReplicationRetry + StepMarkReplicatedReady + StepMarkReplicatedRetry + StepPermanentError + StepCompleted +) + +type FilesystemVersion interface { + SnapshotTime() time.Time + GetName() string // name without @ or # + RelName() string // name with @ or # +} + +type ReplicationStep struct { + // only protects state, err + // from, to and parent are assumed to be immutable + lock sync.Mutex + + state StepState + from, to FilesystemVersion + parent *Replication + + // both retry and permanent error + err error + + byteCounter *util.ByteCounterReader + expectedSize int64 // 0 means no size estimate present / possible +} + +func (f *Replication) TakeStep(ctx context.Context, sender Sender, receiver Receiver) (post State, nextStepDate, retryWaitUntil time.Time) { + + var u updater = func(fu func(*Replication)) State { + f.lock.Lock() + defer f.lock.Unlock() + if fu != nil { + fu(f) + } + return f.state + } + var s state = u(nil).fsrsf() + + pre := u(nil) + preTime := time.Now() + s = s(ctx, sender, receiver, u) + delta := time.Now().Sub(preTime) + + post = u(func(f *Replication) { + if len(f.pending) == 0 { + return + } + nextStepDate = f.pending[0].to.SnapshotTime() + retryWaitUntil = f.retryWaitUntil + }) + + getLogger(ctx). + WithField("fs", f.fs). + WithField("transition", fmt.Sprintf("%s => %s", pre, post)). + WithField("duration", delta). + Debug("fsr step taken") + + return post, nextStepDate, retryWaitUntil +} + +func (f *Replication) RetryWaitUntil() time.Time { + f.lock.Lock() + defer f.lock.Unlock() + return f.retryWaitUntil +} + +type updater func(func(fsr *Replication)) State + +type state func(ctx context.Context, sender Sender, receiver Receiver, u updater) state + +var RetrySleepDuration = 10 * time.Second // FIXME make configurable + +func stateReady(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + + var current *ReplicationStep + s := u(func(f *Replication) { + if len(f.pending) == 0 { + f.state = Completed + return + } + current = f.pending[0] + }) + if s != Ready { + return s.fsrsf() + } + + stepState := current.doReplication(ctx, sender, receiver) + + return u(func(f *Replication) { + switch stepState { + case StepCompleted: + f.completed = append(f.completed, current) + f.pending = f.pending[1:] + if len(f.pending) > 0 { + f.state = Ready + } else { + f.state = Completed + } + case StepReplicationRetry: + fallthrough + case StepMarkReplicatedRetry: + f.retryWaitUntil = time.Now().Add(RetrySleepDuration) + f.state = RetryWait + case StepPermanentError: + f.state = PermanentError + f.err = errors.New("a replication step failed with a permanent error") + default: + panic(f) + } + }).fsrsf() +} + +func stateRetryWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + var sleepUntil time.Time + u(func(f *Replication) { + sleepUntil = f.retryWaitUntil + }) + if time.Now().Before(sleepUntil) { + return u(nil).fsrsf() + } + return u(func(f *Replication) { + f.state = Ready + }).fsrsf() +} + +func (fsr *Replication) Report() *Report { + fsr.lock.Lock() + defer fsr.lock.Unlock() + + rep := Report{ + Filesystem: fsr.fs, + Status: fsr.state.String(), + } + + if fsr.state&PermanentError != 0 { + rep.Problem = fsr.err.Error() + return &rep + } + + rep.Completed = make([]*StepReport, len(fsr.completed)) + for i := range fsr.completed { + rep.Completed[i] = fsr.completed[i].Report() + } + rep.Pending = make([]*StepReport, len(fsr.pending)) + for i := range fsr.pending { + rep.Pending[i] = fsr.pending[i].Report() + } + + if fsr.state&RetryWait != 0 { + if len(rep.Pending) != 0 { // should always be true for RetryWait == true? + rep.Problem = rep.Pending[0].Problem + } + } + + return &rep +} + +func shouldRetry(err error) bool { + switch err { + case io.EOF: + fallthrough + case io.ErrUnexpectedEOF: + fallthrough + case io.ErrClosedPipe: + return true + } + if _, ok := err.(net.Error); ok { + return true + } + return false +} + +func (s *ReplicationStep) doReplication(ctx context.Context, sender Sender, receiver Receiver) StepState { + + fs := s.parent.fs + + log := getLogger(ctx). + WithField("filesystem", fs). + WithField("step", s.String()) + + updateStateError := func(err error) StepState { + s.lock.Lock() + defer s.lock.Unlock() + + s.err = err + if shouldRetry(s.err) { + s.state = StepReplicationRetry + return s.state + } + s.state = StepPermanentError + return s.state + } + + updateStateCompleted := func() StepState { + s.lock.Lock() + defer s.lock.Unlock() + s.err = nil + s.state = StepMarkReplicatedReady + return s.state + } + + sr := s.buildSendRequest(false) + + log.Debug("initiate send request") + sres, sstream, err := sender.Send(ctx, sr) + if err != nil { + log.WithError(err).Error("send request failed") + return updateStateError(err) + } + if sstream == nil { + err := errors.New("send request did not return a stream, broken endpoint implementation") + return updateStateError(err) + } + + s.byteCounter = util.NewByteCounterReader(sstream) + defer func() { + s.parent.promBytesReplicated.Add(float64(s.byteCounter.Bytes())) + }() + sstream = s.byteCounter + + rr := &pdu.ReceiveReq{ + Filesystem: fs, + ClearResumeToken: !sres.UsedResumeToken, + } + log.Debug("initiate receive request") + err = receiver.Receive(ctx, rr, sstream) + if err != nil { + log. + WithError(err). + WithField("errType", fmt.Sprintf("%T", err)). + Error("receive request failed (might also be error on sender)") + sstream.Close() + // This failure could be due to + // - an unexpected exit of ZFS on the sending side + // - an unexpected exit of ZFS on the receiving side + // - a connectivity issue + return updateStateError(err) + } + log.Debug("receive finished") + + updateStateCompleted() + + return s.doMarkReplicated(ctx, sender) + +} + +func (s *ReplicationStep) doMarkReplicated(ctx context.Context, sender Sender) StepState { + + log := getLogger(ctx). + WithField("filesystem", s.parent.fs). + WithField("step", s.String()) + + updateStateError := func(err error) StepState { + s.lock.Lock() + defer s.lock.Unlock() + + s.err = err + if shouldRetry(s.err) { + s.state = StepMarkReplicatedRetry + return s.state + } + s.state = StepPermanentError + return s.state + } + + updateStateCompleted := func() StepState { + s.lock.Lock() + defer s.lock.Unlock() + s.state = StepCompleted + return s.state + } + + log.Debug("advance replication cursor") + req := &pdu.ReplicationCursorReq{ + Filesystem: s.parent.fs, + Op: &pdu.ReplicationCursorReq_Set{ + Set: &pdu.ReplicationCursorReq_SetOp{ + Snapshot: s.to.GetName(), + }, + }, + } + res, err := sender.ReplicationCursor(ctx, req) + if err != nil { + log.WithError(err).Error("error advancing replication cursor") + return updateStateError(err) + } + if res.GetError() != "" { + err := fmt.Errorf("cannot advance replication cursor: %s", res.GetError()) + log.Error(err.Error()) + return updateStateError(err) + } + + return updateStateCompleted() +} + +func (s *ReplicationStep) updateSizeEstimate(ctx context.Context, sender Sender) error { + + fs := s.parent.fs + + log := getLogger(ctx). + WithField("filesystem", fs). + WithField("step", s.String()) + + sr := s.buildSendRequest(true) + + log.Debug("initiate dry run send request") + sres, _, err := sender.Send(ctx, sr) + if err != nil { + log.WithError(err).Error("dry run send request failed") + return err + } + s.expectedSize = sres.ExpectedSize + return nil +} + +func (s *ReplicationStep) buildSendRequest(dryRun bool) (sr *pdu.SendReq) { + fs := s.parent.fs + if s.from == nil { + sr = &pdu.SendReq{ + Filesystem: fs, + From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally + DryRun: dryRun, + } + } else { + sr = &pdu.SendReq{ + Filesystem: fs, + From: s.from.RelName(), + To: s.to.RelName(), + DryRun: dryRun, + } + } + return sr +} + +func (s *ReplicationStep) String() string { + if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send + return fmt.Sprintf("%s%s (full)", s.parent.fs, s.to.RelName()) + } else { + return fmt.Sprintf("%s(%s => %s)", s.parent.fs, s.from.RelName(), s.to.RelName()) + } +} + +func (s *ReplicationStep) Report() *StepReport { + var from string // FIXME follow same convention as ZFS: to should be nil on full send + if s.from != nil { + from = s.from.RelName() + } + bytes := int64(0) + if s.byteCounter != nil { + bytes = s.byteCounter.Bytes() + } + problem := "" + if s.err != nil { + problem = s.err.Error() + } + rep := StepReport{ + From: from, + To: s.to.RelName(), + Status: s.state, + Problem: problem, + Bytes: bytes, + ExpectedBytes: s.expectedSize, + } + return &rep +} diff --git a/replication/fsrep/state_string.go b/replication/fsrep/state_string.go new file mode 100644 index 0000000..8bf26f4 --- /dev/null +++ b/replication/fsrep/state_string.go @@ -0,0 +1,29 @@ +// Code generated by "stringer -type=State"; DO NOT EDIT. + +package fsrep + +import "strconv" + +const ( + _State_name_0 = "ReadyRetryWait" + _State_name_1 = "PermanentError" + _State_name_2 = "Completed" +) + +var ( + _State_index_0 = [...]uint8{0, 5, 14} +) + +func (i State) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _State_name_0[_State_index_0[i]:_State_index_0[i+1]] + case i == 4: + return _State_name_1 + case i == 8: + return _State_name_2 + default: + return "State(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/replication/fsrep/stepstate_string.go b/replication/fsrep/stepstate_string.go new file mode 100644 index 0000000..aeac7e1 --- /dev/null +++ b/replication/fsrep/stepstate_string.go @@ -0,0 +1,35 @@ +// Code generated by "stringer -type=StepState"; DO NOT EDIT. + +package fsrep + +import "strconv" + +const ( + _StepState_name_0 = "StepReplicationReadyStepReplicationRetry" + _StepState_name_1 = "StepMarkReplicatedReady" + _StepState_name_2 = "StepMarkReplicatedRetry" + _StepState_name_3 = "StepPermanentError" + _StepState_name_4 = "StepCompleted" +) + +var ( + _StepState_index_0 = [...]uint8{0, 20, 40} +) + +func (i StepState) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _StepState_name_0[_StepState_index_0[i]:_StepState_index_0[i+1]] + case i == 4: + return _StepState_name_1 + case i == 8: + return _StepState_name_2 + case i == 16: + return _StepState_name_3 + case i == 32: + return _StepState_name_4 + default: + return "StepState(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/replication/internal/diff/diff.go b/replication/internal/diff/diff.go new file mode 100644 index 0000000..6af5246 --- /dev/null +++ b/replication/internal/diff/diff.go @@ -0,0 +1,114 @@ +package mainfsm + +import ( + "sort" + + . "github.com/zrepl/zrepl/replication/pdu" +) + +type ConflictNoCommonAncestor struct { + SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion +} + +func (c *ConflictNoCommonAncestor) Error() string { + return "no common snapshot or suitable bookmark between sender and receiver" +} + +type ConflictDiverged struct { + SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion + CommonAncestor *FilesystemVersion + SenderOnly, ReceiverOnly []*FilesystemVersion +} + +func (c *ConflictDiverged) Error() string { + return "the receiver's latest snapshot is not present on sender" +} + +func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion { + lesser := func(s []*FilesystemVersion) func(i, j int) bool { + return func(i, j int) bool { + if s[i].CreateTXG < s[j].CreateTXG { + return true + } + if s[i].CreateTXG == s[j].CreateTXG { + // Bookmark < Snapshot + return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot + } + return false + } + } + if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) { + return fsvslice + } + sorted := make([]*FilesystemVersion, len(fsvslice)) + copy(sorted, fsvslice) + sort.Slice(sorted, lesser(sorted)) + return sorted +} + +// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor +func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) { + + receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver) + sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender) + + // Find most recent common ancestor by name, preferring snapshots over bookmarks + + mrcaRcv := len(receiver) - 1 + mrcaSnd := len(sender) - 1 + + for mrcaRcv >= 0 && mrcaSnd >= 0 { + if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid { + // Since we arrive from the end of the array, and because we defined bookmark < snapshot, + // this condition will match snapshot first, which is what we want because it gives us + // size estimation + break + } + receiverCreation, err := receiver[mrcaRcv].CreationAsTime() + if err != nil { + panic(err) // FIXME move this to a sorting phase before + } + senderCreation, err := sender[mrcaSnd].CreationAsTime() + if err != nil { + panic(err) // FIXME move this to the sorting phase before + } + + if receiverCreation.Before(senderCreation) { + mrcaSnd-- + } else { + mrcaRcv-- + } + } + + if mrcaRcv == -1 || mrcaSnd == -1 { + return nil, &ConflictNoCommonAncestor{ + SortedSenderVersions: sender, + SortedReceiverVersions: receiver, + } + } + + if mrcaRcv != len(receiver)-1 { + return nil, &ConflictDiverged{ + SortedSenderVersions: sender, + SortedReceiverVersions: receiver, + CommonAncestor: sender[mrcaSnd], + SenderOnly: sender[mrcaSnd+1:], + ReceiverOnly: receiver[mrcaRcv+1:], + } + } + + // incPath must not contain bookmarks except initial one, + incPath = make([]*FilesystemVersion, 0, len(sender)) + incPath = append(incPath, sender[mrcaSnd]) + // it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath + for i := mrcaSnd + 1; i < len(sender); i++ { + if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid { + incPath = append(incPath, sender[i]) + } + } + if len(incPath) == 1 { + // nothing to do + incPath = incPath[1:] + } + return incPath, nil +} diff --git a/replication/internal/queue/queue.go b/replication/internal/queue/queue.go new file mode 100644 index 0000000..323460b --- /dev/null +++ b/replication/internal/queue/queue.go @@ -0,0 +1,121 @@ +package queue + +import ( + "sort" + "time" + + . "github.com/zrepl/zrepl/replication/fsrep" +) + +type replicationQueueItem struct { + // duplicates fsr.state to avoid accessing and locking fsr + state State + // duplicates fsr.current.nextStepDate to avoid accessing & locking fsr + nextStepDate time.Time + // duplicates fsr.retryWaitUntil to avoid accessing & locking fsr + retryWaitUntil time.Time + + fsr *Replication +} + +type ReplicationQueue []*replicationQueueItem + +func NewReplicationQueue() *ReplicationQueue { + q := make(ReplicationQueue, 0) + return &q +} + +func (q ReplicationQueue) Len() int { return len(q) } +func (q ReplicationQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] } + +type lessmapEntry struct { + prio int + less func(a, b *replicationQueueItem) bool +} + +var lessmap = map[State]lessmapEntry{ + Ready: { + prio: 0, + less: func(a, b *replicationQueueItem) bool { + return a.nextStepDate.Before(b.nextStepDate) + }, + }, + RetryWait: { + prio: 1, + less: func(a, b *replicationQueueItem) bool { + return a.retryWaitUntil.Before(b.retryWaitUntil) + }, + }, +} + +func (q ReplicationQueue) Less(i, j int) bool { + + a, b := q[i], q[j] + al, aok := lessmap[a.state] + if !aok { + panic(a) + } + bl, bok := lessmap[b.state] + if !bok { + panic(b) + } + + if al.prio != bl.prio { + return al.prio < bl.prio + } + + return al.less(a, b) +} + +func (q *ReplicationQueue) sort() (done []*Replication) { + // pre-scan for everything that is not ready + newq := make(ReplicationQueue, 0, len(*q)) + done = make([]*Replication, 0, len(*q)) + for _, qitem := range *q { + if _, ok := lessmap[qitem.state]; !ok { + done = append(done, qitem.fsr) + } else { + newq = append(newq, qitem) + } + } + sort.Stable(newq) // stable to avoid flickering in reports + *q = newq + return done +} + +// next remains valid until the next call to GetNext() +func (q *ReplicationQueue) GetNext() (done []*Replication, next *ReplicationQueueItemHandle) { + done = q.sort() + if len(*q) == 0 { + return done, nil + } + next = &ReplicationQueueItemHandle{(*q)[0]} + return done, next +} + +func (q *ReplicationQueue) Add(fsr *Replication) { + *q = append(*q, &replicationQueueItem{ + fsr: fsr, + state: fsr.State(), + }) +} + +func (q *ReplicationQueue) Foreach(fu func(*ReplicationQueueItemHandle)) { + for _, qitem := range *q { + fu(&ReplicationQueueItemHandle{qitem}) + } +} + +type ReplicationQueueItemHandle struct { + i *replicationQueueItem +} + +func (h ReplicationQueueItemHandle) GetFSReplication() *Replication { + return h.i.fsr +} + +func (h ReplicationQueueItemHandle) Update(newState State, nextStepDate, retryWaitUntil time.Time) { + h.i.state = newState + h.i.nextStepDate = nextStepDate + h.i.retryWaitUntil = retryWaitUntil +} diff --git a/replication/mainfsm.go b/replication/mainfsm.go new file mode 100644 index 0000000..086fda1 --- /dev/null +++ b/replication/mainfsm.go @@ -0,0 +1,427 @@ +// Package replication implements replication of filesystems with existing +// versions (snapshots) from a sender to a receiver. +package replication + +import ( + "context" + "errors" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/daemon/job/wakeup" + "math/bits" + "sync" + "time" + + "github.com/zrepl/zrepl/replication/fsrep" + . "github.com/zrepl/zrepl/replication/internal/diff" + . "github.com/zrepl/zrepl/replication/internal/queue" + "github.com/zrepl/zrepl/replication/pdu" +) + +//go:generate enumer -type=State +type State uint + +const ( + Planning State = 1 << iota + PlanningError + Working + WorkingWait + Completed + ContextDone +) + +func (s State) rsf() state { + idx := bits.TrailingZeros(uint(s)) + if idx == bits.UintSize { + panic(s) // invalid value + } + m := []state{ + statePlanning, + statePlanningError, + stateWorking, + stateWorkingWait, + nil, + nil, + } + return m[idx] +} + +// Replication implements the replication of multiple file systems from a Sender to a Receiver. +// +// It is a state machine that is driven by the Drive method +// and provides asynchronous reporting via the Report method (i.e. from another goroutine). +type Replication struct { + // not protected by lock + promSecsPerState *prometheus.HistogramVec // labels: state + promBytesReplicated *prometheus.CounterVec // labels: filesystem + + // lock protects all fields of this struct (but not the fields behind pointers!) + lock sync.Mutex + + state State + + // Working, WorkingWait, Completed, ContextDone + queue *ReplicationQueue + completed []*fsrep.Replication + active *ReplicationQueueItemHandle + + // PlanningError + planningError error + + // ContextDone + contextError error + + // PlanningError, WorkingWait + sleepUntil time.Time +} + +type Report struct { + Status string + Problem string + SleepUntil time.Time + Completed []*fsrep.Report + Pending []*fsrep.Report + Active *fsrep.Report +} + +func NewReplication(secsPerState *prometheus.HistogramVec, bytesReplicated *prometheus.CounterVec) *Replication { + r := Replication{ + promSecsPerState: secsPerState, + promBytesReplicated: bytesReplicated, + state: Planning, + } + return &r +} + +// Endpoint represents one side of the replication. +// +// An endpoint is either in Sender or Receiver mode, represented by the correspondingly +// named interfaces defined in this package. +type Endpoint interface { + // Does not include placeholder filesystems + ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error) + // FIXME document FilteredError handling + ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) // fix depS + DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) +} + +type Sender interface { + Endpoint + fsrep.Sender +} + +type Receiver interface { + Endpoint + fsrep.Receiver +} + +type FilteredError struct{ fs string } + +func NewFilteredError(fs string) *FilteredError { + return &FilteredError{fs} +} + +func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs } + +type updater func(func(*Replication)) (newState State) +type state func(ctx context.Context, sender Sender, receiver Receiver, u updater) state + +// Drive starts the state machine and returns only after replication has finished (with or without errors). +// The Logger in ctx is used for both debug and error logging, but is not guaranteed to be stable +// or end-user friendly. +// User-facing replication progress reports and can be obtained using the Report method, +// whose output will not change after Drive returns. +// +// FIXME: Drive may be only called once per instance of Replication +func (r *Replication) Drive(ctx context.Context, sender Sender, receiver Receiver) { + + var u updater = func(f func(*Replication)) State { + r.lock.Lock() + defer r.lock.Unlock() + if f != nil { + f(r) + } + return r.state + } + + var s state = statePlanning + var pre, post State + for s != nil { + preTime := time.Now() + pre = u(nil) + s = s(ctx, sender, receiver, u) + delta := time.Now().Sub(preTime) + r.promSecsPerState.WithLabelValues(pre.String()).Observe(delta.Seconds()) + post = u(nil) + getLogger(ctx). + WithField("transition", fmt.Sprintf("%s => %s", pre, post)). + WithField("duration", delta). + Debug("main state transition") + if post == Working && pre != post { + getLogger(ctx).Info("start working") + } + } + + getLogger(ctx). + WithField("final_state", post). + Debug("main final state") +} + +func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string) { + if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok { + if len(noCommonAncestor.SortedReceiverVersions) == 0 { + // TODO this is hard-coded replication policy: most recent snapshot as source + var mostRecentSnap *pdu.FilesystemVersion + for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- { + if noCommonAncestor.SortedSenderVersions[n].Type == pdu.FilesystemVersion_Snapshot { + mostRecentSnap = noCommonAncestor.SortedSenderVersions[n] + break + } + } + if mostRecentSnap == nil { + return nil, "no snapshots available on sender side" + } + return []*pdu.FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName()) + } + } + return nil, "no automated way to handle conflict type" +} + +var PlanningRetryInterval = 10 * time.Second // FIXME make constant onfigurable + +func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + + log := getLogger(ctx) + + log.Info("start planning") + + handlePlanningError := func(err error) state { + // FIXME classify error as temporary or permanent / max retry counter + return u(func(r *Replication) { + r.sleepUntil = time.Now().Add(PlanningRetryInterval) + r.planningError = err + r.state = PlanningError + }).rsf() + } + + sfss, err := sender.ListFilesystems(ctx) + if err != nil { + log.WithError(err).Error("error listing sender filesystems") + return handlePlanningError(err) + } + + rfss, err := receiver.ListFilesystems(ctx) + if err != nil { + log.WithError(err).Error("error listing receiver filesystems") + return handlePlanningError(err) + } + + q := NewReplicationQueue() + mainlog := log + for _, fs := range sfss { + + log := mainlog.WithField("filesystem", fs.Path) + + log.Debug("assessing filesystem") + + sfsvs, err := sender.ListFilesystemVersions(ctx, fs.Path) + if err != nil { + log.WithError(err).Error("cannot get remote filesystem versions") + return handlePlanningError(err) + } + + if len(sfsvs) < 1 { + err := errors.New("sender does not have any versions") + log.Error(err.Error()) + q.Add(fsrep.NewReplicationWithPermanentError(fs.Path, err)) + continue + } + + receiverFSExists := false + for _, rfs := range rfss { + if rfs.Path == fs.Path { + receiverFSExists = true + } + } + + var rfsvs []*pdu.FilesystemVersion + if receiverFSExists { + rfsvs, err = receiver.ListFilesystemVersions(ctx, fs.Path) + if err != nil { + if _, ok := err.(*FilteredError); ok { + log.Info("receiver ignores filesystem") + continue + } + log.WithError(err).Error("receiver error") + return handlePlanningError(err) + } + } else { + rfsvs = []*pdu.FilesystemVersion{} + } + + path, conflict := IncrementalPath(rfsvs, sfsvs) + if conflict != nil { + var msg string + path, msg = resolveConflict(conflict) // no shadowing allowed! + if path != nil { + log.WithField("conflict", conflict).Info("conflict") + log.WithField("resolution", msg).Info("automatically resolved") + } else { + log.WithField("conflict", conflict).Error("conflict") + log.WithField("problem", msg).Error("cannot resolve conflict") + } + } + if path == nil { + q.Add(fsrep.NewReplicationWithPermanentError(fs.Path, conflict)) + continue + } + + var promBytesReplicated *prometheus.CounterVec + u(func(replication *Replication) { // FIXME args struct like in pruner (also use for sender and receiver) + promBytesReplicated = replication.promBytesReplicated + }) + fsrfsm := fsrep.BuildReplication(fs.Path, promBytesReplicated.WithLabelValues(fs.Path)) + if len(path) == 1 { + fsrfsm.AddStep(nil, path[0]) + } else { + for i := 0; i < len(path)-1; i++ { + fsrfsm.AddStep(path[i], path[i+1]) + } + } + qitem := fsrfsm.Done() + + log.Debug("compute send size estimate") + if err = qitem.UpdateSizeEsitmate(ctx, sender); err != nil { + log.WithError(err).Error("error computing size estimate") + return handlePlanningError(err) + } + q.Add(qitem) + } + + return u(func(r *Replication) { + r.completed = nil + r.queue = q + r.planningError = nil + r.state = Working + }).rsf() +} + +func statePlanningError(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + var sleepUntil time.Time + u(func(r *Replication) { + sleepUntil = r.sleepUntil + }) + t := time.NewTimer(sleepUntil.Sub(time.Now())) + getLogger(ctx).WithField("until", sleepUntil).Info("retry wait after planning error") + defer t.Stop() + select { + case <-ctx.Done(): + return u(func(r *Replication) { + r.state = ContextDone + r.contextError = ctx.Err() + }).rsf() + case <-t.C: + case <-wakeup.Wait(ctx): + } + return u(func(r *Replication) { + r.state = Planning + }).rsf() +} + +func stateWorking(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + + var active *ReplicationQueueItemHandle + rsfNext := u(func(r *Replication) { + done, next := r.queue.GetNext() + r.completed = append(r.completed, done...) + if next == nil { + r.state = Completed + } + r.active = next + active = next + }).rsf() + + if active == nil { + return rsfNext + } + + retryWaitUntil := active.GetFSReplication().RetryWaitUntil() + if retryWaitUntil.After(time.Now()) { + return u(func(r *Replication) { + r.sleepUntil = retryWaitUntil + r.state = WorkingWait + }).rsf() + } + + state, nextStepDate, retryWaitUntil := active.GetFSReplication().TakeStep(ctx, sender, receiver) + return u(func(r *Replication) { + active.Update(state, nextStepDate, retryWaitUntil) + r.active = nil + }).rsf() +} + +func stateWorkingWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state { + var sleepUntil time.Time + u(func(r *Replication) { + sleepUntil = r.sleepUntil + }) + t := time.NewTimer(PlanningRetryInterval) + getLogger(ctx).WithField("until", sleepUntil).Info("retry wait because no filesystems are ready") + defer t.Stop() + select { + case <-ctx.Done(): + return u(func(r *Replication) { + r.state = ContextDone + r.contextError = ctx.Err() + }).rsf() + + case <-t.C: + case <-wakeup.Wait(ctx): + } + return u(func(r *Replication) { + r.state = Working + }).rsf() +} + +// Report provides a summary of the progress of the Replication, +// i.e., a condensed dump of the internal state machine. +// Report is safe to be called asynchronously while Drive is running. +func (r *Replication) Report() *Report { + r.lock.Lock() + defer r.lock.Unlock() + + rep := Report{ + Status: r.state.String(), + SleepUntil: r.sleepUntil, + } + + if r.state&(Planning|PlanningError|ContextDone) != 0 { + switch r.state { + case PlanningError: + rep.Problem = r.planningError.Error() + case ContextDone: + rep.Problem = r.contextError.Error() + } + return &rep + } + + rep.Pending = make([]*fsrep.Report, 0, r.queue.Len()) + rep.Completed = make([]*fsrep.Report, 0, len(r.completed)) // room for active (potentially) + + var active *fsrep.Replication + if r.active != nil { + active = r.active.GetFSReplication() + rep.Active = active.Report() + } + r.queue.Foreach(func(h *ReplicationQueueItemHandle) { + fsr := h.GetFSReplication() + if active != fsr { + rep.Pending = append(rep.Pending, fsr.Report()) + } + }) + for _, fsr := range r.completed { + rep.Completed = append(rep.Completed, fsr.Report()) + } + + return &rep +} diff --git a/replication/pdu/pdu.pb.go b/replication/pdu/pdu.pb.go new file mode 100644 index 0000000..1171e07 --- /dev/null +++ b/replication/pdu/pdu.pb.go @@ -0,0 +1,1134 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: pdu.proto + +package pdu + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type FilesystemVersion_VersionType int32 + +const ( + FilesystemVersion_Snapshot FilesystemVersion_VersionType = 0 + FilesystemVersion_Bookmark FilesystemVersion_VersionType = 1 +) + +var FilesystemVersion_VersionType_name = map[int32]string{ + 0: "Snapshot", + 1: "Bookmark", +} +var FilesystemVersion_VersionType_value = map[string]int32{ + "Snapshot": 0, + "Bookmark": 1, +} + +func (x FilesystemVersion_VersionType) String() string { + return proto.EnumName(FilesystemVersion_VersionType_name, int32(x)) +} +func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{5, 0} +} + +type ListFilesystemReq struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListFilesystemReq) Reset() { *m = ListFilesystemReq{} } +func (m *ListFilesystemReq) String() string { return proto.CompactTextString(m) } +func (*ListFilesystemReq) ProtoMessage() {} +func (*ListFilesystemReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{0} +} +func (m *ListFilesystemReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFilesystemReq.Unmarshal(m, b) +} +func (m *ListFilesystemReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFilesystemReq.Marshal(b, m, deterministic) +} +func (dst *ListFilesystemReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFilesystemReq.Merge(dst, src) +} +func (m *ListFilesystemReq) XXX_Size() int { + return xxx_messageInfo_ListFilesystemReq.Size(m) +} +func (m *ListFilesystemReq) XXX_DiscardUnknown() { + xxx_messageInfo_ListFilesystemReq.DiscardUnknown(m) +} + +var xxx_messageInfo_ListFilesystemReq proto.InternalMessageInfo + +type ListFilesystemRes struct { + Filesystems []*Filesystem `protobuf:"bytes,1,rep,name=Filesystems,proto3" json:"Filesystems,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListFilesystemRes) Reset() { *m = ListFilesystemRes{} } +func (m *ListFilesystemRes) String() string { return proto.CompactTextString(m) } +func (*ListFilesystemRes) ProtoMessage() {} +func (*ListFilesystemRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{1} +} +func (m *ListFilesystemRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFilesystemRes.Unmarshal(m, b) +} +func (m *ListFilesystemRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFilesystemRes.Marshal(b, m, deterministic) +} +func (dst *ListFilesystemRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFilesystemRes.Merge(dst, src) +} +func (m *ListFilesystemRes) XXX_Size() int { + return xxx_messageInfo_ListFilesystemRes.Size(m) +} +func (m *ListFilesystemRes) XXX_DiscardUnknown() { + xxx_messageInfo_ListFilesystemRes.DiscardUnknown(m) +} + +var xxx_messageInfo_ListFilesystemRes proto.InternalMessageInfo + +func (m *ListFilesystemRes) GetFilesystems() []*Filesystem { + if m != nil { + return m.Filesystems + } + return nil +} + +type Filesystem struct { + Path string `protobuf:"bytes,1,opt,name=Path,proto3" json:"Path,omitempty"` + ResumeToken string `protobuf:"bytes,2,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Filesystem) Reset() { *m = Filesystem{} } +func (m *Filesystem) String() string { return proto.CompactTextString(m) } +func (*Filesystem) ProtoMessage() {} +func (*Filesystem) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{2} +} +func (m *Filesystem) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Filesystem.Unmarshal(m, b) +} +func (m *Filesystem) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Filesystem.Marshal(b, m, deterministic) +} +func (dst *Filesystem) XXX_Merge(src proto.Message) { + xxx_messageInfo_Filesystem.Merge(dst, src) +} +func (m *Filesystem) XXX_Size() int { + return xxx_messageInfo_Filesystem.Size(m) +} +func (m *Filesystem) XXX_DiscardUnknown() { + xxx_messageInfo_Filesystem.DiscardUnknown(m) +} + +var xxx_messageInfo_Filesystem proto.InternalMessageInfo + +func (m *Filesystem) GetPath() string { + if m != nil { + return m.Path + } + return "" +} + +func (m *Filesystem) GetResumeToken() string { + if m != nil { + return m.ResumeToken + } + return "" +} + +type ListFilesystemVersionsReq struct { + Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListFilesystemVersionsReq) Reset() { *m = ListFilesystemVersionsReq{} } +func (m *ListFilesystemVersionsReq) String() string { return proto.CompactTextString(m) } +func (*ListFilesystemVersionsReq) ProtoMessage() {} +func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{3} +} +func (m *ListFilesystemVersionsReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFilesystemVersionsReq.Unmarshal(m, b) +} +func (m *ListFilesystemVersionsReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFilesystemVersionsReq.Marshal(b, m, deterministic) +} +func (dst *ListFilesystemVersionsReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFilesystemVersionsReq.Merge(dst, src) +} +func (m *ListFilesystemVersionsReq) XXX_Size() int { + return xxx_messageInfo_ListFilesystemVersionsReq.Size(m) +} +func (m *ListFilesystemVersionsReq) XXX_DiscardUnknown() { + xxx_messageInfo_ListFilesystemVersionsReq.DiscardUnknown(m) +} + +var xxx_messageInfo_ListFilesystemVersionsReq proto.InternalMessageInfo + +func (m *ListFilesystemVersionsReq) GetFilesystem() string { + if m != nil { + return m.Filesystem + } + return "" +} + +type ListFilesystemVersionsRes struct { + Versions []*FilesystemVersion `protobuf:"bytes,1,rep,name=Versions,proto3" json:"Versions,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListFilesystemVersionsRes) Reset() { *m = ListFilesystemVersionsRes{} } +func (m *ListFilesystemVersionsRes) String() string { return proto.CompactTextString(m) } +func (*ListFilesystemVersionsRes) ProtoMessage() {} +func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{4} +} +func (m *ListFilesystemVersionsRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListFilesystemVersionsRes.Unmarshal(m, b) +} +func (m *ListFilesystemVersionsRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListFilesystemVersionsRes.Marshal(b, m, deterministic) +} +func (dst *ListFilesystemVersionsRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListFilesystemVersionsRes.Merge(dst, src) +} +func (m *ListFilesystemVersionsRes) XXX_Size() int { + return xxx_messageInfo_ListFilesystemVersionsRes.Size(m) +} +func (m *ListFilesystemVersionsRes) XXX_DiscardUnknown() { + xxx_messageInfo_ListFilesystemVersionsRes.DiscardUnknown(m) +} + +var xxx_messageInfo_ListFilesystemVersionsRes proto.InternalMessageInfo + +func (m *ListFilesystemVersionsRes) GetVersions() []*FilesystemVersion { + if m != nil { + return m.Versions + } + return nil +} + +type FilesystemVersion struct { + Type FilesystemVersion_VersionType `protobuf:"varint,1,opt,name=Type,proto3,enum=pdu.FilesystemVersion_VersionType" json:"Type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` + Guid uint64 `protobuf:"varint,3,opt,name=Guid,proto3" json:"Guid,omitempty"` + CreateTXG uint64 `protobuf:"varint,4,opt,name=CreateTXG,proto3" json:"CreateTXG,omitempty"` + Creation string `protobuf:"bytes,5,opt,name=Creation,proto3" json:"Creation,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FilesystemVersion) Reset() { *m = FilesystemVersion{} } +func (m *FilesystemVersion) String() string { return proto.CompactTextString(m) } +func (*FilesystemVersion) ProtoMessage() {} +func (*FilesystemVersion) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{5} +} +func (m *FilesystemVersion) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FilesystemVersion.Unmarshal(m, b) +} +func (m *FilesystemVersion) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FilesystemVersion.Marshal(b, m, deterministic) +} +func (dst *FilesystemVersion) XXX_Merge(src proto.Message) { + xxx_messageInfo_FilesystemVersion.Merge(dst, src) +} +func (m *FilesystemVersion) XXX_Size() int { + return xxx_messageInfo_FilesystemVersion.Size(m) +} +func (m *FilesystemVersion) XXX_DiscardUnknown() { + xxx_messageInfo_FilesystemVersion.DiscardUnknown(m) +} + +var xxx_messageInfo_FilesystemVersion proto.InternalMessageInfo + +func (m *FilesystemVersion) GetType() FilesystemVersion_VersionType { + if m != nil { + return m.Type + } + return FilesystemVersion_Snapshot +} + +func (m *FilesystemVersion) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *FilesystemVersion) GetGuid() uint64 { + if m != nil { + return m.Guid + } + return 0 +} + +func (m *FilesystemVersion) GetCreateTXG() uint64 { + if m != nil { + return m.CreateTXG + } + return 0 +} + +func (m *FilesystemVersion) GetCreation() string { + if m != nil { + return m.Creation + } + return "" +} + +type SendReq struct { + Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"` + From string `protobuf:"bytes,2,opt,name=From,proto3" json:"From,omitempty"` + // May be empty / null to request a full transfer of From + To string `protobuf:"bytes,3,opt,name=To,proto3" json:"To,omitempty"` + // If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender. + // The sender MUST indicate in SendRes.UsedResumeToken + // If it does not work, the sender SHOULD clear the resume token on their side + // and use From and To instead + // If ResumeToken is not empty, the GUIDs of From and To + // MUST correspond to those encoded in the ResumeToken. + // Otherwise, the Sender MUST return an error. + ResumeToken string `protobuf:"bytes,4,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"` + Compress bool `protobuf:"varint,5,opt,name=Compress,proto3" json:"Compress,omitempty"` + Dedup bool `protobuf:"varint,6,opt,name=Dedup,proto3" json:"Dedup,omitempty"` + DryRun bool `protobuf:"varint,7,opt,name=DryRun,proto3" json:"DryRun,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SendReq) Reset() { *m = SendReq{} } +func (m *SendReq) String() string { return proto.CompactTextString(m) } +func (*SendReq) ProtoMessage() {} +func (*SendReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{6} +} +func (m *SendReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SendReq.Unmarshal(m, b) +} +func (m *SendReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SendReq.Marshal(b, m, deterministic) +} +func (dst *SendReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_SendReq.Merge(dst, src) +} +func (m *SendReq) XXX_Size() int { + return xxx_messageInfo_SendReq.Size(m) +} +func (m *SendReq) XXX_DiscardUnknown() { + xxx_messageInfo_SendReq.DiscardUnknown(m) +} + +var xxx_messageInfo_SendReq proto.InternalMessageInfo + +func (m *SendReq) GetFilesystem() string { + if m != nil { + return m.Filesystem + } + return "" +} + +func (m *SendReq) GetFrom() string { + if m != nil { + return m.From + } + return "" +} + +func (m *SendReq) GetTo() string { + if m != nil { + return m.To + } + return "" +} + +func (m *SendReq) GetResumeToken() string { + if m != nil { + return m.ResumeToken + } + return "" +} + +func (m *SendReq) GetCompress() bool { + if m != nil { + return m.Compress + } + return false +} + +func (m *SendReq) GetDedup() bool { + if m != nil { + return m.Dedup + } + return false +} + +func (m *SendReq) GetDryRun() bool { + if m != nil { + return m.DryRun + } + return false +} + +type Property struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Property) Reset() { *m = Property{} } +func (m *Property) String() string { return proto.CompactTextString(m) } +func (*Property) ProtoMessage() {} +func (*Property) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{7} +} +func (m *Property) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Property.Unmarshal(m, b) +} +func (m *Property) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Property.Marshal(b, m, deterministic) +} +func (dst *Property) XXX_Merge(src proto.Message) { + xxx_messageInfo_Property.Merge(dst, src) +} +func (m *Property) XXX_Size() int { + return xxx_messageInfo_Property.Size(m) +} +func (m *Property) XXX_DiscardUnknown() { + xxx_messageInfo_Property.DiscardUnknown(m) +} + +var xxx_messageInfo_Property proto.InternalMessageInfo + +func (m *Property) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Property) GetValue() string { + if m != nil { + return m.Value + } + return "" +} + +type SendRes struct { + // Whether the resume token provided in the request has been used or not. + UsedResumeToken bool `protobuf:"varint,1,opt,name=UsedResumeToken,proto3" json:"UsedResumeToken,omitempty"` + // Expected stream size determined by dry run, not exact. + // 0 indicates that for the given SendReq, no size estimate could be made. + ExpectedSize int64 `protobuf:"varint,2,opt,name=ExpectedSize,proto3" json:"ExpectedSize,omitempty"` + Properties []*Property `protobuf:"bytes,3,rep,name=Properties,proto3" json:"Properties,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SendRes) Reset() { *m = SendRes{} } +func (m *SendRes) String() string { return proto.CompactTextString(m) } +func (*SendRes) ProtoMessage() {} +func (*SendRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{8} +} +func (m *SendRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SendRes.Unmarshal(m, b) +} +func (m *SendRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SendRes.Marshal(b, m, deterministic) +} +func (dst *SendRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_SendRes.Merge(dst, src) +} +func (m *SendRes) XXX_Size() int { + return xxx_messageInfo_SendRes.Size(m) +} +func (m *SendRes) XXX_DiscardUnknown() { + xxx_messageInfo_SendRes.DiscardUnknown(m) +} + +var xxx_messageInfo_SendRes proto.InternalMessageInfo + +func (m *SendRes) GetUsedResumeToken() bool { + if m != nil { + return m.UsedResumeToken + } + return false +} + +func (m *SendRes) GetExpectedSize() int64 { + if m != nil { + return m.ExpectedSize + } + return 0 +} + +func (m *SendRes) GetProperties() []*Property { + if m != nil { + return m.Properties + } + return nil +} + +type ReceiveReq struct { + Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"` + // If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request + ClearResumeToken bool `protobuf:"varint,2,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReceiveReq) Reset() { *m = ReceiveReq{} } +func (m *ReceiveReq) String() string { return proto.CompactTextString(m) } +func (*ReceiveReq) ProtoMessage() {} +func (*ReceiveReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{9} +} +func (m *ReceiveReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReceiveReq.Unmarshal(m, b) +} +func (m *ReceiveReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReceiveReq.Marshal(b, m, deterministic) +} +func (dst *ReceiveReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReceiveReq.Merge(dst, src) +} +func (m *ReceiveReq) XXX_Size() int { + return xxx_messageInfo_ReceiveReq.Size(m) +} +func (m *ReceiveReq) XXX_DiscardUnknown() { + xxx_messageInfo_ReceiveReq.DiscardUnknown(m) +} + +var xxx_messageInfo_ReceiveReq proto.InternalMessageInfo + +func (m *ReceiveReq) GetFilesystem() string { + if m != nil { + return m.Filesystem + } + return "" +} + +func (m *ReceiveReq) GetClearResumeToken() bool { + if m != nil { + return m.ClearResumeToken + } + return false +} + +type ReceiveRes struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReceiveRes) Reset() { *m = ReceiveRes{} } +func (m *ReceiveRes) String() string { return proto.CompactTextString(m) } +func (*ReceiveRes) ProtoMessage() {} +func (*ReceiveRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{10} +} +func (m *ReceiveRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReceiveRes.Unmarshal(m, b) +} +func (m *ReceiveRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReceiveRes.Marshal(b, m, deterministic) +} +func (dst *ReceiveRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReceiveRes.Merge(dst, src) +} +func (m *ReceiveRes) XXX_Size() int { + return xxx_messageInfo_ReceiveRes.Size(m) +} +func (m *ReceiveRes) XXX_DiscardUnknown() { + xxx_messageInfo_ReceiveRes.DiscardUnknown(m) +} + +var xxx_messageInfo_ReceiveRes proto.InternalMessageInfo + +type DestroySnapshotsReq struct { + Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"` + // Path to filesystem, snapshot or bookmark to be destroyed + Snapshots []*FilesystemVersion `protobuf:"bytes,2,rep,name=Snapshots,proto3" json:"Snapshots,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DestroySnapshotsReq) Reset() { *m = DestroySnapshotsReq{} } +func (m *DestroySnapshotsReq) String() string { return proto.CompactTextString(m) } +func (*DestroySnapshotsReq) ProtoMessage() {} +func (*DestroySnapshotsReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{11} +} +func (m *DestroySnapshotsReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DestroySnapshotsReq.Unmarshal(m, b) +} +func (m *DestroySnapshotsReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DestroySnapshotsReq.Marshal(b, m, deterministic) +} +func (dst *DestroySnapshotsReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_DestroySnapshotsReq.Merge(dst, src) +} +func (m *DestroySnapshotsReq) XXX_Size() int { + return xxx_messageInfo_DestroySnapshotsReq.Size(m) +} +func (m *DestroySnapshotsReq) XXX_DiscardUnknown() { + xxx_messageInfo_DestroySnapshotsReq.DiscardUnknown(m) +} + +var xxx_messageInfo_DestroySnapshotsReq proto.InternalMessageInfo + +func (m *DestroySnapshotsReq) GetFilesystem() string { + if m != nil { + return m.Filesystem + } + return "" +} + +func (m *DestroySnapshotsReq) GetSnapshots() []*FilesystemVersion { + if m != nil { + return m.Snapshots + } + return nil +} + +type DestroySnapshotRes struct { + Snapshot *FilesystemVersion `protobuf:"bytes,1,opt,name=Snapshot,proto3" json:"Snapshot,omitempty"` + Error string `protobuf:"bytes,2,opt,name=Error,proto3" json:"Error,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DestroySnapshotRes) Reset() { *m = DestroySnapshotRes{} } +func (m *DestroySnapshotRes) String() string { return proto.CompactTextString(m) } +func (*DestroySnapshotRes) ProtoMessage() {} +func (*DestroySnapshotRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{12} +} +func (m *DestroySnapshotRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DestroySnapshotRes.Unmarshal(m, b) +} +func (m *DestroySnapshotRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DestroySnapshotRes.Marshal(b, m, deterministic) +} +func (dst *DestroySnapshotRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DestroySnapshotRes.Merge(dst, src) +} +func (m *DestroySnapshotRes) XXX_Size() int { + return xxx_messageInfo_DestroySnapshotRes.Size(m) +} +func (m *DestroySnapshotRes) XXX_DiscardUnknown() { + xxx_messageInfo_DestroySnapshotRes.DiscardUnknown(m) +} + +var xxx_messageInfo_DestroySnapshotRes proto.InternalMessageInfo + +func (m *DestroySnapshotRes) GetSnapshot() *FilesystemVersion { + if m != nil { + return m.Snapshot + } + return nil +} + +func (m *DestroySnapshotRes) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +type DestroySnapshotsRes struct { + Results []*DestroySnapshotRes `protobuf:"bytes,1,rep,name=Results,proto3" json:"Results,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DestroySnapshotsRes) Reset() { *m = DestroySnapshotsRes{} } +func (m *DestroySnapshotsRes) String() string { return proto.CompactTextString(m) } +func (*DestroySnapshotsRes) ProtoMessage() {} +func (*DestroySnapshotsRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{13} +} +func (m *DestroySnapshotsRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DestroySnapshotsRes.Unmarshal(m, b) +} +func (m *DestroySnapshotsRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DestroySnapshotsRes.Marshal(b, m, deterministic) +} +func (dst *DestroySnapshotsRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DestroySnapshotsRes.Merge(dst, src) +} +func (m *DestroySnapshotsRes) XXX_Size() int { + return xxx_messageInfo_DestroySnapshotsRes.Size(m) +} +func (m *DestroySnapshotsRes) XXX_DiscardUnknown() { + xxx_messageInfo_DestroySnapshotsRes.DiscardUnknown(m) +} + +var xxx_messageInfo_DestroySnapshotsRes proto.InternalMessageInfo + +func (m *DestroySnapshotsRes) GetResults() []*DestroySnapshotRes { + if m != nil { + return m.Results + } + return nil +} + +type ReplicationCursorReq struct { + Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"` + // Types that are valid to be assigned to Op: + // *ReplicationCursorReq_Get + // *ReplicationCursorReq_Set + Op isReplicationCursorReq_Op `protobuf_oneof:"op"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReplicationCursorReq) Reset() { *m = ReplicationCursorReq{} } +func (m *ReplicationCursorReq) String() string { return proto.CompactTextString(m) } +func (*ReplicationCursorReq) ProtoMessage() {} +func (*ReplicationCursorReq) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{14} +} +func (m *ReplicationCursorReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReplicationCursorReq.Unmarshal(m, b) +} +func (m *ReplicationCursorReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReplicationCursorReq.Marshal(b, m, deterministic) +} +func (dst *ReplicationCursorReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReplicationCursorReq.Merge(dst, src) +} +func (m *ReplicationCursorReq) XXX_Size() int { + return xxx_messageInfo_ReplicationCursorReq.Size(m) +} +func (m *ReplicationCursorReq) XXX_DiscardUnknown() { + xxx_messageInfo_ReplicationCursorReq.DiscardUnknown(m) +} + +var xxx_messageInfo_ReplicationCursorReq proto.InternalMessageInfo + +func (m *ReplicationCursorReq) GetFilesystem() string { + if m != nil { + return m.Filesystem + } + return "" +} + +type isReplicationCursorReq_Op interface { + isReplicationCursorReq_Op() +} + +type ReplicationCursorReq_Get struct { + Get *ReplicationCursorReq_GetOp `protobuf:"bytes,2,opt,name=get,proto3,oneof"` +} + +type ReplicationCursorReq_Set struct { + Set *ReplicationCursorReq_SetOp `protobuf:"bytes,3,opt,name=set,proto3,oneof"` +} + +func (*ReplicationCursorReq_Get) isReplicationCursorReq_Op() {} + +func (*ReplicationCursorReq_Set) isReplicationCursorReq_Op() {} + +func (m *ReplicationCursorReq) GetOp() isReplicationCursorReq_Op { + if m != nil { + return m.Op + } + return nil +} + +func (m *ReplicationCursorReq) GetGet() *ReplicationCursorReq_GetOp { + if x, ok := m.GetOp().(*ReplicationCursorReq_Get); ok { + return x.Get + } + return nil +} + +func (m *ReplicationCursorReq) GetSet() *ReplicationCursorReq_SetOp { + if x, ok := m.GetOp().(*ReplicationCursorReq_Set); ok { + return x.Set + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ReplicationCursorReq) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ReplicationCursorReq_OneofMarshaler, _ReplicationCursorReq_OneofUnmarshaler, _ReplicationCursorReq_OneofSizer, []interface{}{ + (*ReplicationCursorReq_Get)(nil), + (*ReplicationCursorReq_Set)(nil), + } +} + +func _ReplicationCursorReq_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ReplicationCursorReq) + // op + switch x := m.Op.(type) { + case *ReplicationCursorReq_Get: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Get); err != nil { + return err + } + case *ReplicationCursorReq_Set: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Set); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("ReplicationCursorReq.Op has unexpected type %T", x) + } + return nil +} + +func _ReplicationCursorReq_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ReplicationCursorReq) + switch tag { + case 2: // op.get + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ReplicationCursorReq_GetOp) + err := b.DecodeMessage(msg) + m.Op = &ReplicationCursorReq_Get{msg} + return true, err + case 3: // op.set + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ReplicationCursorReq_SetOp) + err := b.DecodeMessage(msg) + m.Op = &ReplicationCursorReq_Set{msg} + return true, err + default: + return false, nil + } +} + +func _ReplicationCursorReq_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ReplicationCursorReq) + // op + switch x := m.Op.(type) { + case *ReplicationCursorReq_Get: + s := proto.Size(x.Get) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ReplicationCursorReq_Set: + s := proto.Size(x.Set) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type ReplicationCursorReq_GetOp struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReplicationCursorReq_GetOp) Reset() { *m = ReplicationCursorReq_GetOp{} } +func (m *ReplicationCursorReq_GetOp) String() string { return proto.CompactTextString(m) } +func (*ReplicationCursorReq_GetOp) ProtoMessage() {} +func (*ReplicationCursorReq_GetOp) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{14, 0} +} +func (m *ReplicationCursorReq_GetOp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReplicationCursorReq_GetOp.Unmarshal(m, b) +} +func (m *ReplicationCursorReq_GetOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReplicationCursorReq_GetOp.Marshal(b, m, deterministic) +} +func (dst *ReplicationCursorReq_GetOp) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReplicationCursorReq_GetOp.Merge(dst, src) +} +func (m *ReplicationCursorReq_GetOp) XXX_Size() int { + return xxx_messageInfo_ReplicationCursorReq_GetOp.Size(m) +} +func (m *ReplicationCursorReq_GetOp) XXX_DiscardUnknown() { + xxx_messageInfo_ReplicationCursorReq_GetOp.DiscardUnknown(m) +} + +var xxx_messageInfo_ReplicationCursorReq_GetOp proto.InternalMessageInfo + +type ReplicationCursorReq_SetOp struct { + Snapshot string `protobuf:"bytes,2,opt,name=Snapshot,proto3" json:"Snapshot,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReplicationCursorReq_SetOp) Reset() { *m = ReplicationCursorReq_SetOp{} } +func (m *ReplicationCursorReq_SetOp) String() string { return proto.CompactTextString(m) } +func (*ReplicationCursorReq_SetOp) ProtoMessage() {} +func (*ReplicationCursorReq_SetOp) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{14, 1} +} +func (m *ReplicationCursorReq_SetOp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReplicationCursorReq_SetOp.Unmarshal(m, b) +} +func (m *ReplicationCursorReq_SetOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReplicationCursorReq_SetOp.Marshal(b, m, deterministic) +} +func (dst *ReplicationCursorReq_SetOp) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReplicationCursorReq_SetOp.Merge(dst, src) +} +func (m *ReplicationCursorReq_SetOp) XXX_Size() int { + return xxx_messageInfo_ReplicationCursorReq_SetOp.Size(m) +} +func (m *ReplicationCursorReq_SetOp) XXX_DiscardUnknown() { + xxx_messageInfo_ReplicationCursorReq_SetOp.DiscardUnknown(m) +} + +var xxx_messageInfo_ReplicationCursorReq_SetOp proto.InternalMessageInfo + +func (m *ReplicationCursorReq_SetOp) GetSnapshot() string { + if m != nil { + return m.Snapshot + } + return "" +} + +type ReplicationCursorRes struct { + // Types that are valid to be assigned to Result: + // *ReplicationCursorRes_Guid + // *ReplicationCursorRes_Error + Result isReplicationCursorRes_Result `protobuf_oneof:"Result"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReplicationCursorRes) Reset() { *m = ReplicationCursorRes{} } +func (m *ReplicationCursorRes) String() string { return proto.CompactTextString(m) } +func (*ReplicationCursorRes) ProtoMessage() {} +func (*ReplicationCursorRes) Descriptor() ([]byte, []int) { + return fileDescriptor_pdu_cbdc4740ab26577c, []int{15} +} +func (m *ReplicationCursorRes) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReplicationCursorRes.Unmarshal(m, b) +} +func (m *ReplicationCursorRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReplicationCursorRes.Marshal(b, m, deterministic) +} +func (dst *ReplicationCursorRes) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReplicationCursorRes.Merge(dst, src) +} +func (m *ReplicationCursorRes) XXX_Size() int { + return xxx_messageInfo_ReplicationCursorRes.Size(m) +} +func (m *ReplicationCursorRes) XXX_DiscardUnknown() { + xxx_messageInfo_ReplicationCursorRes.DiscardUnknown(m) +} + +var xxx_messageInfo_ReplicationCursorRes proto.InternalMessageInfo + +type isReplicationCursorRes_Result interface { + isReplicationCursorRes_Result() +} + +type ReplicationCursorRes_Guid struct { + Guid uint64 `protobuf:"varint,1,opt,name=Guid,proto3,oneof"` +} + +type ReplicationCursorRes_Error struct { + Error string `protobuf:"bytes,2,opt,name=Error,proto3,oneof"` +} + +func (*ReplicationCursorRes_Guid) isReplicationCursorRes_Result() {} + +func (*ReplicationCursorRes_Error) isReplicationCursorRes_Result() {} + +func (m *ReplicationCursorRes) GetResult() isReplicationCursorRes_Result { + if m != nil { + return m.Result + } + return nil +} + +func (m *ReplicationCursorRes) GetGuid() uint64 { + if x, ok := m.GetResult().(*ReplicationCursorRes_Guid); ok { + return x.Guid + } + return 0 +} + +func (m *ReplicationCursorRes) GetError() string { + if x, ok := m.GetResult().(*ReplicationCursorRes_Error); ok { + return x.Error + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ReplicationCursorRes) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ReplicationCursorRes_OneofMarshaler, _ReplicationCursorRes_OneofUnmarshaler, _ReplicationCursorRes_OneofSizer, []interface{}{ + (*ReplicationCursorRes_Guid)(nil), + (*ReplicationCursorRes_Error)(nil), + } +} + +func _ReplicationCursorRes_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ReplicationCursorRes) + // Result + switch x := m.Result.(type) { + case *ReplicationCursorRes_Guid: + b.EncodeVarint(1<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Guid)) + case *ReplicationCursorRes_Error: + b.EncodeVarint(2<<3 | proto.WireBytes) + b.EncodeStringBytes(x.Error) + case nil: + default: + return fmt.Errorf("ReplicationCursorRes.Result has unexpected type %T", x) + } + return nil +} + +func _ReplicationCursorRes_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ReplicationCursorRes) + switch tag { + case 1: // Result.Guid + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Result = &ReplicationCursorRes_Guid{x} + return true, err + case 2: // Result.Error + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Result = &ReplicationCursorRes_Error{x} + return true, err + default: + return false, nil + } +} + +func _ReplicationCursorRes_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ReplicationCursorRes) + // Result + switch x := m.Result.(type) { + case *ReplicationCursorRes_Guid: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Guid)) + case *ReplicationCursorRes_Error: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.Error))) + n += len(x.Error) + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +func init() { + proto.RegisterType((*ListFilesystemReq)(nil), "pdu.ListFilesystemReq") + proto.RegisterType((*ListFilesystemRes)(nil), "pdu.ListFilesystemRes") + proto.RegisterType((*Filesystem)(nil), "pdu.Filesystem") + proto.RegisterType((*ListFilesystemVersionsReq)(nil), "pdu.ListFilesystemVersionsReq") + proto.RegisterType((*ListFilesystemVersionsRes)(nil), "pdu.ListFilesystemVersionsRes") + proto.RegisterType((*FilesystemVersion)(nil), "pdu.FilesystemVersion") + proto.RegisterType((*SendReq)(nil), "pdu.SendReq") + proto.RegisterType((*Property)(nil), "pdu.Property") + proto.RegisterType((*SendRes)(nil), "pdu.SendRes") + proto.RegisterType((*ReceiveReq)(nil), "pdu.ReceiveReq") + proto.RegisterType((*ReceiveRes)(nil), "pdu.ReceiveRes") + proto.RegisterType((*DestroySnapshotsReq)(nil), "pdu.DestroySnapshotsReq") + proto.RegisterType((*DestroySnapshotRes)(nil), "pdu.DestroySnapshotRes") + proto.RegisterType((*DestroySnapshotsRes)(nil), "pdu.DestroySnapshotsRes") + proto.RegisterType((*ReplicationCursorReq)(nil), "pdu.ReplicationCursorReq") + proto.RegisterType((*ReplicationCursorReq_GetOp)(nil), "pdu.ReplicationCursorReq.GetOp") + proto.RegisterType((*ReplicationCursorReq_SetOp)(nil), "pdu.ReplicationCursorReq.SetOp") + proto.RegisterType((*ReplicationCursorRes)(nil), "pdu.ReplicationCursorRes") + proto.RegisterEnum("pdu.FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value) +} + +func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_cbdc4740ab26577c) } + +var fileDescriptor_pdu_cbdc4740ab26577c = []byte{ + // 657 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x55, 0xcb, 0x6e, 0xdb, 0x3a, + 0x10, 0xb5, 0x6c, 0xd9, 0x96, 0xc7, 0xb9, 0x79, 0x30, 0x41, 0xae, 0x6e, 0x70, 0x71, 0xaf, 0xc1, + 0x6e, 0xdc, 0x02, 0x35, 0x50, 0x27, 0xe8, 0xa6, 0x3b, 0xe7, 0xe5, 0x45, 0x91, 0x04, 0xb4, 0x1b, + 0x64, 0x55, 0x40, 0x8d, 0x06, 0x8d, 0xe0, 0x07, 0x15, 0x92, 0x2a, 0xea, 0x7e, 0x40, 0xff, 0xa9, + 0xff, 0xd1, 0x45, 0x3f, 0xa7, 0xe0, 0x58, 0x92, 0x15, 0xdb, 0x0d, 0xbc, 0x32, 0xcf, 0xf0, 0x70, + 0xe6, 0xcc, 0xa1, 0x86, 0x86, 0x46, 0x1c, 0x26, 0x9d, 0x58, 0x49, 0x23, 0x59, 0x25, 0x0e, 0x13, + 0xbe, 0x0f, 0x7b, 0xef, 0x23, 0x6d, 0x2e, 0xa2, 0x31, 0xea, 0x99, 0x36, 0x38, 0x11, 0xf8, 0xc8, + 0x2f, 0x56, 0x83, 0x9a, 0xbd, 0x81, 0xe6, 0x22, 0xa0, 0x7d, 0xa7, 0x55, 0x69, 0x37, 0xbb, 0x3b, + 0x1d, 0x9b, 0xaf, 0x40, 0x2c, 0x72, 0x78, 0x0f, 0x60, 0x01, 0x19, 0x03, 0xf7, 0x26, 0x30, 0x0f, + 0xbe, 0xd3, 0x72, 0xda, 0x0d, 0x41, 0x6b, 0xd6, 0x82, 0xa6, 0x40, 0x9d, 0x4c, 0x70, 0x28, 0x47, + 0x38, 0xf5, 0xcb, 0xb4, 0x55, 0x0c, 0xf1, 0x77, 0xf0, 0xcf, 0x53, 0x2d, 0xb7, 0xa8, 0x74, 0x24, + 0xa7, 0x5a, 0xe0, 0x23, 0xfb, 0xaf, 0x58, 0x20, 0x4d, 0x5c, 0x88, 0xf0, 0xeb, 0x3f, 0x1f, 0xd6, + 0xac, 0x0b, 0x5e, 0x06, 0xd3, 0x6e, 0x0e, 0x97, 0xba, 0x49, 0xb7, 0x45, 0xce, 0xe3, 0xbf, 0x1c, + 0xd8, 0x5b, 0xd9, 0x67, 0x6f, 0xc1, 0x1d, 0xce, 0x62, 0x24, 0x01, 0xdb, 0x5d, 0xbe, 0x3e, 0x4b, + 0x27, 0xfd, 0xb5, 0x4c, 0x41, 0x7c, 0xeb, 0xc8, 0x55, 0x30, 0xc1, 0xb4, 0x6d, 0x5a, 0xdb, 0xd8, + 0x65, 0x12, 0x85, 0x7e, 0xa5, 0xe5, 0xb4, 0x5d, 0x41, 0x6b, 0xf6, 0x2f, 0x34, 0x4e, 0x15, 0x06, + 0x06, 0x87, 0x77, 0x97, 0xbe, 0x4b, 0x1b, 0x8b, 0x00, 0x3b, 0x02, 0x8f, 0x40, 0x24, 0xa7, 0x7e, + 0x95, 0x32, 0xe5, 0x98, 0xbf, 0x84, 0x66, 0xa1, 0x2c, 0xdb, 0x02, 0x6f, 0x30, 0x0d, 0x62, 0xfd, + 0x20, 0xcd, 0x6e, 0xc9, 0xa2, 0x9e, 0x94, 0xa3, 0x49, 0xa0, 0x46, 0xbb, 0x0e, 0xff, 0xe1, 0x40, + 0x7d, 0x80, 0xd3, 0x70, 0x03, 0x5f, 0xad, 0xc8, 0x0b, 0x25, 0x27, 0x99, 0x70, 0xbb, 0x66, 0xdb, + 0x50, 0x1e, 0x4a, 0x92, 0xdd, 0x10, 0xe5, 0xa1, 0x5c, 0xbe, 0x5a, 0x77, 0xe5, 0x6a, 0x49, 0xb8, + 0x9c, 0xc4, 0x0a, 0xb5, 0x26, 0xe1, 0x9e, 0xc8, 0x31, 0x3b, 0x80, 0xea, 0x19, 0x86, 0x49, 0xec, + 0xd7, 0x68, 0x63, 0x0e, 0xd8, 0x21, 0xd4, 0xce, 0xd4, 0x4c, 0x24, 0x53, 0xbf, 0x4e, 0xe1, 0x14, + 0xf1, 0x13, 0xf0, 0x6e, 0x94, 0x8c, 0x51, 0x99, 0x59, 0x6e, 0xaa, 0x53, 0x30, 0xf5, 0x00, 0xaa, + 0xb7, 0xc1, 0x38, 0xc9, 0x9c, 0x9e, 0x03, 0xfe, 0x3d, 0xef, 0x58, 0xb3, 0x36, 0xec, 0x7c, 0xd0, + 0x18, 0x16, 0x15, 0x3b, 0x54, 0x62, 0x39, 0xcc, 0x38, 0x6c, 0x9d, 0x7f, 0x8d, 0xf1, 0xde, 0x60, + 0x38, 0x88, 0xbe, 0xcd, 0x53, 0x56, 0xc4, 0x93, 0x18, 0x7b, 0x0d, 0x90, 0xea, 0x89, 0x50, 0xfb, + 0x15, 0xfa, 0xb8, 0xfe, 0xa2, 0xcf, 0x22, 0x93, 0x29, 0x0a, 0x04, 0x7e, 0x07, 0x20, 0xf0, 0x1e, + 0xa3, 0x2f, 0xb8, 0x89, 0xf9, 0xaf, 0x60, 0xf7, 0x74, 0x8c, 0x81, 0x5a, 0x1e, 0x1c, 0x4f, 0xac, + 0xc4, 0xf9, 0x56, 0x21, 0xb3, 0xe6, 0x23, 0xd8, 0x3f, 0x43, 0x6d, 0x94, 0x9c, 0x65, 0x5f, 0xc1, + 0x26, 0x53, 0xc4, 0x4e, 0xa0, 0x91, 0xf3, 0xfd, 0xf2, 0xb3, 0x93, 0xb2, 0x20, 0xf2, 0x8f, 0xc0, + 0x96, 0x8a, 0xa5, 0x43, 0x97, 0x41, 0xaa, 0xf4, 0xcc, 0xd0, 0x65, 0x3c, 0x7b, 0x7b, 0xe7, 0x4a, + 0x49, 0x95, 0xdd, 0x1e, 0x01, 0xde, 0x5f, 0xd7, 0x8c, 0x7d, 0xa6, 0xea, 0xd6, 0x80, 0xb1, 0xc9, + 0x86, 0xfa, 0x6f, 0xca, 0xbf, 0x2a, 0x45, 0x64, 0x3c, 0xfe, 0xd3, 0x81, 0x03, 0x81, 0xf1, 0x38, + 0xba, 0xa7, 0xa1, 0x39, 0x4d, 0x94, 0x96, 0x6a, 0x13, 0x63, 0x8e, 0xa1, 0xf2, 0x19, 0x0d, 0xc9, + 0x6a, 0x76, 0xff, 0xa7, 0x3a, 0xeb, 0xf2, 0x74, 0x2e, 0xd1, 0x5c, 0xc7, 0xfd, 0x92, 0xb0, 0x6c, + 0x7b, 0x48, 0xa3, 0xa1, 0x41, 0x79, 0xf6, 0xd0, 0x20, 0x3b, 0xa4, 0xd1, 0x1c, 0xd5, 0xa1, 0x4a, + 0x49, 0x8e, 0x5e, 0x40, 0x95, 0x36, 0xec, 0xf0, 0xe4, 0x46, 0xce, 0x7d, 0xc9, 0x71, 0xcf, 0x85, + 0xb2, 0x8c, 0xf9, 0xd5, 0xda, 0xae, 0xec, 0x68, 0xcd, 0x5f, 0x18, 0xdb, 0x8f, 0xdb, 0x2f, 0xa5, + 0x6f, 0xcc, 0xe1, 0x13, 0x93, 0xfb, 0xa5, 0xd4, 0xe6, 0x9e, 0x07, 0xb5, 0xb9, 0x4f, 0x9f, 0x6a, + 0xf4, 0xb7, 0x71, 0xfc, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x02, 0x35, 0xe7, 0x43, 0x43, 0x06, 0x00, + 0x00, +} diff --git a/replication/pdu/pdu.proto b/replication/pdu/pdu.proto new file mode 100644 index 0000000..d07902c --- /dev/null +++ b/replication/pdu/pdu.proto @@ -0,0 +1,118 @@ +syntax = "proto3"; + +package pdu; + +message ListFilesystemReq {} + +message ListFilesystemRes { + repeated Filesystem Filesystems = 1; +} + +message Filesystem { + string Path = 1; + string ResumeToken = 2; +} + +message ListFilesystemVersionsReq { + string Filesystem = 1; +} + +message ListFilesystemVersionsRes { + repeated FilesystemVersion Versions = 1; +} + +message FilesystemVersion { + enum VersionType { + Snapshot = 0; + Bookmark = 1; + } + VersionType Type = 1; + string Name = 2; + uint64 Guid = 3; + uint64 CreateTXG = 4; + string Creation = 5; // RFC 3339 +} + + +message SendReq { + string Filesystem = 1; + string From = 2; + // May be empty / null to request a full transfer of From + string To = 3; + + // If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender. + // The sender MUST indicate in SendRes.UsedResumeToken + // If it does not work, the sender SHOULD clear the resume token on their side + // and use From and To instead + // If ResumeToken is not empty, the GUIDs of From and To + // MUST correspond to those encoded in the ResumeToken. + // Otherwise, the Sender MUST return an error. + string ResumeToken = 4; + bool Compress = 5; + bool Dedup = 6; + + bool DryRun = 7; +} + +message Property { + string Name = 1; + string Value = 2; +} + +message SendRes { + // The actual stream is in the stream part of the streamrpc response + + // Whether the resume token provided in the request has been used or not. + bool UsedResumeToken = 1; + + // Expected stream size determined by dry run, not exact. + // 0 indicates that for the given SendReq, no size estimate could be made. + int64 ExpectedSize = 2; + + repeated Property Properties = 3; +} + +message ReceiveReq { + // The stream part of the streamrpc request contains the zfs send stream + + string Filesystem = 1; + + // If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request + bool ClearResumeToken = 2; +} + +message ReceiveRes {} + +message DestroySnapshotsReq { + string Filesystem = 1; + // Path to filesystem, snapshot or bookmark to be destroyed + repeated FilesystemVersion Snapshots = 2; +} + +message DestroySnapshotRes { + FilesystemVersion Snapshot = 1; + string Error = 2; +} + +message DestroySnapshotsRes { + repeated DestroySnapshotRes Results = 1; +} + +message ReplicationCursorReq { + string Filesystem = 1; + message GetOp {} + message SetOp { + string Snapshot = 2; + } + oneof op { + GetOp get = 2; + SetOp set = 3; + } +} + +message ReplicationCursorRes { + oneof Result { + uint64 Guid = 1; + string Error = 2; + } +} diff --git a/replication/pdu/pdu_extras.go b/replication/pdu/pdu_extras.go new file mode 100644 index 0000000..9cf52c9 --- /dev/null +++ b/replication/pdu/pdu_extras.go @@ -0,0 +1,76 @@ +package pdu + +import ( + "fmt" + "github.com/zrepl/zrepl/zfs" + "time" +) + +func (v *FilesystemVersion) RelName() string { + zv, err := v.ZFSFilesystemVersion() + if err != nil { + panic(err) + } + return zv.String() +} + +func (v FilesystemVersion_VersionType) ZFSVersionType() zfs.VersionType { + switch v { + case FilesystemVersion_Snapshot: + return zfs.Snapshot + case FilesystemVersion_Bookmark: + return zfs.Bookmark + default: + panic(fmt.Sprintf("unexpected v.Type %#v", v)) + } +} + +func FilesystemVersionFromZFS(fsv *zfs.FilesystemVersion) *FilesystemVersion { + var t FilesystemVersion_VersionType + switch fsv.Type { + case zfs.Bookmark: + t = FilesystemVersion_Bookmark + case zfs.Snapshot: + t = FilesystemVersion_Snapshot + default: + panic("unknown fsv.Type: " + fsv.Type) + } + return &FilesystemVersion{ + Type: t, + Name: fsv.Name, + Guid: fsv.Guid, + CreateTXG: fsv.CreateTXG, + Creation: fsv.Creation.Format(time.RFC3339), + } +} + +func FilesystemVersionCreation(t time.Time) string { + return t.Format(time.RFC3339) +} + +func (v *FilesystemVersion) CreationAsTime() (time.Time, error) { + return time.Parse(time.RFC3339, v.Creation) +} + +// implement fsfsm.FilesystemVersion +func (v *FilesystemVersion) SnapshotTime() time.Time { + t, err := v.CreationAsTime() + if err != nil { + panic(err) // FIXME + } + return t +} + +func (v *FilesystemVersion) ZFSFilesystemVersion() (*zfs.FilesystemVersion, error) { + ct, err := v.CreationAsTime() + if err != nil { + return nil, err + } + return &zfs.FilesystemVersion{ + Type: v.Type.ZFSVersionType(), + Name: v.Name, + Guid: v.Guid, + CreateTXG: v.CreateTXG, + Creation: ct, + }, nil +} diff --git a/replication/pdu/pdu_test.go b/replication/pdu/pdu_test.go new file mode 100644 index 0000000..79315a6 --- /dev/null +++ b/replication/pdu/pdu_test.go @@ -0,0 +1,68 @@ +package pdu + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestFilesystemVersion_RelName(t *testing.T) { + + type TestCase struct { + In FilesystemVersion + Out string + Panic bool + } + + creat := FilesystemVersionCreation(time.Now()) + tcs := []TestCase{ + { + In: FilesystemVersion{ + Type: FilesystemVersion_Snapshot, + Name: "foobar", + Creation: creat, + }, + Out: "@foobar", + }, + { + In: FilesystemVersion{ + Type: FilesystemVersion_Bookmark, + Name: "foobar", + Creation: creat, + }, + Out: "#foobar", + }, + { + In: FilesystemVersion{ + Type: 2342, + Name: "foobar", + Creation: creat, + }, + Panic: true, + }, + } + + for _, tc := range tcs { + if tc.Panic { + assert.Panics(t, func() { + tc.In.RelName() + }) + } else { + o := tc.In.RelName() + assert.Equal(t, tc.Out, o) + } + } + +} + +func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) { + + empty := &FilesystemVersion{} + _, err:= empty.ZFSFilesystemVersion() + assert.Error(t, err) + + dateInvalid := &FilesystemVersion{Creation: "foobar"} + _, err = dateInvalid.ZFSFilesystemVersion() + assert.Error(t, err) + +} diff --git a/replication/state_enumer.go b/replication/state_enumer.go new file mode 100644 index 0000000..7343d69 --- /dev/null +++ b/replication/state_enumer.go @@ -0,0 +1,76 @@ +// Code generated by "enumer -type=State"; DO NOT EDIT. + +package replication + +import ( + "fmt" +) + +const ( + _StateName_0 = "PlanningPlanningError" + _StateName_1 = "Working" + _StateName_2 = "WorkingWait" + _StateName_3 = "Completed" + _StateName_4 = "ContextDone" +) + +var ( + _StateIndex_0 = [...]uint8{0, 8, 21} + _StateIndex_1 = [...]uint8{0, 7} + _StateIndex_2 = [...]uint8{0, 11} + _StateIndex_3 = [...]uint8{0, 9} + _StateIndex_4 = [...]uint8{0, 11} +) + +func (i State) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _StateName_0[_StateIndex_0[i]:_StateIndex_0[i+1]] + case i == 4: + return _StateName_1 + case i == 8: + return _StateName_2 + case i == 16: + return _StateName_3 + case i == 32: + return _StateName_4 + default: + return fmt.Sprintf("State(%d)", i) + } +} + +var _StateValues = []State{1, 2, 4, 8, 16, 32} + +var _StateNameToValueMap = map[string]State{ + _StateName_0[0:8]: 1, + _StateName_0[8:21]: 2, + _StateName_1[0:7]: 4, + _StateName_2[0:11]: 8, + _StateName_3[0:9]: 16, + _StateName_4[0:11]: 32, +} + +// StateString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func StateString(s string) (State, error) { + if val, ok := _StateNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to State values", s) +} + +// StateValues returns all values of the enum +func StateValues() []State { + return _StateValues +} + +// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise +func (i State) IsAState() bool { + for _, v := range _StateValues { + if i == v { + return true + } + } + return false +} diff --git a/rpc/client.go b/rpc/client.go deleted file mode 100644 index 262b9d1..0000000 --- a/rpc/client.go +++ /dev/null @@ -1,136 +0,0 @@ -package rpc - -import ( - "bytes" - "encoding/json" - "io" - "reflect" - - "github.com/pkg/errors" -) - -type Client struct { - ml *MessageLayer - logger Logger -} - -func NewClient(rwc io.ReadWriteCloser) *Client { - return &Client{NewMessageLayer(rwc), noLogger{}} -} - -func (c *Client) SetLogger(logger Logger, logMessageLayer bool) { - c.logger = logger - if logMessageLayer { - c.ml.logger = logger - } else { - c.ml.logger = noLogger{} - } -} - -func (c *Client) Close() (err error) { - - c.logger.Printf("sending Close request") - header := Header{ - DataType: DataTypeControl, - Endpoint: ControlEndpointClose, - Accept: DataTypeControl, - } - err = c.ml.WriteHeader(&header) - if err != nil { - return - } - - c.logger.Printf("reading Close ACK") - ack, err := c.ml.ReadHeader() - if err != nil { - return err - } - c.logger.Printf("received Close ACK: %#v", ack) - if ack.Error != StatusOK { - err = errors.Errorf("error hanging up: remote error (%s) %s", ack.Error, ack.ErrorMessage) - return - } - - c.logger.Printf("closing MessageLayer") - if err = c.ml.Close(); err != nil { - c.logger.Printf("error closing RWC: %+v", err) - return - } - - return err -} - -func (c *Client) recvResponse() (h *Header, err error) { - h, err = c.ml.ReadHeader() - if err != nil { - return nil, errors.Wrap(err, "cannot read header") - } - // TODO validate - return -} - -func (c *Client) writeRequest(h *Header) (err error) { - // TODO validate - err = c.ml.WriteHeader(h) - if err != nil { - return errors.Wrap(err, "cannot write header") - } - return -} - -func (c *Client) Call(endpoint string, in, out interface{}) (err error) { - - var accept DataType - { - outType := reflect.TypeOf(out) - if typeIsIOReaderPtr(outType) { - accept = DataTypeOctets - } else { - accept = DataTypeMarshaledJSON - } - } - - h := Header{ - Endpoint: endpoint, - DataType: DataTypeMarshaledJSON, - Accept: accept, - } - - if err = c.writeRequest(&h); err != nil { - return err - } - - var buf bytes.Buffer - if err = json.NewEncoder(&buf).Encode(in); err != nil { - panic("cannot encode 'in' parameter") - } - if err = c.ml.WriteData(&buf); err != nil { - return err - } - - rh, err := c.recvResponse() - if err != nil { - return err - } - if rh.Error != StatusOK { - return &RPCError{rh} - } - - rd := c.ml.ReadData() - - switch accept { - case DataTypeOctets: - c.logger.Printf("setting out to ML data reader") - outPtr := out.(*io.Reader) // we checked that above - *outPtr = rd - case DataTypeMarshaledJSON: - c.logger.Printf("decoding marshaled json") - if err = json.NewDecoder(c.ml.ReadData()).Decode(out); err != nil { - return errors.Wrap(err, "cannot decode marshaled reply") - } - default: - panic("implementation error") // accept is controlled by us - } - - return -} diff --git a/rpc/datatype_string.go b/rpc/datatype_string.go deleted file mode 100644 index e7293b7..0000000 --- a/rpc/datatype_string.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by "stringer -type=DataType"; DO NOT EDIT. - -package rpc - -import "strconv" - -const _DataType_name = "DataTypeNoneDataTypeControlDataTypeMarshaledJSONDataTypeOctets" - -var _DataType_index = [...]uint8{0, 12, 27, 48, 62} - -func (i DataType) String() string { - i -= 1 - if i >= DataType(len(_DataType_index)-1) { - return "DataType(" + strconv.FormatInt(int64(i+1), 10) + ")" - } - return _DataType_name[_DataType_index[i]:_DataType_index[i+1]] -} diff --git a/rpc/frame_layer.go b/rpc/frame_layer.go deleted file mode 100644 index 9d6327a..0000000 --- a/rpc/frame_layer.go +++ /dev/null @@ -1,302 +0,0 @@ -package rpc - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "io" - - "github.com/pkg/errors" -) - -type Frame struct { - Type FrameType - NoMoreFrames bool - PayloadLength uint32 -} - -//go:generate stringer -type=FrameType -type FrameType uint8 - -const ( - FrameTypeHeader FrameType = 0x01 - FrameTypeData FrameType = 0x02 - FrameTypeTrailer FrameType = 0x03 - FrameTypeRST FrameType = 0xff -) - -//go:generate stringer -type=Status -type Status uint64 - -const ( - StatusOK Status = 1 + iota - StatusRequestError - StatusServerError - // Returned when an error occurred but the side at fault cannot be determined - StatusError -) - -type Header struct { - // Request-only - Endpoint string - // Data type of body (request & reply) - DataType DataType - // Request-only - Accept DataType - // Reply-only - Error Status - // Reply-only - ErrorMessage string -} - -func NewErrorHeader(status Status, format string, args ...interface{}) (h *Header) { - h = &Header{} - h.Error = status - h.ErrorMessage = fmt.Sprintf(format, args...) - return -} - -//go:generate stringer -type=DataType -type DataType uint8 - -const ( - DataTypeNone DataType = 1 + iota - DataTypeControl - DataTypeMarshaledJSON - DataTypeOctets -) - -const ( - MAX_PAYLOAD_LENGTH = 4 * 1024 * 1024 - MAX_HEADER_LENGTH = 4 * 1024 -) - -type frameBridgingReader struct { - l *MessageLayer - frameType FrameType - // < 0 means no limit - bytesLeftToLimit int - f Frame -} - -func NewFrameBridgingReader(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingReader { - return &frameBridgingReader{l, frameType, totalLimit, Frame{}} -} - -func (r *frameBridgingReader) Read(b []byte) (n int, err error) { - if r.bytesLeftToLimit == 0 { - r.l.logger.Printf("limit reached, returning EOF") - return 0, io.EOF - } - log := r.l.logger - if r.f.PayloadLength == 0 { - - if r.f.NoMoreFrames { - r.l.logger.Printf("no more frames flag set, returning EOF") - err = io.EOF - return - } - - log.Printf("reading frame") - r.f, err = r.l.readFrame() - if err != nil { - log.Printf("error reading frame: %+v", err) - return 0, err - } - log.Printf("read frame: %#v", r.f) - if r.f.Type != r.frameType { - err = errors.Wrapf(err, "expected frame of type %s", r.frameType) - return 0, err - } - } - maxread := len(b) - if maxread > int(r.f.PayloadLength) { - maxread = int(r.f.PayloadLength) - } - if r.bytesLeftToLimit > 0 && maxread > r.bytesLeftToLimit { - maxread = r.bytesLeftToLimit - } - nb, err := r.l.rwc.Read(b[:maxread]) - log.Printf("read %v from rwc\n", nb) - if nb < 0 { - panic("should not return negative number of bytes") - } - r.f.PayloadLength -= uint32(nb) - r.bytesLeftToLimit -= nb - return nb, err // TODO io.EOF for maxread = r.f.PayloadLength ? -} - -type frameBridgingWriter struct { - l *MessageLayer - frameType FrameType - // < 0 means no limit - bytesLeftToLimit int - payloadLength int - buffer *bytes.Buffer -} - -func NewFrameBridgingWriter(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingWriter { - return &frameBridgingWriter{l, frameType, totalLimit, MAX_PAYLOAD_LENGTH, bytes.NewBuffer(make([]byte, 0, MAX_PAYLOAD_LENGTH))} -} - -func (w *frameBridgingWriter) Write(b []byte) (n int, err error) { - for n = 0; n < len(b); { - i, err := w.writeUntilFrameFull(b[n:]) - n += i - if err != nil { - return n, errors.WithStack(err) - } - } - return -} - -func (w *frameBridgingWriter) writeUntilFrameFull(b []byte) (n int, err error) { - if len(b) <= 0 { - return - } - if w.bytesLeftToLimit == 0 { - err = errors.Errorf("message exceeds max number of allowed bytes") - return - } - maxwrite := len(b) - remainingInFrame := w.payloadLength - w.buffer.Len() - - if maxwrite > remainingInFrame { - maxwrite = remainingInFrame - } - if w.bytesLeftToLimit > 0 && maxwrite > w.bytesLeftToLimit { - maxwrite = w.bytesLeftToLimit - } - w.buffer.Write(b[:maxwrite]) - w.bytesLeftToLimit -= maxwrite - n = maxwrite - if w.bytesLeftToLimit == 0 { - err = w.flush(true) - } else if w.buffer.Len() == w.payloadLength { - err = w.flush(false) - } - return -} - -func (w *frameBridgingWriter) flush(nomore bool) (err error) { - - f := Frame{w.frameType, nomore, uint32(w.buffer.Len())} - err = w.l.writeFrame(f) - if err != nil { - errors.WithStack(err) - } - _, err = w.buffer.WriteTo(w.l.rwc) - return -} - -func (w *frameBridgingWriter) Close() (err error) { - return w.flush(true) -} - -type MessageLayer struct { - rwc io.ReadWriteCloser - logger Logger -} - -func NewMessageLayer(rwc io.ReadWriteCloser) *MessageLayer { - return &MessageLayer{rwc, noLogger{}} -} - -func (l *MessageLayer) Close() (err error) { - f := Frame{ - Type: FrameTypeRST, - NoMoreFrames: true, - } - if err = l.writeFrame(f); err != nil { - l.logger.Printf("error sending RST frame: %s", err) - return errors.WithStack(err) - } - return nil -} - -var RST error = fmt.Errorf("reset frame observed on connection") - -func (l *MessageLayer) readFrame() (f Frame, err error) { - err = binary.Read(l.rwc, binary.LittleEndian, &f.Type) - if err != nil { - err = errors.WithStack(err) - return - } - err = binary.Read(l.rwc, binary.LittleEndian, &f.NoMoreFrames) - if err != nil { - err = errors.WithStack(err) - return - } - err = binary.Read(l.rwc, binary.LittleEndian, &f.PayloadLength) - if err != nil { - err = errors.WithStack(err) - return - } - if f.Type == FrameTypeRST { - l.logger.Printf("read RST frame") - err = RST - return - } - if f.PayloadLength > MAX_PAYLOAD_LENGTH { - err = errors.Errorf("frame exceeds max payload length") - return - } - return -} - -func (l *MessageLayer) writeFrame(f Frame) (err error) { - err = binary.Write(l.rwc, binary.LittleEndian, &f.Type) - if err != nil { - return errors.WithStack(err) - } - err = binary.Write(l.rwc, binary.LittleEndian, &f.NoMoreFrames) - if err != nil { - return errors.WithStack(err) - } - err = binary.Write(l.rwc, binary.LittleEndian, &f.PayloadLength) - if err != nil { - return errors.WithStack(err) - } - if f.PayloadLength > MAX_PAYLOAD_LENGTH { - err = errors.Errorf("frame exceeds max payload length") - return - } - return -} - -func (l *MessageLayer) ReadHeader() (h *Header, err error) { - - r := NewFrameBridgingReader(l, FrameTypeHeader, MAX_HEADER_LENGTH) - h = &Header{} - if err = json.NewDecoder(r).Decode(&h); err != nil { - l.logger.Printf("cannot decode marshaled header: %s", err) - return nil, err - } - return h, nil -} - -func (l *MessageLayer) WriteHeader(h *Header) (err error) { - w := NewFrameBridgingWriter(l, FrameTypeHeader, MAX_HEADER_LENGTH) - err = json.NewEncoder(w).Encode(h) - if err != nil { - return errors.Wrap(err, "cannot encode header, probably fatal") - } - w.Close() - return -} - -func (l *MessageLayer) ReadData() (reader io.Reader) { - r := NewFrameBridgingReader(l, FrameTypeData, -1) - return r -} - -func (l *MessageLayer) WriteData(source io.Reader) (err error) { - w := NewFrameBridgingWriter(l, FrameTypeData, -1) - _, err = io.Copy(w, source) - if err != nil { - return errors.WithStack(err) - } - err = w.Close() - return -} diff --git a/rpc/frametype_string.go b/rpc/frametype_string.go deleted file mode 100644 index 9e3ced3..0000000 --- a/rpc/frametype_string.go +++ /dev/null @@ -1,26 +0,0 @@ -// Code generated by "stringer -type=FrameType"; DO NOT EDIT. - -package rpc - -import "strconv" - -const ( - _FrameType_name_0 = "FrameTypeHeaderFrameTypeDataFrameTypeTrailer" - _FrameType_name_1 = "FrameTypeRST" -) - -var ( - _FrameType_index_0 = [...]uint8{0, 15, 28, 44} -) - -func (i FrameType) String() string { - switch { - case 1 <= i && i <= 3: - i -= 1 - return _FrameType_name_0[_FrameType_index_0[i]:_FrameType_index_0[i+1]] - case i == 255: - return _FrameType_name_1 - default: - return "FrameType(" + strconv.FormatInt(int64(i), 10) + ")" - } -} diff --git a/rpc/local.go b/rpc/local.go deleted file mode 100644 index 4f729d7..0000000 --- a/rpc/local.go +++ /dev/null @@ -1,63 +0,0 @@ -package rpc - -import ( - "github.com/pkg/errors" - "reflect" -) - -type LocalRPC struct { - endpoints map[string]reflect.Value -} - -func NewLocalRPC() *LocalRPC { - return &LocalRPC{make(map[string]reflect.Value, 0)} -} - -func (s *LocalRPC) RegisterEndpoint(name string, handler interface{}) (err error) { - _, ok := s.endpoints[name] - if ok { - return errors.Errorf("already set up an endpoint for '%s'", name) - } - ep, err := makeEndpointDescr(handler) - if err != nil { - return err - } - s.endpoints[name] = ep.handler - return nil -} - -func (s *LocalRPC) Serve() (err error) { - panic("local cannot serve") -} - -func (c *LocalRPC) Call(endpoint string, in, out interface{}) (err error) { - ep, ok := c.endpoints[endpoint] - if !ok { - panic("implementation error: implementation should not call local RPC without knowing which endpoints exist") - } - - args := []reflect.Value{reflect.ValueOf(in), reflect.ValueOf(out)} - - if err = checkRPCParamTypes(args[0].Type(), args[1].Type()); err != nil { - return - } - - rets := ep.Call(args) - - if len(rets) != 1 { - panic("implementation error: endpoints must have one error ") - } - if err = checkRPCReturnType(rets[0].Type()); err != nil { - panic(err) - } - - err = nil - if !rets[0].IsNil() { - err = rets[0].Interface().(error) // we checked that above - } - return -} - -func (c *LocalRPC) Close() (err error) { - return nil -} diff --git a/rpc/server.go b/rpc/server.go deleted file mode 100644 index 7d7ee16..0000000 --- a/rpc/server.go +++ /dev/null @@ -1,259 +0,0 @@ -package rpc - -import ( - "bytes" - "encoding/json" - "io" - "reflect" - - "github.com/pkg/errors" -) - -type Server struct { - ml *MessageLayer - logger Logger - endpoints map[string]endpointDescr -} - -type typeMap struct { - local reflect.Type - proto DataType -} -type endpointDescr struct { - inType typeMap - outType typeMap - handler reflect.Value -} - -type MarshaledJSONEndpoint func(bodyJSON interface{}) - -func NewServer(rwc io.ReadWriteCloser) *Server { - ml := NewMessageLayer(rwc) - return &Server{ - ml, noLogger{}, make(map[string]endpointDescr), - } -} - -func (s *Server) SetLogger(logger Logger, logMessageLayer bool) { - s.logger = logger - if logMessageLayer { - s.ml.logger = logger - } else { - s.ml.logger = noLogger{} - } -} - -func (s *Server) RegisterEndpoint(name string, handler interface{}) (err error) { - _, ok := s.endpoints[name] - if ok { - return errors.Errorf("already set up an endpoint for '%s'", name) - } - s.endpoints[name], err = makeEndpointDescr(handler) - return -} - -func checkResponseHeader(h *Header) (err error) { - var statusNotSet Status - if h.Error == statusNotSet { - return errors.Errorf("status has zero-value") - } - return nil -} - -func (s *Server) writeResponse(h *Header) (err error) { - // TODO validate - return s.ml.WriteHeader(h) -} - -func (s *Server) recvRequest() (h *Header, err error) { - h, err = s.ml.ReadHeader() - if err != nil { - s.logger.Printf("error reading header: %s", err) - return nil, err - } - - s.logger.Printf("validating request") - err = nil // TODO validate - if err == nil { - return h, nil - } - s.logger.Printf("request validation error: %s", err) - - r := NewErrorHeader(StatusRequestError, "%s", err) - return nil, s.writeResponse(r) -} - -var doneServeNext error = errors.New("this should not cause a HangUp() in the server") -var doneStopServing error = errors.New("this should cause the server to close the connection") - -var ProtocolError error = errors.New("protocol error, server should hang up") - -const ControlEndpointClose string = "Close" - -// Serve the connection until failure or the client hangs up -func (s *Server) Serve() (err error) { - for { - - err = s.ServeRequest() - - if err == nil { - continue - } - if err == doneServeNext { - s.logger.Printf("subroutine returned pseudo-error indicating early-exit") - err = nil - continue - } - - if err == doneStopServing { - s.logger.Printf("subroutine returned pseudo-error indicating close request") - err = nil - break - } - - break - } - - if err != nil { - s.logger.Printf("an error occurred that could not be handled on PRC protocol level: %+v", err) - } - - s.logger.Printf("cloing MessageLayer") - if mlErr := s.ml.Close(); mlErr != nil { - s.logger.Printf("error closing MessageLayer: %+v", mlErr) - } - - return err -} - -// Serve a single request -// * wait for request to come in -// * call handler -// * reply -// -// The connection is left open, the next bytes on the conn should be -// the next request header. -// -// Returns an err != nil if the error is bad enough to hang up on the client. -// Examples: protocol version mismatches, protocol errors in general, ... -// Non-Examples: a handler error -func (s *Server) ServeRequest() (err error) { - - ml := s.ml - - s.logger.Printf("reading header") - h, err := s.recvRequest() - if err != nil { - return err - } - - if h.DataType == DataTypeControl { - switch h.Endpoint { - case ControlEndpointClose: - ack := Header{Error: StatusOK, DataType: DataTypeControl} - err = s.writeResponse(&ack) - if err != nil { - return err - } - return doneStopServing - default: - r := NewErrorHeader(StatusRequestError, "unregistered control endpoint %s", h.Endpoint) - return s.writeResponse(r) - } - } - - ep, ok := s.endpoints[h.Endpoint] - if !ok { - r := NewErrorHeader(StatusRequestError, "unregistered endpoint %s", h.Endpoint) - return s.writeResponse(r) - } - - if ep.inType.proto != h.DataType { - r := NewErrorHeader(StatusRequestError, "wrong DataType for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.inType.proto, h.DataType) - return s.writeResponse(r) - } - - if ep.outType.proto != h.Accept { - r := NewErrorHeader(StatusRequestError, "wrong Accept for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.outType.proto, h.Accept) - return s.writeResponse(r) - } - - dr := ml.ReadData() - - // Determine inval - var inval reflect.Value - switch ep.inType.proto { - case DataTypeMarshaledJSON: - // Unmarshal input - inval = reflect.New(ep.inType.local.Elem()) - invalIface := inval.Interface() - err = json.NewDecoder(dr).Decode(invalIface) - if err != nil { - r := NewErrorHeader(StatusRequestError, "cannot decode marshaled JSON: %s", err) - return s.writeResponse(r) - } - case DataTypeOctets: - // Take data as is - inval = reflect.ValueOf(dr) - default: - panic("not implemented") - } - - outval := reflect.New(ep.outType.local.Elem()) // outval is a double pointer - - s.logger.Printf("before handler, inval=%v outval=%v", inval, outval) - - // Call the handler - errs := ep.handler.Call([]reflect.Value{inval, outval}) - - if !errs[0].IsNil() { - he := errs[0].Interface().(error) // we checked that before... - s.logger.Printf("handler returned error: %s", err) - r := NewErrorHeader(StatusError, "%s", he.Error()) - return s.writeResponse(r) - } - - switch ep.outType.proto { - - case DataTypeMarshaledJSON: - - var dataBuf bytes.Buffer - // Marshal output - err = json.NewEncoder(&dataBuf).Encode(outval.Interface()) - if err != nil { - r := NewErrorHeader(StatusServerError, "cannot marshal response: %s", err) - return s.writeResponse(r) - } - - replyHeader := Header{ - Error: StatusOK, - DataType: ep.outType.proto, - } - if err = s.writeResponse(&replyHeader); err != nil { - return err - } - - if err = ml.WriteData(&dataBuf); err != nil { - return - } - - case DataTypeOctets: - - h := Header{ - Error: StatusOK, - DataType: DataTypeOctets, - } - if err = s.writeResponse(&h); err != nil { - return - } - - reader := outval.Interface().(*io.Reader) // we checked that when adding the endpoint - err = ml.WriteData(*reader) - if err != nil { - return err - } - - } - - return nil -} diff --git a/rpc/shared.go b/rpc/shared.go deleted file mode 100644 index 052f441..0000000 --- a/rpc/shared.go +++ /dev/null @@ -1,111 +0,0 @@ -package rpc - -import ( - "fmt" - "github.com/pkg/errors" - "io" - "reflect" -) - -type RPCServer interface { - Serve() (err error) - RegisterEndpoint(name string, handler interface{}) (err error) -} - -type RPCClient interface { - Call(endpoint string, in, out interface{}) (err error) - Close() (err error) -} - -type Logger interface { - Printf(format string, args ...interface{}) -} - -type noLogger struct{} - -func (l noLogger) Printf(format string, args ...interface{}) {} -func typeIsIOReader(t reflect.Type) bool { - return t == reflect.TypeOf((*io.Reader)(nil)).Elem() -} - -func typeIsIOReaderPtr(t reflect.Type) bool { - return t == reflect.TypeOf((*io.Reader)(nil)) -} - -// An error returned by the Client if the response indicated a status code other than StatusOK -type RPCError struct { - ResponseHeader *Header -} - -func (e *RPCError) Error() string { - return fmt.Sprintf("%s: %s", e.ResponseHeader.Error, e.ResponseHeader.ErrorMessage) -} - -type RPCProtoError struct { - Message string - UnderlyingError error -} - -func (e *RPCProtoError) Error() string { - return e.Message -} - -func checkRPCParamTypes(in, out reflect.Type) (err error) { - if !(in.Kind() == reflect.Ptr || typeIsIOReader(in)) { - err = errors.Errorf("input parameter must be a pointer or an io.Reader, is of kind %s, type %s", in.Kind(), in) - return - } - if !(out.Kind() == reflect.Ptr) { - err = errors.Errorf("second input parameter (the non-error output parameter) must be a pointer or an *io.Reader") - return - } - return nil -} - -func checkRPCReturnType(rt reflect.Type) (err error) { - errInterfaceType := reflect.TypeOf((*error)(nil)).Elem() - if !rt.Implements(errInterfaceType) { - err = errors.Errorf("handler must return an error") - return - } - return nil -} - -func makeEndpointDescr(handler interface{}) (descr endpointDescr, err error) { - - ht := reflect.TypeOf(handler) - - if ht.Kind() != reflect.Func { - err = errors.Errorf("handler must be of kind reflect.Func") - return - } - - if ht.NumIn() != 2 || ht.NumOut() != 1 { - err = errors.Errorf("handler must have exactly two input parameters and one output parameter") - return - } - if err = checkRPCParamTypes(ht.In(0), ht.In(1)); err != nil { - return - } - if err = checkRPCReturnType(ht.Out(0)); err != nil { - return - } - - descr.handler = reflect.ValueOf(handler) - descr.inType.local = ht.In(0) - descr.outType.local = ht.In(1) - - if typeIsIOReader(ht.In(0)) { - descr.inType.proto = DataTypeOctets - } else { - descr.inType.proto = DataTypeMarshaledJSON - } - - if typeIsIOReaderPtr(ht.In(1)) { - descr.outType.proto = DataTypeOctets - } else { - descr.outType.proto = DataTypeMarshaledJSON - } - - return -} diff --git a/rpc/status_string.go b/rpc/status_string.go deleted file mode 100644 index 10e801d..0000000 --- a/rpc/status_string.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by "stringer -type=Status"; DO NOT EDIT. - -package rpc - -import "strconv" - -const _Status_name = "StatusOKStatusRequestErrorStatusServerErrorStatusError" - -var _Status_index = [...]uint8{0, 8, 26, 43, 54} - -func (i Status) String() string { - i -= 1 - if i >= Status(len(_Status_index)-1) { - return "Status(" + strconv.FormatInt(int64(i+1), 10) + ")" - } - return _Status_name[_Status_index[i]:_Status_index[i+1]] -} diff --git a/tlsconf/tlsconf.go b/tlsconf/tlsconf.go new file mode 100644 index 0000000..48fc382 --- /dev/null +++ b/tlsconf/tlsconf.go @@ -0,0 +1,111 @@ +package tlsconf + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "io/ioutil" + "net" + "time" +) + +func ParseCAFile(certfile string) (*x509.CertPool, error) { + pool := x509.NewCertPool() + pem, err := ioutil.ReadFile(certfile) + if err != nil { + return nil, err + } + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("PEM parsing error") + } + return pool, nil +} + +type ClientAuthListener struct { + l net.Listener + handshakeTimeout time.Duration +} + +func NewClientAuthListener( + l net.Listener, ca *x509.CertPool, serverCert tls.Certificate, + handshakeTimeout time.Duration) *ClientAuthListener { + + if ca == nil { + panic(ca) + } + if serverCert.Certificate == nil || serverCert.PrivateKey == nil { + panic(serverCert) + } + + tlsConf := tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientCAs: ca, + ClientAuth: tls.RequireAndVerifyClientCert, + PreferServerCipherSuites: true, + } + l = tls.NewListener(l, &tlsConf) + return &ClientAuthListener{ + l, + handshakeTimeout, + } +} + +func (l *ClientAuthListener) Accept() (c net.Conn, clientCN string, err error) { + c, err = l.l.Accept() + if err != nil { + return nil, "", err + } + tlsConn, ok := c.(*tls.Conn) + if !ok { + return c, "", err + } + + var ( + cn string + peerCerts []*x509.Certificate + ) + if err = tlsConn.SetDeadline(time.Now().Add(l.handshakeTimeout)); err != nil { + goto CloseAndErr + } + if err = tlsConn.Handshake(); err != nil { + goto CloseAndErr + } + + peerCerts = tlsConn.ConnectionState().PeerCertificates + if len(peerCerts) != 1 { + err = errors.New("unexpected number of certificates presented by TLS client") + goto CloseAndErr + } + cn = peerCerts[0].Subject.CommonName + return c, cn, nil +CloseAndErr: + c.Close() + return nil, "", err +} + +func (l *ClientAuthListener) Addr() net.Addr { + return l.l.Addr() +} + +func (l *ClientAuthListener) Close() error { + return l.l.Close() +} + +func ClientAuthClient(serverName string, rootCA *x509.CertPool, clientCert tls.Certificate) (*tls.Config, error) { + if serverName == "" { + panic(serverName) + } + if rootCA == nil { + panic(rootCA) + } + if clientCert.Certificate == nil || clientCert.PrivateKey == nil { + panic(clientCert) + } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + RootCAs: rootCA, + ServerName: serverName, + } + tlsConfig.BuildNameToCertificate() + return tlsConfig, nil +} diff --git a/util/contextflexibletimeout.go b/util/contextflexibletimeout.go new file mode 100644 index 0000000..422318d --- /dev/null +++ b/util/contextflexibletimeout.go @@ -0,0 +1,83 @@ +package util + +import ( + "context" + "sync" + "time" +) + +type contextWithOptionalDeadline struct { + context.Context + + m sync.Mutex + deadline time.Time + + done chan struct{} + err error +} + +func (c *contextWithOptionalDeadline) Deadline() (deadline time.Time, ok bool) { + c.m.Lock() + defer c.m.Unlock() + return c.deadline, !c.deadline.IsZero() +} + +func (c *contextWithOptionalDeadline) Err() error { + c.m.Lock() + defer c.m.Unlock() + return c.err +} + +func (c *contextWithOptionalDeadline) Done() <-chan struct{} { + return c.done +} + +func ContextWithOptionalDeadline(pctx context.Context) (ctx context.Context, enforceDeadline func(deadline time.Time)) { + + // mctx can only be cancelled by cancelMctx, not by a potential cancel of pctx + rctx := &contextWithOptionalDeadline{ + Context: pctx, + done: make(chan struct{}), + err: nil, + } + enforceDeadline = func(deadline time.Time) { + + // Set deadline and prohibit multiple calls + rctx.m.Lock() + alreadyCalled := !rctx.deadline.IsZero() + if !alreadyCalled { + rctx.deadline = deadline + } + rctx.m.Unlock() + if alreadyCalled { + return + } + + // Deadline in past? + sleepTime := deadline.Sub(time.Now()) + if sleepTime <= 0 { + rctx.m.Lock() + rctx.err = context.DeadlineExceeded + rctx.m.Unlock() + close(rctx.done) + return + } + go func() { + // Set a timer and wait for timer or parent context to be cancelled + timer := time.NewTimer(sleepTime) + var setErr error + select { + case <-pctx.Done(): + timer.Stop() + setErr = pctx.Err() + case <-timer.C: + setErr = context.DeadlineExceeded + } + rctx.m.Lock() + rctx.err = setErr + rctx.m.Unlock() + close(rctx.done) + }() + } + return rctx, enforceDeadline +} diff --git a/util/contextflexibletimeout_test.go b/util/contextflexibletimeout_test.go new file mode 100644 index 0000000..e6a1128 --- /dev/null +++ b/util/contextflexibletimeout_test.go @@ -0,0 +1,84 @@ +package util + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestContextWithOptionalDeadline(t *testing.T) { + + ctx := context.Background() + cctx, enforceDeadline := ContextWithOptionalDeadline(ctx) + + begin := time.Now() + var receivedCancellation time.Time + var cancellationError error + go func() { + select { + case <-cctx.Done(): + receivedCancellation = time.Now() + cancellationError = cctx.Err() + case <-time.After(600 * time.Millisecond): + t.Fatalf("should have been cancelled by deadline") + } + }() + time.Sleep(100 * time.Millisecond) + if !receivedCancellation.IsZero() { + t.Fatalf("no enforcement means no cancellation") + } + require.Nil(t, cctx.Err(), "no error while not cancelled") + dl, ok := cctx.Deadline() + require.False(t, ok) + require.Zero(t, dl) + enforceDeadline(begin.Add(200 * time.Millisecond)) + // second call must be ignored, i.e. we expect the deadline to be at begin+200ms, not begin+400ms + enforceDeadline(begin.Add(400 * time.Millisecond)) + + time.Sleep(300 * time.Millisecond) // 100ms margin for scheduler + if receivedCancellation.Sub(begin) > 250*time.Millisecond { + t.Fatalf("cancellation is beyond acceptable scheduler latency") + } + require.Equal(t, context.DeadlineExceeded, cancellationError) +} + +func TestContextWithOptionalDeadlineNegativeDeadline(t *testing.T) { + ctx := context.Background() + cctx, enforceDeadline := ContextWithOptionalDeadline(ctx) + enforceDeadline(time.Now().Add(-10 * time.Second)) + select { + case <-cctx.Done(): + default: + t.FailNow() + } +} + +func TestContextWithOptionalDeadlineParentCancellation(t *testing.T) { + + pctx, cancel := context.WithCancel(context.Background()) + cctx, enforceDeadline := ContextWithOptionalDeadline(pctx) + + // 0 ms + start := time.Now() + enforceDeadline(start.Add(400 * time.Millisecond)) + time.Sleep(100 * time.Millisecond) + cancel() // cancel @ ~100ms + time.Sleep(100 * time.Millisecond) // give 100ms time to propagate cancel + // @ ~200ms + select { + case <-cctx.Done(): + assert.True(t, time.Now().Before(start.Add(300*time.Millisecond))) + assert.Equal(t, context.Canceled, cctx.Err()) + default: + t.FailNow() + } + +} + +func TestContextWithOptionalDeadlineValue(t *testing.T) { + pctx := context.WithValue(context.Background(), "key", "value") + cctx, _ := ContextWithOptionalDeadline(pctx) + assert.Equal(t, "value", cctx.Value("key")) +} diff --git a/util/io.go b/util/io.go index 68ae286..f9210b3 100644 --- a/util/io.go +++ b/util/io.go @@ -2,18 +2,20 @@ package util import ( "io" + "net" "os" + "sync/atomic" ) -type ReadWriteCloserLogger struct { - RWC io.ReadWriteCloser +type NetConnLogger struct { + net.Conn ReadFile *os.File WriteFile *os.File } -func NewReadWriteCloserLogger(rwc io.ReadWriteCloser, readlog, writelog string) (l *ReadWriteCloserLogger, err error) { - l = &ReadWriteCloserLogger{ - RWC: rwc, +func NewNetConnLogger(conn net.Conn, readlog, writelog string) (l *NetConnLogger, err error) { + l = &NetConnLogger{ + Conn: conn, } flags := os.O_CREATE | os.O_WRONLY if readlog != "" { @@ -29,8 +31,8 @@ func NewReadWriteCloserLogger(rwc io.ReadWriteCloser, readlog, writelog string) return } -func (c *ReadWriteCloserLogger) Read(buf []byte) (n int, err error) { - n, err = c.RWC.Read(buf) +func (c *NetConnLogger) Read(buf []byte) (n int, err error) { + n, err = c.Conn.Read(buf) if c.WriteFile != nil { if _, writeErr := c.ReadFile.Write(buf[0:n]); writeErr != nil { panic(writeErr) @@ -39,8 +41,8 @@ func (c *ReadWriteCloserLogger) Read(buf []byte) (n int, err error) { return } -func (c *ReadWriteCloserLogger) Write(buf []byte) (n int, err error) { - n, err = c.RWC.Write(buf) +func (c *NetConnLogger) Write(buf []byte) (n int, err error) { + n, err = c.Conn.Write(buf) if c.ReadFile != nil { if _, writeErr := c.WriteFile.Write(buf[0:n]); writeErr != nil { panic(writeErr) @@ -48,8 +50,8 @@ func (c *ReadWriteCloserLogger) Write(buf []byte) (n int, err error) { } return } -func (c *ReadWriteCloserLogger) Close() (err error) { - err = c.RWC.Close() +func (c *NetConnLogger) Close() (err error) { + err = c.Conn.Close() if err != nil { return } @@ -96,3 +98,28 @@ func (c *ChainedReader) Read(buf []byte) (n int, err error) { return } + +type ByteCounterReader struct { + reader io.ReadCloser + bytes int64 +} + +func NewByteCounterReader(reader io.ReadCloser) *ByteCounterReader { + return &ByteCounterReader{ + reader: reader, + } +} + +func (b *ByteCounterReader) Close() error { + return b.reader.Close() +} + +func (b *ByteCounterReader) Read(p []byte) (n int, err error) { + n, err = b.reader.Read(p) + atomic.AddInt64(&b.bytes, int64(n)) + return n, err +} + +func (b *ByteCounterReader) Bytes() int64 { + return atomic.LoadInt64(&b.bytes) +} diff --git a/util/iocommand.go b/util/iocommand.go index 44671da..0433446 100644 --- a/util/iocommand.go +++ b/util/iocommand.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "os" "os/exec" "syscall" ) @@ -11,8 +12,8 @@ import ( // An IOCommand exposes a forked process's std(in|out|err) through the io.ReadWriteCloser interface. type IOCommand struct { Cmd *exec.Cmd - Stdin io.Writer - Stdout io.Reader + Stdin io.WriteCloser + Stdout io.ReadCloser StderrBuf *bytes.Buffer ExitResult *IOCommandExitResult } @@ -88,9 +89,23 @@ func (c *IOCommand) Read(buf []byte) (n int, err error) { func (c *IOCommand) doWait() (err error) { waitErr := c.Cmd.Wait() - waitStatus := c.Cmd.ProcessState.Sys().(syscall.WaitStatus) // Fail hard if we're not on UNIX + var wasUs bool = false + var waitStatus syscall.WaitStatus + if c.Cmd.ProcessState == nil { + fmt.Fprintf(os.Stderr, "util.IOCommand: c.Cmd.ProcessState is nil after c.Cmd.Wait()\n") + } + if c.Cmd.ProcessState != nil { + sysSpecific := c.Cmd.ProcessState.Sys() + var ok bool + waitStatus, ok = sysSpecific.(syscall.WaitStatus) + if !ok { + fmt.Fprintf(os.Stderr, "util.IOCommand: c.Cmd.ProcessState.Sys() could not be converted to syscall.WaitStatus: %T\n", sysSpecific) + os.Stderr.Sync() + panic(sysSpecific) // this can only be true if we are not on UNIX, and we don't support that + } + wasUs = waitStatus.Signaled() && waitStatus.Signal() == syscall.SIGTERM // in Close() + } - wasUs := waitStatus.Signaled() && waitStatus.Signal() == syscall.SIGTERM // in Close() if waitErr != nil && !wasUs { err = IOCommandError{ WaitErr: waitErr, diff --git a/util/socketpair/socketpair.go b/util/socketpair/socketpair.go new file mode 100644 index 0000000..615c8f2 --- /dev/null +++ b/util/socketpair/socketpair.go @@ -0,0 +1,49 @@ +package socketpair + +import ( + "golang.org/x/sys/unix" + "net" + "os" +) +type fileConn struct { + net.Conn // net.FileConn + f *os.File +} + +func (c fileConn) Close() error { + if err := c.Conn.Close(); err != nil { + return err + } + if err := c.f.Close(); err != nil { + return err + } + return nil +} + +func SocketPair() (a, b net.Conn, err error) { + // don't use net.Pipe, as it doesn't implement things like lingering, which our code relies on + sockpair, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0) + if err != nil { + return nil, nil, err + } + toConn := func(fd int) (net.Conn, error) { + f := os.NewFile(uintptr(fd), "fileconn") + if f == nil { + panic(fd) + } + c, err := net.FileConn(f) + if err != nil { + f.Close() + return nil, err + } + return fileConn{Conn: c, f: f}, nil + } + if a, err = toConn(sockpair[0]); err != nil { // shadowing + return nil, nil, err + } + if b, err = toConn(sockpair[1]); err != nil { // shadowing + a.Close() + return nil, nil, err + } + return a, b, nil +} diff --git a/cmd/version.go b/version/version.go similarity index 62% rename from cmd/version.go rename to version/version.go index 4597d2d..19e7aa9 100644 --- a/cmd/version.go +++ b/version/version.go @@ -1,24 +1,13 @@ -package cmd +package version import ( "fmt" - "github.com/spf13/cobra" "runtime" ) -var versionCmd = &cobra.Command{ - Use: "version", - Short: "print version of zrepl binary (for running daemon 'zrepl control version' command)", - Run: doVersion, -} - -func init() { - RootCmd.AddCommand(versionCmd) -} - -func doVersion(cmd *cobra.Command, args []string) { - fmt.Println(NewZreplVersionInformation().String()) -} +var ( + zreplVersion string // set by build infrastructure +) type ZreplVersionInformation struct { Version string diff --git a/zfs/diff.go b/zfs/diff.go index 12f9174..31886e3 100644 --- a/zfs/diff.go +++ b/zfs/diff.go @@ -5,6 +5,7 @@ import ( "crypto/sha512" "encoding/hex" "fmt" + "io" "os/exec" "sort" ) @@ -245,6 +246,19 @@ func IsPlaceholder(p *DatasetPath, placeholderPropertyValue string) (isPlacehold return } +// for nonexistent FS, isPlaceholder == false && err == nil +func ZFSIsPlaceholderFilesystem(p *DatasetPath) (isPlaceholder bool, err error) { + props, err := zfsGet(p.ToString(), []string{ZREPL_PLACEHOLDER_PROPERTY_NAME}, sourceLocal) + if err == io.ErrUnexpectedEOF { + // interpret this as an early exit of the zfs binary due to the fs not existing + return false, nil + } else if err != nil { + return false, err + } + isPlaceholder, _ = IsPlaceholder(p, props.Get(ZREPL_PLACEHOLDER_PROPERTY_NAME)) + return +} + func ZFSCreatePlaceholderFilesystem(p *DatasetPath) (err error) { v := PlaceholderPropertyValue(p) cmd := exec.Command(ZFS_BINARY, "create", diff --git a/zfs/mapping.go b/zfs/mapping.go index 4ccfc70..56a85b3 100644 --- a/zfs/mapping.go +++ b/zfs/mapping.go @@ -10,26 +10,56 @@ type DatasetFilter interface { } func ZFSListMapping(filter DatasetFilter) (datasets []*DatasetPath, err error) { + res, err := ZFSListMappingProperties(filter, nil) + if err != nil { + return nil, err + } + datasets = make([]*DatasetPath, len(res)) + for i, r := range res { + datasets[i] = r.Path + } + return datasets, nil +} + +type ZFSListMappingPropertiesResult struct { + Path *DatasetPath + // Guaranteed to have the same length as properties in the originating call + Fields []string +} + +// properties must not contain 'name' +func ZFSListMappingProperties(filter DatasetFilter, properties []string) (datasets []ZFSListMappingPropertiesResult, err error) { if filter == nil { panic("filter must not be nil") } + for _, p := range properties { + if p == "name" { + panic("properties must not contain 'name'") + } + } + newProps := make([]string, len(properties)+1) + newProps[0] = "name" + copy(newProps[1:], properties) + properties = newProps + ctx, cancel := context.WithCancel(context.Background()) defer cancel() rchan := make(chan ZFSListResult) - go ZFSListChan(ctx, rchan, []string{"name"}, "-r", "-t", "filesystem,volume") - datasets = make([]*DatasetPath, 0) + go ZFSListChan(ctx, rchan, properties, "-r", "-t", "filesystem,volume") + + datasets = make([]ZFSListMappingPropertiesResult, 0) for r := range rchan { - if r.err != nil { - err = r.err + if r.Err != nil { + err = r.Err return } var path *DatasetPath - if path, err = NewDatasetPath(r.fields[0]); err != nil { + if path, err = NewDatasetPath(r.Fields[0]); err != nil { return } @@ -38,7 +68,10 @@ func ZFSListMapping(filter DatasetFilter) (datasets []*DatasetPath, err error) { return nil, fmt.Errorf("error calling filter: %s", filterErr) } if pass { - datasets = append(datasets, path) + datasets = append(datasets, ZFSListMappingPropertiesResult{ + Path: path, + Fields: r.Fields[1:], + }) } } diff --git a/zfs/prometheus.go b/zfs/prometheus.go index f27d797..3934418 100644 --- a/zfs/prometheus.go +++ b/zfs/prometheus.go @@ -3,10 +3,10 @@ package zfs import "github.com/prometheus/client_golang/prometheus" var prom struct { - ZFSListFilesystemVersionDuration *prometheus.HistogramVec - ZFSDestroyFilesystemVersionDuration *prometheus.HistogramVec - ZFSSnapshotDuration *prometheus.HistogramVec - ZFSBookmarkDuration *prometheus.HistogramVec + ZFSListFilesystemVersionDuration *prometheus.HistogramVec + ZFSSnapshotDuration *prometheus.HistogramVec + ZFSBookmarkDuration *prometheus.HistogramVec + ZFSDestroyDuration *prometheus.HistogramVec } func init() { @@ -16,12 +16,6 @@ func init() { Name: "list_filesystem_versions_duration", Help: "Seconds it took for listing the versions of a given filesystem", }, []string{"filesystem"}) - prom.ZFSDestroyFilesystemVersionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "zrepl", - Subsystem: "zfs", - Name: "destroy_filesystem_version_duration", - Help: "Seconds it took to destroy a version of a given filesystem", - }, []string{"filesystem", "version_type"}) prom.ZFSSnapshotDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "zrepl", Subsystem: "zfs", @@ -34,20 +28,26 @@ func init() { Name: "bookmark_duration", Help: "Duration it took to bookmark a given snapshot", }, []string{"filesystem"}) + prom.ZFSDestroyDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "zrepl", + Subsystem: "zfs", + Name: "destroy_duration", + Help: "Duration it took to destroy a dataset", + }, []string{"dataset_type", "filesystem"}) } func PrometheusRegister(registry prometheus.Registerer) error { if err := registry.Register(prom.ZFSListFilesystemVersionDuration); err != nil { return err } - if err := registry.Register(prom.ZFSDestroyFilesystemVersionDuration); err != nil { - return err - } if err := registry.Register(prom.ZFSBookmarkDuration); err != nil { return err } if err := registry.Register(prom.ZFSSnapshotDuration); err != nil { return err } + if err := registry.Register(prom.ZFSDestroyDuration); err != nil { + return err + } return nil } diff --git a/zfs/replication_history.go b/zfs/replication_history.go new file mode 100644 index 0000000..2ad5abf --- /dev/null +++ b/zfs/replication_history.go @@ -0,0 +1,58 @@ +package zfs + +import ( + "fmt" + "github.com/pkg/errors" + "strconv" +) + +const ReplicationCursorBookmarkName = "zrepl_replication_cursor" + +// may return nil for both values, indicating there is no cursor +func ZFSGetReplicationCursor(fs *DatasetPath) (*FilesystemVersion, error) { + versions, err := ZFSListFilesystemVersions(fs, nil) + if err != nil { + return nil, err + } + for _, v := range versions { + if v.Type == Bookmark && v.Name == ReplicationCursorBookmarkName { + return &v, nil + } + } + return nil, nil +} + +func ZFSSetReplicationCursor(fs *DatasetPath, snapname string) (guid uint64, err error) { + snapPath := fmt.Sprintf("%s@%s", fs.ToString(), snapname) + propsSnap, err := zfsGet(snapPath, []string{"createtxg", "guid"}, sourceAny) + if err != nil { + return 0, err + } + snapGuid, err := strconv.ParseUint(propsSnap.Get("guid"), 10, 64) + bookmarkPath := fmt.Sprintf("%s#%s", fs.ToString(), ReplicationCursorBookmarkName) + propsBookmark, err := zfsGet(bookmarkPath, []string{"createtxg"}, sourceAny) + _, bookmarkNotExistErr := err.(*DatasetDoesNotExist) + if err != nil && !bookmarkNotExistErr { + return 0, err + } + if err == nil { + bookmarkTxg, err := strconv.ParseUint(propsBookmark.Get("createtxg"), 10, 64) + if err != nil { + return 0, errors.Wrap(err, "cannot parse bookmark createtxg") + } + snapTxg, err := strconv.ParseUint(propsSnap.Get("createtxg"), 10, 64) + if err != nil { + return 0, errors.Wrap(err, "cannot parse snapshot createtxg") + } + if snapTxg < bookmarkTxg { + return 0, errors.New("replication cursor can only be advanced, not set back") + } + if err := ZFSDestroy(bookmarkPath); err != nil { // FIXME make safer by using new temporary bookmark, then rename, possible with channel programs + return 0, err + } + } + if err := ZFSBookmark(fs, snapname, ReplicationCursorBookmarkName); err != nil { + return 0, err + } + return snapGuid, nil +} diff --git a/zfs/resume_token.go b/zfs/resume_token.go new file mode 100644 index 0000000..e83c8f2 --- /dev/null +++ b/zfs/resume_token.go @@ -0,0 +1,121 @@ +package zfs + +import ( + "context" + "errors" + "os/exec" + "regexp" + "strconv" + "time" +) + +type ResumeToken struct { + HasFromGUID, HasToGUID bool + FromGUID, ToGUID uint64 + // no support for other fields +} + +var resumeTokenNVListRE = regexp.MustCompile(`\t(\S+) = (.*)`) +var resumeTokenContentsRE = regexp.MustCompile(`resume token contents:\nnvlist version: 0`) +var resumeTokenIsCorruptRE = regexp.MustCompile(`resume token is corrupt`) + +var ResumeTokenCorruptError = errors.New("resume token is corrupt") +var ResumeTokenDecodingNotSupported = errors.New("zfs binary does not allow decoding resume token or zrepl cannot scrape zfs output") +var ResumeTokenParsingError = errors.New("zrepl cannot parse resume token values") + +// Abuse 'zfs send' to decode the resume token +// +// FIXME: implement nvlist unpacking in Go and read through libzfs_sendrecv.c +func ParseResumeToken(ctx context.Context, token string) (*ResumeToken, error) { + + // Example resume tokens: + // + // From a non-incremental send + // 1-bf31b879a-b8-789c636064000310a500c4ec50360710e72765a5269740f80cd8e4d3d28a534b18e00024cf86249f5459925acc802a8facbf243fbd3433858161f5ddb9ab1ae7c7466a20c97382e5f312735319180af2f3730cf58166953824c2cc0200cde81651 + + // From an incremental send + // 1-c49b979a2-e0-789c636064000310a501c49c50360710a715e5e7a69766a63040c1eabb735735ce8f8d5400b2d991d4e52765a5269740f82080219f96569c5ac2000720793624f9a4ca92d46206547964fd25f91057f09e37babb88c9bf5503499e132c9f97989bcac050909f9f63a80f34abc421096616007c881d4c + + // Resulting output of zfs send -nvt + // + //resume token contents: + //nvlist version: 0 + // fromguid = 0x595d9f81aa9dddab + // object = 0x1 + // offset = 0x0 + // bytes = 0x0 + // toguid = 0x854f02a2dd32cf0d + // toname = pool1/test@b + //cannot resume send: 'pool1/test@b' used in the initial send no longer exists + + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + cmd := exec.CommandContext(ctx, ZFS_BINARY, "send", "-nvt", string(token)) + output, err := cmd.CombinedOutput() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if !exitErr.Exited() { + return nil, err + } + // we abuse zfs send for decoding, the exit error may be due to + // a) the token being from a third machine + // b) it no longer exists on the machine where + } else { + return nil, err + } + } + + if !resumeTokenContentsRE.Match(output) { + if resumeTokenIsCorruptRE.Match(output) { + return nil, ResumeTokenCorruptError + } + return nil, ResumeTokenDecodingNotSupported + } + + matches := resumeTokenNVListRE.FindAllStringSubmatch(string(output), -1) + if matches == nil { + return nil, ResumeTokenDecodingNotSupported + } + + rt := &ResumeToken{} + + for _, m := range matches { + attr, val := m[1], m[2] + switch attr { + case "fromguid": + rt.FromGUID, err = strconv.ParseUint(val, 0, 64) + if err != nil { + return nil, ResumeTokenParsingError + } + rt.HasFromGUID = true + case "toguid": + rt.ToGUID, err = strconv.ParseUint(val, 0, 64) + if err != nil { + return nil, ResumeTokenParsingError + } + rt.HasToGUID = true + } + } + + if !rt.HasToGUID { + return nil, ResumeTokenDecodingNotSupported + } + + return rt, nil + +} + +func ZFSGetReceiveResumeToken(fs *DatasetPath) (string, error) { + const prop_receive_resume_token = "receive_resume_token" + props, err := ZFSGet(fs, []string{prop_receive_resume_token}) + if err != nil { + return "", err + } + res := props.m[prop_receive_resume_token] + if res == "-" { + return "", nil + } else { + return res, nil + } + +} diff --git a/zfs/resume_token_test.go b/zfs/resume_token_test.go new file mode 100644 index 0000000..4a35595 --- /dev/null +++ b/zfs/resume_token_test.go @@ -0,0 +1,66 @@ +package zfs_test + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/zrepl/zrepl/zfs" + "testing" +) + +type ResumeTokenTest struct { + Msg string + Token string + ExpectToken *zfs.ResumeToken + ExpectError error +} + +func (rtt *ResumeTokenTest) Test(t *testing.T) { + t.Log(rtt.Msg) + res, err := zfs.ParseResumeToken(context.TODO(), rtt.Token) + + if rtt.ExpectError != nil { + assert.EqualValues(t, rtt.ExpectError, err) + return + } + if rtt.ExpectToken != nil { + assert.Nil(t, err) + assert.EqualValues(t, rtt.ExpectToken, res) + return + } +} + +func TestParseResumeToken(t *testing.T) { + + t.SkipNow() // FIXME not compatible with docker + + tbl := []ResumeTokenTest{ + { + Msg: "normal send (non-incremental)", + Token: `1-bf31b879a-b8-789c636064000310a500c4ec50360710e72765a5269740f80cd8e4d3d28a534b18e00024cf86249f5459925acc802a8facbf243fbd3433858161f5ddb9ab1ae7c7466a20c97382e5f312735319180af2f3730cf58166953824c2cc0200cde81651`, + ExpectToken: &zfs.ResumeToken{ + HasToGUID: true, + ToGUID: 0x595d9f81aa9dddab, + }, + }, + { + Msg: "normal send (incremental)", + Token: `1-c49b979a2-e0-789c636064000310a501c49c50360710a715e5e7a69766a63040c1eabb735735ce8f8d5400b2d991d4e52765a5269740f82080219f96569c5ac2000720793624f9a4ca92d46206547964fd25f91057f09e37babb88c9bf5503499e132c9f97989bcac050909f9f63a80f34abc421096616007c881d4c`, + ExpectToken: &zfs.ResumeToken{ + HasToGUID: true, + ToGUID: 0x854f02a2dd32cf0d, + HasFromGUID: true, + FromGUID: 0x595d9f81aa9dddab, + }, + }, + { + Msg: "corrupted token", + Token: `1-bf31b879a-b8-789c636064000310a500c4ec50360710e72765a5269740f80cd8e4d3d28a534b18e00024cf86249f5459925acc802a8facbf243fbd3433858161f5ddb9ab1ae7c7466a20c97382e5f312735319180af2f3730cf58166953824c2cc0200cd12345`, + ExpectError: zfs.ResumeTokenCorruptError, + }, + } + + for _, test := range tbl { + test.Test(t) + } + +} diff --git a/zfs/versions.go b/zfs/versions.go index 1ef9e20..29303c0 100644 --- a/zfs/versions.go +++ b/zfs/versions.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/prometheus/client_golang/prometheus" + "io" "strconv" "strings" "time" @@ -33,6 +34,26 @@ func (t VersionType) String() string { return string(t) } +func DecomposeVersionString(v string) (fs string, versionType VersionType, name string, err error) { + if len(v) < 3 { + err = errors.New(fmt.Sprintf("snapshot or bookmark name implausibly short: %s", v)) + return + } + + snapSplit := strings.SplitN(v, "@", 2) + bookmarkSplit := strings.SplitN(v, "#", 2) + if len(snapSplit)*len(bookmarkSplit) != 2 { + err = errors.New(fmt.Sprintf("dataset cannot be snapshot and bookmark at the same time: %s", v)) + return + } + + if len(snapSplit) == 2 { + return snapSplit[0], Snapshot, snapSplit[1], nil + } else { + return bookmarkSplit[0], Bookmark, bookmarkSplit[1], nil + } +} + type FilesystemVersion struct { Type VersionType @@ -63,7 +84,7 @@ func (v FilesystemVersion) ToAbsPath(p *DatasetPath) string { } type FilesystemVersionFilter interface { - Filter(fsv FilesystemVersion) (accept bool, err error) + Filter(t VersionType, name string) (accept bool, err error) } func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) (res []FilesystemVersion, err error) { @@ -82,31 +103,21 @@ func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) res = make([]FilesystemVersion, 0) for listResult := range listResults { - if listResult.err != nil { - return nil, listResult.err + if listResult.Err != nil { + if listResult.Err == io.ErrUnexpectedEOF { + // Since we specified the fs on the command line, we'll treat this like the filesystem doesn't exist + return []FilesystemVersion{}, nil + } + return nil, listResult.Err } - line := listResult.fields - - if len(line[0]) < 3 { - err = errors.New(fmt.Sprintf("snapshot or bookmark name implausibly short: %s", line[0])) - return - } - - snapSplit := strings.SplitN(line[0], "@", 2) - bookmarkSplit := strings.SplitN(line[0], "#", 2) - if len(snapSplit)*len(bookmarkSplit) != 2 { - err = errors.New(fmt.Sprintf("dataset cannot be snapshot and bookmark at the same time: %s", line[0])) - return - } + line := listResult.Fields var v FilesystemVersion - if len(snapSplit) == 2 { - v.Name = snapSplit[1] - v.Type = Snapshot - } else { - v.Name = bookmarkSplit[1] - v.Type = Bookmark + + _, v.Type, v.Name, err = DecomposeVersionString(line[0]) + if err != nil { + return nil, err } if v.Guid, err = strconv.ParseUint(line[1], 10, 64); err != nil { @@ -129,7 +140,7 @@ func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) accept := true if filter != nil { - accept, err = filter.Filter(v) + accept, err = filter.Filter(v.Type, v.Name) if err != nil { err = fmt.Errorf("error executing filter: %s", err) return nil, err @@ -143,10 +154,7 @@ func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) return } -func ZFSDestroyFilesystemVersion(filesystem *DatasetPath, version FilesystemVersion) (err error) { - - promTimer := prometheus.NewTimer(prom.ZFSDestroyFilesystemVersionDuration.WithLabelValues(filesystem.ToString(), version.Type.String())) - defer promTimer.ObserveDuration() +func ZFSDestroyFilesystemVersion(filesystem *DatasetPath, version *FilesystemVersion) (err error) { datasetPath := version.ToAbsPath(filesystem) diff --git a/zfs/zfs.go b/zfs/zfs.go index c0541ef..faae6e2 100644 --- a/zfs/zfs.go +++ b/zfs/zfs.go @@ -14,6 +14,8 @@ import ( "github.com/problame/go-rwccmd" "github.com/prometheus/client_golang/prometheus" "github.com/zrepl/zrepl/util" + "regexp" + "strconv" ) type DatasetPath struct { @@ -140,7 +142,7 @@ type ZFSError struct { } func (e ZFSError) Error() string { - return fmt.Sprintf("zfs exited with error: %s", e.WaitErr.Error()) + return fmt.Sprintf("zfs exited with error: %s\nstderr:\n%s", e.WaitErr.Error(), e.Stderr) } var ZFS_BINARY string = "zfs" @@ -195,13 +197,13 @@ func ZFSList(properties []string, zfsArgs ...string) (res [][]string, err error) } type ZFSListResult struct { - fields []string - err error + Fields []string + Err error } // ZFSListChan executes `zfs list` and sends the results to the `out` channel. // The `out` channel is always closed by ZFSListChan: -// If an error occurs, it is closed after sending a result with the err field set. +// If an error occurs, it is closed after sending a result with the Err field set. // If no error occurs, it is just closed. // If the operation is cancelled via context, the channel is just closed. // @@ -256,15 +258,56 @@ func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []strin return } -func ZFSSend(fs *DatasetPath, from, to *FilesystemVersion) (stream io.Reader, err error) { +func validateRelativeZFSVersion(s string) error { + if len(s) <= 1 { + return errors.New("version must start with a delimiter char followed by at least one character") + } + if !(s[0] == '#' || s[0] == '@') { + return errors.New("version name starts with invalid delimiter char") + } + // FIXME whitespace check... + return nil +} + +func validateZFSFilesystem(fs string) error { + if len(fs) < 1 { + return errors.New("filesystem path must have length > 0") + } + return nil +} + +func absVersion(fs, v string) (full string, err error) { + if err := validateZFSFilesystem(fs); err != nil { + return "", err + } + if err := validateRelativeZFSVersion(v); err != nil { + return "", err + } + return fmt.Sprintf("%s%s", fs, v), nil +} + +func ZFSSend(fs string, from, to string) (stream io.ReadCloser, err error) { + + fromV, err := absVersion(fs, from) + if err != nil { + return nil, err + } + + toV := "" + if to != "" { + toV, err = absVersion(fs, to) + if err != nil { + return nil, err + } + } args := make([]string, 0) args = append(args, "send") - if to == nil { // Initial - args = append(args, from.ToAbsPath(fs)) + if toV == "" { // Initial + args = append(args, fromV) } else { - args = append(args, "-i", from.ToAbsPath(fs), to.ToAbsPath(fs)) + args = append(args, "-i", fromV, toV) } stream, err = util.RunIOCommand(ZFS_BINARY, args...) @@ -272,14 +315,74 @@ func ZFSSend(fs *DatasetPath, from, to *FilesystemVersion) (stream io.Reader, er return } -func ZFSRecv(fs *DatasetPath, stream io.Reader, additionalArgs ...string) (err error) { +var BookmarkSizeEstimationNotSupported error = fmt.Errorf("size estimation is not supported for bookmarks") + +// May return BookmarkSizeEstimationNotSupported as err if from is a bookmark. +func ZFSSendDry(fs string, from, to string) (size int64, err error) { + + fromV, err := absVersion(fs, from) + if err != nil { + return 0, err + } + + toV := "" + if to != "" { + toV, err = absVersion(fs, to) + if err != nil { + return 0, err + } + } + + if strings.Contains(fromV, "#") { + /* TODO: + * ZFS at the time of writing does not support dry-run send because size-estimation + * uses fromSnap's deadlist. However, for a bookmark, that deadlist no longer exists. + * Redacted send & recv will bring this functionality, see + * https://github.com/openzfs/openzfs/pull/484 + */ + return 0, BookmarkSizeEstimationNotSupported + } + + args := make([]string, 0) + args = append(args, "send", "-n", "-v", "-P") + + if toV == "" { // Initial + args = append(args, fromV) + } else { + args = append(args, "-i", fromV, toV) + } + + cmd := exec.Command(ZFS_BINARY, args...) + output, err := cmd.CombinedOutput() + if err != nil { + return 0, err + } + o := string(output) + lines := strings.Split(o, "\n") + if len(lines) < 2 { + return 0, errors.New("zfs send -n did not return the expected number of lines") + } + fields := strings.Fields(lines[1]) + if len(fields) != 2 { + return 0, errors.New("zfs send -n returned unexpexted output") + } + + size, err = strconv.ParseInt(fields[1], 10, 64) + return size, err +} + +func ZFSRecv(fs string, stream io.Reader, additionalArgs ...string) (err error) { + + if err := validateZFSFilesystem(fs); err != nil { + return err + } args := make([]string, 0) args = append(args, "recv") if len(args) > 0 { args = append(args, additionalArgs...) } - args = append(args, fs.ToString()) + args = append(args, fs) cmd := exec.Command(ZFS_BINARY, args...) @@ -310,6 +413,27 @@ func ZFSRecv(fs *DatasetPath, stream io.Reader, additionalArgs ...string) (err e return nil } +func ZFSRecvWriter(fs *DatasetPath, additionalArgs ...string) (io.WriteCloser, error) { + + args := make([]string, 0) + args = append(args, "recv") + if len(args) > 0 { + args = append(args, additionalArgs...) + } + args = append(args, fs.ToString()) + + cmd, err := util.NewIOCommand(ZFS_BINARY, args, 1024) + if err != nil { + return nil, err + } + + if err = cmd.Start(); err != nil { + return nil, err + } + + return cmd.Stdin, nil +} + type ZFSProperties struct { m map[string]string } @@ -322,6 +446,10 @@ func (p *ZFSProperties) Set(key, val string) { p.m[key] = val } +func (p *ZFSProperties) Get(key string) string { + return p.m[key] +} + func (p *ZFSProperties) appendArgs(args *[]string) (err error) { for prop, val := range p.m { if strings.Contains(prop, "=") { @@ -333,14 +461,17 @@ func (p *ZFSProperties) appendArgs(args *[]string) (err error) { } func ZFSSet(fs *DatasetPath, props *ZFSProperties) (err error) { + return zfsSet(fs.ToString(), props) +} +func zfsSet(path string, props *ZFSProperties) (err error) { args := make([]string, 0) args = append(args, "set") err = props.appendArgs(&args) if err != nil { return err } - args = append(args, fs.ToString()) + args = append(args, path) cmd := exec.Command(ZFS_BINARY, args...) @@ -361,8 +492,101 @@ func ZFSSet(fs *DatasetPath, props *ZFSProperties) (err error) { return } +func ZFSGet(fs *DatasetPath, props []string) (*ZFSProperties, error) { + return zfsGet(fs.ToString(), props, sourceAny) +} + +var zfsGetDatasetDoesNotExistRegexp = regexp.MustCompile(`^cannot open '(\S+)': (dataset does not exist|no such pool or dataset)`) + +type DatasetDoesNotExist struct { + Path string +} + +func (d *DatasetDoesNotExist) Error() string { return fmt.Sprintf("dataset %q does not exist", d.Path) } + +type zfsPropertySource uint + +const ( + sourceLocal zfsPropertySource = 1 << iota + sourceDefault + sourceInherited + sourceNone + sourceTemporary + + sourceAny zfsPropertySource = ^zfsPropertySource(0) +) + +func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string { + prefixes := make([]string, 0, 5) + if s&sourceLocal != 0 {prefixes = append(prefixes, "local")} + if s&sourceDefault != 0 {prefixes = append(prefixes, "default")} + if s&sourceInherited != 0 {prefixes = append(prefixes, "inherited")} + if s&sourceNone != 0 {prefixes = append(prefixes, "-")} + if s&sourceTemporary != 0 { prefixes = append(prefixes, "temporary")} + return prefixes +} + +func zfsGet(path string, props []string, allowedSources zfsPropertySource) (*ZFSProperties, error) { + args := []string{"get", "-Hp", "-o", "property,value,source", strings.Join(props, ","), path} + cmd := exec.Command(ZFS_BINARY, args...) + stdout, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if exitErr.Exited() { + // screen-scrape output + if sm := zfsGetDatasetDoesNotExistRegexp.FindSubmatch(exitErr.Stderr); sm != nil { + if string(sm[1]) == path { + return nil, &DatasetDoesNotExist{path} + } + } + } + } + return nil, err + } + o := string(stdout) + lines := strings.Split(o, "\n") + if len(lines) < 1 || // account for newlines + len(lines)-1 != len(props) { + return nil, fmt.Errorf("zfs get did not return the number of expected property values") + } + res := &ZFSProperties{ + make(map[string]string, len(lines)), + } + allowedPrefixes := allowedSources.zfsGetSourceFieldPrefixes() + for _, line := range lines[:len(lines)-1] { + fields := strings.FieldsFunc(line, func(r rune) bool { + return r == '\t' + }) + if len(fields) != 3 { + return nil, fmt.Errorf("zfs get did not return property,value,source tuples") + } + for _, p := range allowedPrefixes { + if strings.HasPrefix(fields[2],p) { + res.m[fields[0]] = fields[1] + break + } + } + } + return res, nil +} + func ZFSDestroy(dataset string) (err error) { + var dstype, filesystem string + idx := strings.IndexAny(dataset, "@#") + if idx == -1 { + dstype = "filesystem" + filesystem = dataset + } else { + switch dataset[idx] { + case '@': dstype = "snapshot" + case '#': dstype = "bookmark" + } + filesystem = dataset[:idx] + } + + defer prometheus.NewTimer(prom.ZFSDestroyDuration.WithLabelValues(dstype, filesystem)) + cmd := exec.Command(ZFS_BINARY, "destroy", dataset) stderr := bytes.NewBuffer(make([]byte, 0, 1024)) diff --git a/zfs/zfs_test.go b/zfs/zfs_test.go index cf89b40..b46a36a 100644 --- a/zfs/zfs_test.go +++ b/zfs/zfs_test.go @@ -6,6 +6,8 @@ import ( ) func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) { + t.SkipNow() // FIXME ZFS_BINARY does not work if tests run in parallel + var err error ZFS_BINARY = "./test_helpers/zfs_failer.sh" @@ -28,3 +30,41 @@ func TestDatasetPathTrimNPrefixComps(t *testing.T) { p.TrimNPrefixComps((1)) assert.True(t, p.Empty(), "empty trimming shouldn't do harm") } + +func TestZFSPropertySource(t *testing.T) { + + tcs := []struct{ + in zfsPropertySource + exp []string + }{ + { + in: sourceAny, + exp: []string{"local", "default", "inherited", "-", "temporary"}, + }, + { + in: sourceTemporary, + exp: []string{"temporary"}, + }, + { + in: sourceLocal|sourceInherited, + exp: []string{"local", "inherited"}, + }, + } + + toSet := func(in []string) map[string]struct{} { + m := make(map[string]struct{}, len(in)) + for _, s := range in { + m[s] = struct{}{} + } + return m + } + + for _, tc := range tcs { + + res := tc.in.zfsGetSourceFieldPrefixes() + resSet := toSet(res) + expSet := toSet(tc.exp) + assert.Equal(t, expSet, resSet) + } + +}