adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move implementation to internal/ directory (#828)

Browse Source
This commit is contained in:
Christian Schwarz
2024-10-18 19:21:17 +02:00
committed by GitHub
parent b9b9ad10cf
commit 908807bd59
360 changed files with 507 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/client/configcheck.go
+131
View File
@@ -0,0 +1,131 @@
package client
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/kr/pretty"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon/job"
"github.com/zrepl/zrepl/internal/daemon/logging"
"github.com/zrepl/zrepl/internal/logger"
)
var configcheckArgs struct {
format string
what string
skipCertCheck bool
}
var ConfigcheckCmd = &cli.Subcommand{
Use: "configcheck",
Short: "check if config can be parsed without errors",
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
formatMap := map[string]func(interface{}){
"": func(i interface{}) {},
"pretty": func(i interface{}) {
if _, err := pretty.Println(i); err != nil {
panic(err)
}
},
"json": func(i interface{}) {
if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
panic(err)
}
},
"yaml": func(i interface{}) {
if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
panic(err)
}
},
}
formatter, ok := formatMap[configcheckArgs.format]
if !ok {
return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
}
var hadErr bool
parseFlags := config.ParseFlagsNone
if configcheckArgs.skipCertCheck {
parseFlags |= config.ParseFlagsNoCertCheck
}
// further: try to build jobs
confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
if err != nil {
err := errors.Wrap(err, "cannot build jobs from config")
if configcheckArgs.what == "jobs" {
return err
} else {
fmt.Fprintf(os.Stderr, "%s\n", err)
confJobs = nil
hadErr = true
}
}
// further: try to build logging outlets
outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
if err != nil {
err := errors.Wrap(err, "cannot build logging from config")
if configcheckArgs.what == "logging" {
return err
} else {
fmt.Fprintf(os.Stderr, "%s\n", err)
outlets = nil
hadErr = true
}
}
whatMap := map[string]func(){
"all": func() {
o := struct {
config *config.Config
jobs []job.Job
logging *logger.Outlets
}{
subcommand.Config(),
confJobs,
outlets,
}
formatter(o)
},
"config": func() {
formatter(subcommand.Config())
},
"jobs": func() {
formatter(confJobs)
},
"logging": func() {
formatter(outlets)
},
}
wf, ok := whatMap[configcheckArgs.what]
if !ok {
return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
}
wf()
if hadErr {
return fmt.Errorf("config parsing failed")
} else {
return nil
}
},
}
internal/client/jsonclient.go
+49
View File
@@ -0,0 +1,49 @@
package client
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"github.com/pkg/errors"
)
func controlHttpClient(sockpath string) (client http.Client, err error) {
return http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", sockpath)
},
},
}, nil
}
func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
var buf bytes.Buffer
encodeErr := json.NewEncoder(&buf).Encode(req)
if encodeErr != nil {
return encodeErr
}
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
var msg bytes.Buffer
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
return errors.Errorf("%s", msg.String())
}
decodeError := json.NewDecoder(resp.Body).Decode(&res)
if decodeError != nil {
return decodeError
}
return nil
}
internal/client/migrate.go
+268
View File
@@ -0,0 +1,268 @@
package client
import (
"context"
"fmt"
"github.com/fatih/color"
"github.com/kr/pretty"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/daemon/job"
"github.com/zrepl/zrepl/internal/endpoint"
"github.com/zrepl/zrepl/internal/zfs"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
)
var (
MigrateCmd = &cli.Subcommand{
Use: "migrate",
Short: "perform migration of the on-disk / zfs properties",
SetupSubcommands: func() []*cli.Subcommand {
return migrations
},
}
)
var migrations = []*cli.Subcommand{
&cli.Subcommand{
Use: "0.0.X:0.1:placeholder",
Run: doMigratePlaceholder0_1,
SetupFlags: func(f *pflag.FlagSet) {
f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
},
},
&cli.Subcommand{
Use: "replication-cursor:v1-v2",
Run: doMigrateReplicationCursor,
SetupFlags: func(f *pflag.FlagSet) {
f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
},
},
}
var migratePlaceholder0_1Args struct {
dryRun bool
}
func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
if len(args) != 0 {
return fmt.Errorf("migration does not take arguments, got %v", args)
}
cfg := sc.Config()
allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
if err != nil {
return errors.Wrap(err, "cannot list filesystems")
}
type workItem struct {
jobName string
rootFS *zfs.DatasetPath
fss []*zfs.DatasetPath
}
var wis []workItem
for i, j := range cfg.Jobs {
var rfsS string
switch job := j.Ret.(type) {
case *config.SinkJob:
rfsS = job.RootFS
case *config.PullJob:
rfsS = job.RootFS
default:
fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
continue
}
rfs, err := zfs.NewDatasetPath(rfsS)
if err != nil {
return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
}
var fss []*zfs.DatasetPath
for _, fs := range allFSS {
if fs.HasPrefix(rfs) {
fss = append(fss, fs)
}
}
wis = append(wis, workItem{j.Name(), rfs, fss})
}
for _, wi := range wis {
fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
if len(wi.fss) == 0 {
fmt.Printf("\tno filesystems\n")
continue
}
for _, fs := range wi.fss {
fmt.Printf("\t%q ... ", fs.ToString())
r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
if err != nil {
fmt.Printf("error: %s\n", err)
} else if !r.NeedsModification {
fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
} else {
fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
}
}
}
return nil
}
var migrateReplicationCursorArgs struct {
dryRun bool
}
var bold = color.New(color.Bold)
var succ = color.New(color.FgGreen)
var fail = color.New(color.FgRed)
var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
if len(args) != 0 {
return fmt.Errorf("migration does not take arguments, got %v", args)
}
cfg := sc.Config()
jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
if err != nil {
fmt.Printf("cannot parse config:\n%s\n\n", err)
fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
return fmt.Errorf("exiting migration after error")
}
v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
for i, j := range cfg.Jobs {
if jobs[i].Name() != j.Name() {
panic("implementation error")
}
switch j.Ret.(type) {
case *config.PushJob:
v1cursorJobs = append(v1cursorJobs, jobs[i])
case *config.SourceJob:
v1cursorJobs = append(v1cursorJobs, jobs[i])
default:
fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
continue
}
}
// scan all filesystems for v1 replication cursors
fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
if err != nil {
return errors.Wrap(err, "list filesystems")
}
var hadError bool
for _, fs := range fss {
bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
if err == migrateReplicationCursorSkipSentinel {
bold.Printf("FILESYSTEM SKIPPED\n")
} else if err != nil {
hadError = true
fail.Printf("MIGRATION FAILED: %s\n", err)
} else {
succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
}
}
if hadError {
fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
return errors.Errorf("")
}
return nil
}
func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
var owningJob job.Job = nil
for _, job := range v1CursorJobs {
conf := job.SenderConfig()
if conf == nil {
continue
}
pass, err := conf.FSF.Filter(fs)
if err != nil {
return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
}
if !pass {
continue
}
if owningJob != nil {
return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
}
owningJob = job
}
if owningJob == nil {
fmt.Printf("no job's Filesystems filter matches\n")
return migrateReplicationCursorSkipSentinel
}
fmt.Printf("identified owning job %q\n", owningJob.Name())
bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
Types: zfs.Bookmarks,
})
if err != nil {
return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
}
var oldCursor *zfs.FilesystemVersion
for i, fsv := range bookmarks {
_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
if err != endpoint.ErrV1ReplicationCursor {
continue
}
if oldCursor != nil {
fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
}
oldCursor = &bookmarks[i]
}
if oldCursor == nil {
bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
return migrateReplicationCursorSkipSentinel
}
fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
if err != nil {
return errors.Wrapf(err, "get most recent v2 replication cursor")
}
if mostRecentNew == nil {
return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
}
fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
}
fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
if migrateReplicationCursorArgs.dryRun {
succ.Printf("DRY RUN\n")
} else {
if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
return err
}
}
return nil
}
internal/client/migrate_test.go
+16
View File
@@ -0,0 +1,16 @@
package client
import "testing"
func TestMigrationsUnambiguousNames(t *testing.T) {
names := make(map[string]bool)
for _, mig := range migrations {
if _, ok := names[mig.Use]; ok {
t.Errorf("duplicate migration name %q", mig.Use)
t.FailNow()
return
} else {
names[mig.Use] = true
}
}
}
internal/client/pprof.go
+10
View File
@@ -0,0 +1,10 @@
package client
import "github.com/zrepl/zrepl/internal/cli"
var PprofCmd = &cli.Subcommand{
Use: "pprof",
SetupSubcommands: func() []*cli.Subcommand {
return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
},
}
internal/client/pprof_activity_trace.go
+44
View File
@@ -0,0 +1,44 @@
package client
import (
"context"
"io"
"log"
"os"
"golang.org/x/net/websocket"
"github.com/zrepl/zrepl/internal/cli"
)
var pprofActivityTraceCmd = &cli.Subcommand{
Use: "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
Run: runPProfActivityTrace,
}
func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
log := log.New(os.Stderr, "", 0)
die := func() {
log.Printf("exiting after error")
os.Exit(1)
}
if len(args) != 1 {
log.Printf("exactly one positional argument is required")
die()
}
url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
log.Printf("attaching to activity trace stream %s", url)
ws, err := websocket.Dial(url, "", url)
if err != nil {
log.Printf("error: %s", err)
die()
}
_, err = io.Copy(os.Stdout, ws)
return err
}
internal/client/pprof_listen.go
+68
View File
@@ -0,0 +1,68 @@
package client
import (
"context"
"errors"
"log"
"os"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon"
)
var pprofListenCmd struct {
daemon.PprofServerControlMsg
}
var PprofListenCmd = &cli.Subcommand{
Use: "listen off | [on TCP_LISTEN_ADDRESS]",
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
if len(args) < 1 {
goto enargs
}
switch args[0] {
case "on":
pprofListenCmd.Run = true
if len(args) != 2 {
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
}
pprofListenCmd.HttpListenAddress = args[1]
case "off":
if len(args) != 1 {
goto enargs
}
pprofListenCmd.Run = false
}
RunPProf(subcommand.Config())
return nil
enargs:
return errors.New("invalid number of positional arguments")
},
}
func RunPProf(conf *config.Config) {
log := log.New(os.Stderr, "", 0)
die := func() {
log.Printf("exiting after error")
os.Exit(1)
}
log.Printf("connecting to zrepl daemon")
httpc, err := controlHttpClient(conf.Global.Control.SockPath)
if err != nil {
log.Printf("error creating http client: %s", err)
die()
}
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
if err != nil {
log.Printf("error sending control message: %s", err)
die()
}
log.Printf("finished")
}
internal/client/signal.go
+42
View File
@@ -0,0 +1,42 @@
package client
import (
"context"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon"
)
var SignalCmd = &cli.Subcommand{
Use: "signal [wakeup|reset] JOB",
Short: "wake up a job from wait state or abort its current invocation",
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runSignalCmd(subcommand.Config(), args)
},
}
func runSignalCmd(config *config.Config, args []string) error {
if len(args) != 2 {
return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
}
httpc, err := controlHttpClient(config.Global.Control.SockPath)
if err != nil {
return err
}
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
struct {
Name string
Op string
}{
Name: args[1],
Op: args[0],
},
struct{}{},
)
return err
}
internal/client/status/client/client.go
+111
View File
@@ -0,0 +1,111 @@
package client
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/daemon"
)
type Client struct {
h *http.Client
}
func New(network, addr string) (*Client, error) {
httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
if err != nil {
return nil, err
}
return &Client{httpc}, nil
}
func (c *Client) Status() (s daemon.Status, _ error) {
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
struct{}{},
&s,
)
return s, err
}
func (c *Client) StatusRaw() ([]byte, error) {
var r json.RawMessage
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
if err != nil {
return nil, err
}
return r, nil
}
func (c *Client) signal(job, sig string) error {
return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
struct {
Name string
Op string
}{
Name: job,
Op: sig,
},
struct{}{},
)
}
func (c *Client) SignalWakeup(job string) error {
return c.signal(job, "wakeup")
}
func (c *Client) SignalReset(job string) error {
return c.signal(job, "reset")
}
func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
return &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return dialfunc(ctx)
},
},
}, nil
}
func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
var buf bytes.Buffer
encodeErr := json.NewEncoder(&buf).Encode(req)
if encodeErr != nil {
return encodeErr
}
hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
if err != nil {
return err
}
hreq.Header.Set("Content-Type", "application/json")
// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
// Found this by watching http.Server.ConnState changes, then found
// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
hreq.Close = true
resp, err := c.Do(hreq)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
var msg bytes.Buffer
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
return errors.Errorf("%s", msg.String())
}
decodeError := json.NewDecoder(resp.Body).Decode(&res)
if decodeError != nil {
return decodeError
}
return nil
}
internal/client/status/status.go
+97
View File
@@ -0,0 +1,97 @@
package status
import (
"context"
"os"
"time"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/client/status/client"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon"
"github.com/zrepl/zrepl/internal/util/choices"
)
type Client interface {
Status() (daemon.Status, error)
StatusRaw() ([]byte, error)
SignalWakeup(job string) error
SignalReset(job string) error
}
type statusFlags struct {
Mode choices.Choices
Job string
Delay time.Duration
}
var statusv2Flags statusFlags
type statusv2Mode int
const (
StatusV2ModeInteractive statusv2Mode = 1 + iota
StatusV2ModeDump
StatusV2ModeRaw
StatusV2ModeLegacy
)
var Subcommand = &cli.Subcommand{
Use: "status",
Short: "retrieve & display daemon status information",
SetupFlags: func(f *pflag.FlagSet) {
statusv2Flags.Mode.Init(
"interactive", StatusV2ModeInteractive,
"dump", StatusV2ModeDump,
"raw", StatusV2ModeRaw,
"legacy", StatusV2ModeLegacy,
)
statusv2Flags.Mode.SetTypeString("mode")
statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runStatusV2Command(ctx, subcommand.Config(), args)
},
}
func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
c, err := client.New("unix", config.Global.Control.SockPath)
if err != nil {
return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
}
mode := statusv2Flags.Mode.Value().(statusv2Mode)
if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
if err != nil {
panic(err)
}
rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
if err != nil {
panic(err)
}
return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
}
switch mode {
case StatusV2ModeInteractive:
return interactive(c, statusv2Flags)
case StatusV2ModeDump:
return dump(c, statusv2Flags.Job)
case StatusV2ModeRaw:
return raw(c)
case StatusV2ModeLegacy:
return legacy(c, statusv2Flags)
default:
panic("unreachable")
}
}
internal/client/status/status_dump.go
+70
View File
@@ -0,0 +1,70 @@
package status
import (
"fmt"
"os"
"strings"
"github.com/gdamore/tcell/v2"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
)
func dump(c Client, job string) error {
s, err := c.Status()
if err != nil {
return err
}
if job != "" {
if _, ok := s.Jobs[job]; !ok {
return errors.Errorf("job %q not found", job)
}
}
width := (1 << 31) - 1
wrap := false
hline := strings.Repeat("-", 80)
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
hline = strings.Repeat("-", width)
}
m := viewmodel.New()
params := viewmodel.Params{
Report: s.Jobs,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
m.Update(params)
for _, j := range m.Jobs() {
if job != "" && j.Name() != job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Println(m.SelectedJob().FullDescription())
if job != "" {
return nil
} else {
fmt.Println(hline)
}
}
return nil
}
internal/client/status/status_interactive.go
+347
View File
@@ -0,0 +1,347 @@
package status
import (
"fmt"
"regexp"
"strings"
"sync"
"time"
tview "code.rocketnine.space/tslocum/cview"
"github.com/gdamore/tcell/v2"
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
)
func interactive(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
jobDetailSplit := tview.NewFlex()
jobMenu := tview.NewTreeView()
jobMenuRoot := tview.NewTreeNode("jobs")
jobMenuRoot.SetSelectable(true)
jobMenu.SetRoot(jobMenuRoot)
jobMenu.SetCurrentNode(jobMenuRoot)
jobMenu.SetSelectedTextColor(tcell.ColorGreen)
jobTextDetail := tview.NewTextView()
jobTextDetail.SetWrap(false)
jobMenu.SetBorder(true)
jobTextDetail.SetBorder(true)
toolbarSplit := tview.NewFlex()
toolbarSplit.SetDirection(tview.FlexRow)
inputBarContainer := tview.NewFlex()
fsFilterInput := tview.NewInputField()
fsFilterInput.SetBorder(false)
fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
inputBarLabel := tview.NewTextView()
inputBarLabel.SetText("[::b]FILTER ")
inputBarLabel.SetDynamicColors(true)
inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
bottombar := tview.NewFlex()
bottombar.SetDirection(tview.FlexColumn)
bottombarDateView := tview.NewTextView()
bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
bottomBarStatus := tview.NewTextView()
bottomBarStatus.SetDynamicColors(true)
bottomBarStatus.SetTextAlign(tview.AlignRight)
bottombar.AddItem(bottomBarStatus, 0, 10, false)
toolbarSplit.AddItem(bottombar, 1, 0, false)
tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
var tabbable []tview.Primitive
tabbableActiveIndex := 0
tabbableRedraw := func() {
if len(tabbable) == 0 {
app.SetFocus(nil)
return
}
if tabbableActiveIndex >= len(tabbable) {
app.SetFocus(tabbable[0])
return
}
app.SetFocus(tabbable[tabbableActiveIndex])
}
tabbableCycle := func() {
if len(tabbable) == 0 {
return
}
tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
app.SetFocus(tabbable[tabbableActiveIndex])
tabbableRedraw()
}
jobMenuVisisble := false
reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
if jobMenuVisisble == setJobMenuVisible {
return
}
jobMenuVisisble = setJobMenuVisible
if setJobMenuVisible {
jobDetailSplit.RemoveItem(jobTextDetail)
jobDetailSplit.AddItem(jobMenu, 0, 1, true)
jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
tabbable = tabbableWithJobMenu
} else {
jobDetailSplit.RemoveItem(jobMenu)
tabbable = tabbableWithoutJobMenu
}
tabbableRedraw()
}
showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
preModalFocus := app.GetFocus()
m.SetDoneFunc(func(idx int, label string) {
if modalDoneFunc != nil {
modalDoneFunc(idx, label)
}
app.SetRoot(toolbarSplit, true)
app.SetFocus(preModalFocus)
app.Draw()
})
app.SetRoot(m, true)
app.Draw()
}
app.SetRoot(toolbarSplit, true)
// initial focus
tabbableActiveIndex = len(tabbable)
tabbableCycle()
reconfigureJobDetailSplit(true)
m := viewmodel.New()
params := &viewmodel.Params{
Report: nil,
SelectedJob: nil,
FSFilter: func(_ string) bool { return true },
DetailViewWidth: 100,
DetailViewWrap: false,
ShortKeybindingOverview: "[::b]Q[::-] quit [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines [::b]Shift+M[::-] toggle navbar [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
}
paramsMtx := &sync.Mutex{}
var redraw func()
viewmodelupdate := func(cb func(*viewmodel.Params)) {
paramsMtx.Lock()
defer paramsMtx.Unlock()
cb(params)
m.Update(*params)
}
redraw = func() {
jobs := m.Jobs()
if flag.Job != "" {
job_found := false
for _, job := range jobs {
if strings.Compare(flag.Job, job.Name()) == 0 {
jobs = []*viewmodel.Job{job}
job_found = true
break
}
}
if !job_found {
jobs = nil
}
}
redrawJobsList := false
var selectedJobN *tview.TreeNode
if len(jobMenuRoot.GetChildren()) == len(jobs) {
for i, jobN := range jobMenuRoot.GetChildren() {
if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
redrawJobsList = true
break
}
if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
selectedJobN = jobN
}
}
} else {
redrawJobsList = true
}
if redrawJobsList {
selectedJobN = nil
children := make([]*tview.TreeNode, len(jobs))
for i := range jobs {
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
jobN.SetReference(jobs[i])
jobN.SetSelectable(true)
children[i] = jobN
jobN.SetSelectedFunc(func() {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
})
})
if jobs[i] == m.SelectedJob() {
selectedJobN = jobN
}
}
jobMenuRoot.SetChildren(children)
}
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
jobMenu.SetCurrentNode(selectedJobN)
} else if selectedJobN == nil {
// select something, otherwise selection breaks (likely bug in tview)
jobMenu.SetCurrentNode(jobMenuRoot)
}
if selJ := m.SelectedJob(); selJ != nil {
jobTextDetail.SetText(selJ.FullDescription())
} else {
jobTextDetail.SetText("please select a job")
}
bottombardatestring := m.DateString()
bottombarDateView.SetText(bottombardatestring)
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
bottomBarStatus.SetText(m.BottomBarStatus())
app.Draw()
}
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
viewmodelupdate(func(p *viewmodel.Params) {
p.Report = st.Jobs
p.ReportFetchError = err
})
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
})
redraw()
jobTextDetail.ScrollToBeginning()
})
jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
app.SetFocus(jobTextDetail)
})
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
viewmodelupdate(func(p *viewmodel.Params) {
_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
})
return false
})
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
if e.Key() == tcell.KeyTab {
tabbableCycle()
return nil
}
if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == '/' {
if app.GetFocus() != fsFilterInput {
app.SetFocus(fsFilterInput)
}
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
reconfigureJobDetailSplit(!jobMenuVisisble)
return nil
}
if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
app.Stop()
}
if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
if !ok {
return nil
}
signals := []string{"wakeup", "reset"}
clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
sigMod := tview.NewModal()
sigMod.SetBackgroundColor(tcell.ColorDefault)
sigMod.SetBorder(true)
sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
sigMod.AddButtons(signals)
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
showModal(sigMod, func(idx int, _ string) {
go func() {
if idx == -1 {
return
}
err := clientFuncs[idx](job.Name())
if err != nil {
app.QueueUpdate(func() {
me := tview.NewModal()
me.SetText(fmt.Sprintf("signal error: %s", err))
me.AddButtons([]string{"Close"})
showModal(me, nil)
})
}
}()
})
}
return e
})
fsFilterInput.SetChangedFunc(func(searchterm string) {
viewmodelupdate(func(p *viewmodel.Params) {
p.FSFilter = func(fs string) bool {
r, err := regexp.Compile(searchterm)
if err != nil {
return true
}
return r.MatchString(fs)
}
})
redraw()
jobTextDetail.ScrollToBeginning()
})
fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEnter {
app.SetFocus(jobTextDetail)
return nil
}
return event
})
jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
// toggle wrapping
viewmodelupdate(func(p *viewmodel.Params) {
p.DetailViewWrap = !p.DetailViewWrap
})
redraw()
return nil
}
return event
})
return app.Run()
}
internal/client/status/status_legacy.go
+128
View File
@@ -0,0 +1,128 @@
package status
import (
"fmt"
"os"
"strings"
"sync"
"time"
tview "code.rocketnine.space/tslocum/cview"
"github.com/gdamore/tcell/v2"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
)
func legacy(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
textView := tview.NewTextView()
textView.SetWrap(true)
textView.SetScrollable(true) // so that it allows us to set scroll position
textView.SetScrollBarVisibility(tview.ScrollBarNever)
app.SetRoot(textView, true)
width := (1 << 31) - 1
wrap := false
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
}
paramsMtx := &sync.Mutex{}
params := viewmodel.Params{
Report: nil,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
redraw := func() {
textView.Clear()
paramsMtx.Lock()
defer paramsMtx.Unlock()
if params.ReportFetchError != nil {
fmt.Fprintln(textView, params.ReportFetchError.Error())
} else if params.Report != nil {
m := viewmodel.New()
m.Update(params)
for _, j := range m.Jobs() {
if flag.Job != "" && j.Name() != flag.Job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Fprintln(textView, m.SelectedJob().FullDescription())
if flag.Job != "" {
break
} else {
hline := strings.Repeat("-", params.DetailViewWidth)
fmt.Fprintln(textView, hline)
}
}
} else {
fmt.Fprintln(textView, "waiting for request results")
}
textView.ScrollToBeginning()
}
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
// sync resizes to `params`
paramsMtx.Lock()
_, _, newWidth, _ := textView.GetInnerRect()
if newWidth != params.DetailViewWidth {
params.DetailViewWidth = newWidth
app.QueueUpdateDraw(redraw)
}
paramsMtx.Unlock()
textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
return false
})
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
paramsMtx.Lock()
params.Report = st.Jobs
params.ReportFetchError = err
paramsMtx.Unlock()
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
return app.Run()
}
internal/client/status/status_raw.go
+18
View File
@@ -0,0 +1,18 @@
package status
import (
"bytes"
"io"
"os"
)
func raw(c Client) error {
b, err := c.StatusRaw()
if err != nil {
return err
}
if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
return err
}
return nil
}
internal/client/status/viewmodel/bytecountbinary.go
+26
View File
@@ -0,0 +1,26 @@
package viewmodel
import (
"fmt"
"math"
)
func ByteCountBinaryUint(b uint64) string {
if b > math.MaxInt64 {
panic(b)
}
return ByteCountBinary(int64(b))
}
func ByteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := unit, 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
internal/client/status/viewmodel/bytesprogresshistory.go
+49
View File
@@ -0,0 +1,49 @@
package viewmodel
import "time"
type byteProgressMeasurement struct {
time time.Time
val uint64
}
type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional
changeCount int
lastChange time.Time
}
func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
if p.last == nil {
p.last = &byteProgressMeasurement{
time: time.Now(),
val: currentVal,
}
return 0, 0
}
if p.last.val != currentVal {
p.changeCount++
p.lastChange = time.Now()
}
if time.Since(p.lastChange) > 3*time.Second {
p.last = nil
return 0, 0
}
var deltaV int64
if currentVal >= p.last.val {
deltaV = int64(currentVal - p.last.val)
} else {
deltaV = -int64(p.last.val - currentVal)
}
deltaT := time.Since(p.last.time)
rate := float64(deltaV) / deltaT.Seconds()
p.last.time = time.Now()
p.last.val = currentVal
return int64(rate), p.changeCount
}
internal/client/status/viewmodel/render.go
+660
View File
@@ -0,0 +1,660 @@
package viewmodel
import (
"fmt"
"math"
"sort"
"strings"
"time"
"github.com/go-playground/validator/v10"
yaml "github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/internal/client/status/viewmodel/stringbuilder"
"github.com/zrepl/zrepl/internal/daemon"
"github.com/zrepl/zrepl/internal/daemon/job"
"github.com/zrepl/zrepl/internal/daemon/pruner"
"github.com/zrepl/zrepl/internal/daemon/snapper"
"github.com/zrepl/zrepl/internal/replication/report"
)
type M struct {
jobs map[string]*Job
jobsList []*Job
selectedJob *Job
dateString string
bottomBarStatus string
}
type Job struct {
// long-lived
name string
byteProgress *bytesProgressHistory
lastStatus *job.Status
fulldescription string
}
func New() *M {
return &M{
jobs: make(map[string]*Job),
jobsList: make([]*Job, 0),
selectedJob: nil,
}
}
type FilterFunc func(string) bool
type Params struct {
Report map[string]*job.Status
ReportFetchError error
SelectedJob *Job
FSFilter FilterFunc `validate:"required"`
DetailViewWidth int `validate:"gte=1"`
DetailViewWrap bool
ShortKeybindingOverview string
}
var validate = validator.New()
func (m *M) Update(p Params) {
if err := validate.Struct(p); err != nil {
panic(err)
}
if p.ReportFetchError != nil {
m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
} else {
m.bottomBarStatus = p.ShortKeybindingOverview
for jobname, st := range p.Report {
// TODO handle job renames & deletions
j, ok := m.jobs[jobname]
if !ok {
j = &Job{
name: jobname,
byteProgress: &bytesProgressHistory{},
}
m.jobs[jobname] = j
m.jobsList = append(m.jobsList, j)
}
j.lastStatus = st
}
}
// filter out internal jobs
var jobsList []*Job
for _, j := range m.jobsList {
if daemon.IsInternalJobName(j.name) {
continue
}
jobsList = append(jobsList, j)
}
m.jobsList = jobsList
// determinism!
sort.Slice(m.jobsList, func(i, j int) bool {
return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
})
// try to not lose the selected job
m.selectedJob = nil
for _, j := range m.jobsList {
j.updateFullDescription(p)
if j == p.SelectedJob {
m.selectedJob = j
}
}
m.dateString = time.Now().Format(time.RFC3339)
}
func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
func (m *M) Jobs() []*Job { return m.jobsList }
// may be nil
func (m *M) SelectedJob() *Job { return m.selectedJob }
func (m *M) DateString() string { return m.dateString }
func (j *Job) updateFullDescription(p Params) {
width := p.DetailViewWidth
if !p.DetailViewWrap {
width = 10000000 // FIXME
}
b := stringbuilder.New(stringbuilder.Config{
IndentMultiplier: 3,
Width: width,
})
drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
j.fulldescription = b.String()
}
func (j *Job) JobTreeTitle() string {
return j.name
}
func (j *Job) FullDescription() string {
return j.fulldescription
}
func (j *Job) Name() string {
return j.name
}
func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
t.Printf("Job: %s\n", name)
t.Printf("Type: %s\n\n", v.Type)
if v.Type == job.TypePush || v.Type == job.TypePull {
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
if !ok || activeStatus == nil {
t.Printf("ActiveSideStatus is null")
t.Newline()
return
}
t.Printf("Replication:")
t.AddIndentAndNewline(1)
renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Sender:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Receiver:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
t.AddIndentAndNewline(-1)
if v.Type == job.TypePush {
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
}
} else if v.Type == job.TypeSnap {
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
if !ok || snapStatus == nil {
t.Printf("SnapJobStatus is null")
t.Newline()
return
}
t.Printf("Pruning snapshots:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, snapStatus.Pruning, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
} else if v.Type == job.TypeSource {
st := v.JobSpecific.(*job.PassiveStatus)
t.Printf("Snapshotting:\n")
t.AddIndent(1)
renderSnapperReport(t, st.Snapper, fsfilter)
t.AddIndentAndNewline(-1)
} else {
t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
t.Newline()
asYaml, err := yaml.Marshal(v.JobSpecific)
if err != nil {
t.Printf("Error marshaling status to YAML: %s", err)
t.Newline()
return
}
t.Write(string(asYaml))
t.Newline()
}
}
func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
sizeEstimationImpreciseNotice := ""
if containsInvalidSizeEstimates {
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
}
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
}
userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
// `.CurrentStep` is == len(rep.Steps) if all steps are done.
// Until then, it's an index into .Steps that starts at 0.
// For the user, we want it to start at 1.
if rep.CurrentStep >= len(rep.Steps) {
// rep.CurrentStep is what we want to show.
// We check for >= and not == for robustness.
} else {
// We're not done yet, so, make step count start at 1
// (The `.State` is included in the output, indicating we're not done yet)
userVisisbleCurrentStep = rep.CurrentStep + 1
}
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
strings.ToUpper(string(rep.State)),
userVisisbleCurrentStep, userVisibleTotalSteps,
ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
sizeEstimationImpreciseNotice,
)
activeIndicator := " "
if rep.BlockedOn == report.FsBlockedOnNothing &&
(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
activeIndicator = "*"
}
t.AddIndent(1)
t.Printf("%s %s %s ",
activeIndicator,
stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
status)
next := ""
if err := rep.Error(); err != nil {
next = err.Err
} else if rep.State != report.FilesystemDone {
if nextStep := rep.NextStep(); nextStep != nil {
if nextStep.IsIncremental() {
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
} else {
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
}
attribs := []string{}
if nextStep.Info.Resumed {
attribs = append(attribs, "resumed")
}
if len(attribs) > 0 {
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
}
} else {
next = "" // individual FSes may still be in planning state
}
}
t.Printf("%s", next)
t.AddIndent(-1)
t.Newline()
}
func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
if rep == nil {
t.Printf("...\n")
return
}
if rep.WaitReconnectError != nil {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
t.Newline()
}
if !rep.WaitReconnectSince.IsZero() {
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
var until string
if rep.WaitReconnectUntil.IsZero() {
until = "waiting indefinitely"
} else {
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
}
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
rep.WaitReconnectSince, until)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
}
t.Newline()
}
// TODO visualize more than the latest attempt by folding all attempts into one
if len(rep.Attempts) == 0 {
t.Printf("no attempts made yet")
return
} else {
t.Printf("Attempt #%d", len(rep.Attempts))
if len(rep.Attempts) > 1 {
t.Printf(". Previous attempts failed with the following statuses:")
t.AddIndentAndNewline(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
}
t.AddIndentAndNewline(-1)
} else {
t.Newline()
}
}
latest := rep.Attempts[len(rep.Attempts)-1]
sort.Slice(latest.Filesystems, func(i, j int) bool {
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
})
// apply filter
filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
for _, fs := range latest.Filesystems {
if !fsfilter(fs.Info.Name) {
continue
}
filtered = append(filtered, fs)
}
latest.Filesystems = filtered
t.Printf("Status: %s", latest.State)
t.Newline()
if !latest.FinishAt.IsZero() {
t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
} else {
t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
}
if latest.State == report.AttemptPlanningError {
t.Printf("Problem: ")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
t.Newline()
} else if latest.State == report.AttemptFanOutError {
t.Printf("Problem: one or more of the filesystems encountered errors")
t.Newline()
}
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
// Draw global progress bar
// Progress: [---------------]
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
rate, changeCount := history.Update(replicated)
eta := time.Duration(0)
if rate > 0 {
eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
}
if !latest.State.IsTerminal() {
t.Write("Progress: ")
t.DrawBar(50, replicated, expected, changeCount)
t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
if eta != 0 {
t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
}
t.Newline()
}
if containsInvalidSizeEstimates {
t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
t.Newline()
}
if len(latest.Filesystems) == 0 {
t.Write("NOTE: no filesystems were considered for replication!")
t.Newline()
}
var maxFSLen int
for _, fs := range latest.Filesystems {
if len(fs.Info.Name) > maxFSLen {
maxFSLen = len(fs.Info.Name)
}
}
for _, fs := range latest.Filesystems {
printFilesystemStatus(t, fs, maxFSLen)
}
}
}
func humanizeDuration(duration time.Duration) string {
days := int64(duration.Hours() / 24)
hours := int64(math.Mod(duration.Hours(), 24))
minutes := int64(math.Mod(duration.Minutes(), 60))
seconds := int64(math.Mod(duration.Seconds(), 60))
var parts []string
force := false
chunks := []int64{days, hours, minutes, seconds}
for i, chunk := range chunks {
if force || chunk > 0 {
padding := 0
if force {
padding = 2
}
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
force = true
}
}
return strings.Join(parts, " ")
}
func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("...\n")
return
}
state, err := pruner.StateString(r.State)
if err != nil {
t.Printf("Status: %q (parse error: %q)\n", r.State, err)
return
}
t.Printf("Status: %s", state)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
type commonFS struct {
*pruner.FSReport
completed bool
}
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false})
}
for i := range r.Completed {
all = append(all, commonFS{&r.Completed[i], true})
}
// filter all
filtered := make([]commonFS, 0, len(all))
for _, fs := range all {
if fsfilter(fs.FSReport.Filesystem) {
filtered = append(filtered, fs)
}
}
all = filtered
switch state {
case pruner.Plan:
fallthrough
case pruner.PlanErr:
return
}
if len(all) == 0 {
t.Printf("nothing to do\n")
return
}
var totalDestroyCount, completedDestroyCount int
var maxFSname int
for _, fs := range all {
totalDestroyCount += len(fs.DestroyList)
if fs.completed {
completedDestroyCount += len(fs.DestroyList)
}
if maxFSname < len(fs.Filesystem) {
maxFSname = len(fs.Filesystem)
}
}
// global progress bar
if !state.IsTerminal() {
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
t.Write("Progress: ")
t.Write("[")
t.Write(stringbuilder.Times("=", progress))
t.Write(">")
t.Write(stringbuilder.Times("-", 80-progress))
t.Write("]")
t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.Newline()
}
sort.SliceStable(all, func(i, j int) bool {
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
})
// Draw a table-like representation of 'all'
for _, fs := range all {
t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
t.Write(" ")
if !fs.SkipReason.NotSkipped() {
t.Printf("skipped: %s\n", fs.SkipReason)
continue
}
if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") {
t.Printf("ERROR:")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
}
t.Newline()
continue
}
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed {
t.Printf("Completed %s\n", pruneRuleActionStr)
continue
}
t.Write("Pending ") // whitespace is padding 10
if len(fs.DestroyList) == 1 {
t.Write(fs.DestroyList[0].Name)
} else {
t.Write(pruneRuleActionStr)
}
t.Newline()
}
}
func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("<no snapshotting report available>\n")
return
}
t.Printf("Type: %s\n", r.Type)
if r.Periodic != nil {
renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
} else if r.Cron != nil {
renderSnapperReportCron(t, r.Cron, fsfilter)
} else {
t.Printf("<no details available>")
}
}
func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
t.Printf("Status: %s", r.State)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
if !r.SleepUntil.IsZero() {
t.Printf("Sleep until: %s\n", r.SleepUntil)
}
renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
}
func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
t.Printf("State: %s\n", r.State)
now := time.Now()
if r.WakeupTime.After(now) {
t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
} else {
t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
}
renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
}
func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
sort.Slice(fss, func(i, j int) bool {
return strings.Compare(fss[i].Path, fss[j].Path) == -1
})
dur := func(d time.Duration) string {
return d.Round(100 * time.Millisecond).String()
}
type row struct {
path, state, duration, remainder, hookReport string
}
var widths struct {
path, state, duration int
}
rows := make([]*row, 0, len(fss))
for _, fs := range fss {
if !fsfilter(fs.Path) {
continue
}
r := &row{
path: fs.Path,
state: fs.State.String(),
}
if fs.HooksHadError {
r.hookReport = fs.Hooks // FIXME render here, not in daemon
}
switch fs.State {
case snapper.SnapPending:
r.duration = "..."
r.remainder = ""
case snapper.SnapStarted:
r.duration = dur(time.Since(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
case snapper.SnapDone:
fallthrough
case snapper.SnapError:
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
}
rows = append(rows, r)
if len(r.path) > widths.path {
widths.path = len(r.path)
}
if len(r.state) > widths.state {
widths.state = len(r.state)
}
if len(r.duration) > widths.duration {
widths.duration = len(r.duration)
}
}
for _, r := range rows {
path := stringbuilder.RightPad(r.path, widths.path, " ")
state := stringbuilder.RightPad(r.state, widths.state, " ")
duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
t.Printf("%s %s %s", path, state, duration)
t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
if r.hookReport != "" {
t.AddIndent(1)
t.Newline()
t.Printf("%s", r.hookReport)
t.AddIndent(-1)
}
t.Newline()
}
}
internal/client/status/viewmodel/stringbuilder/stringbuilder.go
+119
View File
@@ -0,0 +1,119 @@
package stringbuilder
import (
"fmt"
"strings"
"github.com/go-playground/validator/v10"
)
type B struct {
// const
indentMultiplier int
// mut
sb *strings.Builder
indent int
width int
x, y int
}
type Config struct {
IndentMultiplier int `validate:"gte=1"`
Width int `validate:"gte=1"`
}
var validate = validator.New()
func New(config Config) *B {
if err := validate.Struct(config); err != nil {
panic(err)
}
return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
}
func (b *B) String() string { return b.sb.String() }
func (w *B) Newline() {
w.Write("\n")
}
func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
if strings.ContainsAny(whole, "\n\r") {
w.AddIndent(1)
defer w.AddIndent(-1)
}
w.Write(whole)
}
func (w *B) Printf(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
w.Write(whole)
}
func (t *B) AddIndent(delta int) {
t.indent += delta * t.indentMultiplier
}
func (t *B) AddIndentAndNewline(delta int) {
t.indent += delta * t.indentMultiplier
t.Write("\n")
}
func (w *B) Write(s string) {
for _, c := range s {
if c == '\n' {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
w.y++
continue
}
if w.x >= w.width {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
}
fmt.Fprintf(w.sb, "%c", c)
w.x++
}
}
func Times(str string, n int) (out string) {
for i := 0; i < n; i++ {
out += str
}
return
}
func RightPad(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
// changeCount = 0 indicates stall / no progress
func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
const arrowPositions = `>\|/`
var completedLength int
if totalBytes > 0 {
completedLength = int(uint64(length) * bytes / totalBytes)
if completedLength > length {
completedLength = length
}
} else if totalBytes == bytes {
completedLength = length
}
w.Write("[")
w.Write(Times("=", completedLength))
w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
w.Write(Times("-", length-completedLength))
w.Write("]")
}
internal/client/stdinserver.go
+51
View File
@@ -0,0 +1,51 @@
package client
import (
"os"
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"context"
"errors"
"log"
"path"
)
var StdinserverCmd = &cli.Subcommand{
Use: "stdinserver CLIENT_IDENTITY",
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runStdinserver(subcommand.Config(), args)
},
}
func runStdinserver(config *config.Config, args []string) error {
// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
defer os.Exit(1)
log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
if len(args) != 1 || args[0] == "" {
err := errors.New("must specify client_identity as positional argument")
return err
}
identity := args[0]
unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
ctx := netssh.ContextWithLog(context.TODO(), log)
err := netssh.Proxy(ctx, unixaddr)
if err == nil {
log.Print("proxying finished successfully, exiting with status 0")
os.Exit(0)
}
log.Printf("error proxying: %s", err)
return nil
}
internal/client/testcmd.go
+220
View File
@@ -0,0 +1,220 @@
package client
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon/filters"
"github.com/zrepl/zrepl/internal/zfs"
)
var TestCmd = &cli.Subcommand{
Use: "test",
SetupSubcommands: func() []*cli.Subcommand {
return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
},
}
var testFilterArgs struct {
job string
all bool
input string
}
var testFilter = &cli.Subcommand{
Use: "filesystems --job JOB [--all | --input INPUT]",
Short: "test filesystems filter specified in push or source job",
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
},
Run: runTestFilterCmd,
}
func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
if testFilterArgs.job == "" {
return fmt.Errorf("must specify --job flag")
}
if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
return fmt.Errorf("must set one: --all or --input")
}
conf := subcommand.Config()
var confFilter config.FilesystemsFilter
job, err := conf.Job(testFilterArgs.job)
if err != nil {
return err
}
switch j := job.Ret.(type) {
case *config.SourceJob:
confFilter = j.Filesystems
case *config.PushJob:
confFilter = j.Filesystems
case *config.SnapJob:
confFilter = j.Filesystems
default:
return fmt.Errorf("job type %T does not have filesystems filter", j)
}
f, err := filters.DatasetMapFilterFromConfig(confFilter)
if err != nil {
return fmt.Errorf("filter invalid: %s", err)
}
var fsnames []string
if testFilterArgs.input != "" {
fsnames = []string{testFilterArgs.input}
} else {
out, err := zfs.ZFSList(ctx, []string{"name"})
if err != nil {
return fmt.Errorf("could not list ZFS filesystems: %s", err)
}
for _, row := range out {
fsnames = append(fsnames, row[0])
}
}
fspaths := make([]*zfs.DatasetPath, len(fsnames))
for i, fsname := range fsnames {
path, err := zfs.NewDatasetPath(fsname)
if err != nil {
return err
}
fspaths[i] = path
}
hadFilterErr := false
for _, in := range fspaths {
var res string
var errStr string
pass, err := f.Filter(in)
if err != nil {
res = "ERROR"
errStr = err.Error()
hadFilterErr = true
} else if pass {
res = "ACCEPT"
} else {
res = "REJECT"
}
fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
}
if hadFilterErr {
return fmt.Errorf("filter errors occurred")
}
return nil
}
var testPlaceholderArgs struct {
ds string
all bool
}
var testPlaceholder = &cli.Subcommand{
Use: "placeholder [--all | --dataset DATASET]",
Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
Example: `
placeholder --all
placeholder --dataset path/to/sink/clientident/fs`,
NoRequireConfig: true,
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
},
Run: runTestPlaceholder,
}
func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
var checkDPs []*zfs.DatasetPath
var datasetWasExplicitArgument bool
// all actions first
if testPlaceholderArgs.all {
datasetWasExplicitArgument = false
out, err := zfs.ZFSList(ctx, []string{"name"})
if err != nil {
return errors.Wrap(err, "could not list ZFS filesystems")
}
for _, row := range out {
dp, err := zfs.NewDatasetPath(row[0])
if err != nil {
panic(err)
}
checkDPs = append(checkDPs, dp)
}
} else {
datasetWasExplicitArgument = true
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
if err != nil {
return err
}
if dp.Empty() {
return fmt.Errorf("must specify --dataset DATASET or --all")
}
checkDPs = append(checkDPs, dp)
}
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
for _, dp := range checkDPs {
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
if err != nil {
return errors.Wrap(err, "cannot get placeholder state")
}
if !ph.FSExists {
if datasetWasExplicitArgument {
return errors.Errorf("filesystem %q does not exist", ph.FS)
} else {
// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
continue
}
}
is := "yes"
if !ph.IsPlaceholder {
is = "no"
}
fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
}
return nil
}
var testDecodeResumeTokenArgs struct {
token string
}
var testDecodeResumeToken = &cli.Subcommand{
Use: "decoderesumetoken --token TOKEN",
Short: "decode resume token",
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
},
Run: runTestDecodeResumeTokenCmd,
}
func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
if testDecodeResumeTokenArgs.token == "" {
return fmt.Errorf("token argument must be specified")
}
token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
if err != nil {
return err
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(&token); err != nil {
panic(err)
}
return nil
}
internal/client/version.go
+75
View File
@@ -0,0 +1,75 @@
package client
import (
"context"
"fmt"
"os"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/config"
"github.com/zrepl/zrepl/internal/daemon"
"github.com/zrepl/zrepl/internal/version"
)
var versionArgs struct {
Show string
Config *config.Config
ConfigErr error
}
var VersionCmd = &cli.Subcommand{
Use: "version",
Short: "print version of zrepl binary and running daemon",
NoRequireConfig: true,
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
versionArgs.Config = subcommand.Config()
versionArgs.ConfigErr = subcommand.ConfigParsingError()
return runVersionCmd()
},
}
func runVersionCmd() error {
args := versionArgs
if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
}
var clientVersion, daemonVersion *version.ZreplVersionInformation
if args.Show == "client" || args.Show == "" {
clientVersion = version.NewZreplVersionInformation()
fmt.Printf("client: %s\n", clientVersion.String())
}
if args.Show == "daemon" || args.Show == "" {
if args.ConfigErr != nil {
return fmt.Errorf("config parsing error: %s", args.ConfigErr)
}
httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
if err != nil {
return fmt.Errorf("server: error: %s\n", err)
}
var info version.ZreplVersionInformation
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
if err != nil {
return fmt.Errorf("server: error: %s\n", err)
}
daemonVersion = &info
fmt.Printf("server: %s\n", daemonVersion.String())
}
if args.Show == "" {
if clientVersion.Version != daemonVersion.Version {
fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
}
}
return nil
}
internal/client/zfsabstractions.go
+147
View File
@@ -0,0 +1,147 @@
package client
import (
"fmt"
"sort"
"strings"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/daemon/filters"
"github.com/zrepl/zrepl/internal/endpoint"
"github.com/zrepl/zrepl/internal/zfs"
)
var (
ZFSAbstractionsCmd = &cli.Subcommand{
Use: "zfs-abstraction",
Short: "manage abstractions that zrepl builds on top of ZFS",
SetupSubcommands: func() []*cli.Subcommand {
return []*cli.Subcommand{
zabsCmdList,
zabsCmdReleaseAll,
zabsCmdReleaseStale,
zabsCmdCreate,
}
},
}
)
// a common set of CLI flags that map to the fields of an
// endpoint.ListZFSHoldsAndBookmarksQuery
type zabsFilterFlags struct {
Filesystems FilesystemsFilterFlag
Job JobIDFlag
Types AbstractionTypesFlag
Concurrency int64
}
// produce a query from the CLI flags
func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
q := endpoint.ListZFSHoldsAndBookmarksQuery{
FS: f.Filesystems.FlagValue(),
What: f.Types.FlagValue(),
JobID: f.Job.FlagValue(),
Concurrency: f.Concurrency,
}
return q, q.Validate()
}
func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
// Note: the default value is defined in the .FlagValue methods
s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
for v := range endpoint.AbstractionTypesAll {
variants = append(variants, string(v))
}
sort.Strings(variants)
variantsJoined := strings.Join(variants, "|")
s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
}
type JobIDFlag struct{ J *endpoint.JobID }
func (f *JobIDFlag) Set(s string) error {
if len(s) == 0 {
*f = JobIDFlag{J: nil}
return nil
}
jobID, err := endpoint.MakeJobID(s)
if err != nil {
return err
}
*f = JobIDFlag{J: &jobID}
return nil
}
func (f JobIDFlag) Type() string { return "job-ID" }
func (f JobIDFlag) String() string { return fmt.Sprint(f.J) }
func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
type AbstractionTypesFlag map[endpoint.AbstractionType]bool
func (f *AbstractionTypesFlag) Set(s string) error {
ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
if err != nil {
return err
}
*f = AbstractionTypesFlag(ats)
return nil
}
func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
func (f AbstractionTypesFlag) String() string {
return endpoint.AbstractionTypeSet(f).String()
}
func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
if len(f) > 0 {
return f
}
return endpoint.AbstractionTypesAll
}
type FilesystemsFilterFlag struct {
F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
}
func (flag *FilesystemsFilterFlag) Set(s string) error {
mappings := strings.Split(s, ",")
if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
FS: &mappings[0],
}
return nil
}
f := filters.NewDatasetMapFilter(len(mappings), true)
for _, m := range mappings {
thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
lhsrhs := strings.SplitN(m, ":", 2)
if len(lhsrhs) != 2 {
return thisMappingErr
}
err := f.Add(lhsrhs[0], lhsrhs[1])
if err != nil {
return fmt.Errorf("%s: %s", thisMappingErr, err)
}
}
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
Filter: f,
}
return nil
}
func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
func (flag FilesystemsFilterFlag) String() string {
return fmt.Sprintf("%v", flag.F)
}
func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
var z FilesystemsFilterFlag
if flag == z {
return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
}
return flag.F
}
internal/client/zfsabstractions_create.go
+14
View File
@@ -0,0 +1,14 @@
package client
import "github.com/zrepl/zrepl/internal/cli"
var zabsCmdCreate = &cli.Subcommand{
Use: "create",
NoRequireConfig: true,
Short: `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
SetupSubcommands: func() []*cli.Subcommand {
return []*cli.Subcommand{
zabsCmdCreateStepHold,
}
},
}
internal/client/zfsabstractions_create_step_hold.go
+58
View File
@@ -0,0 +1,58 @@
package client
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/endpoint"
"github.com/zrepl/zrepl/internal/zfs"
)
var zabsCreateStepHoldFlags struct {
target string
jobid JobIDFlag
}
var zabsCmdCreateStepHold = &cli.Subcommand{
Use: "step",
Run: doZabsCreateStep,
NoRequireConfig: true,
Short: `create a step hold or bookmark`,
SetupFlags: func(f *pflag.FlagSet) {
f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
},
}
func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
if len(args) > 0 {
return errors.New("subcommand takes no arguments")
}
f := &zabsCreateStepHoldFlags
fs, _, _, err := zfs.DecomposeVersionString(f.target)
if err != nil {
return errors.Wrapf(err, "%q invalid target", f.target)
}
if f.jobid.FlagValue() == nil {
return errors.Errorf("jobid must be set")
}
v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
if err != nil {
return errors.Wrapf(err, "get info about target %q", f.target)
}
step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
if err != nil {
return errors.Wrap(err, "create step hold")
}
fmt.Println(step.String())
return nil
}
internal/client/zfsabstractions_list.go
+100
View File
@@ -0,0 +1,100 @@
package client
import (
"context"
"encoding/json"
"fmt"
"os"
"sync"
"github.com/fatih/color"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/endpoint"
"github.com/zrepl/zrepl/internal/util/chainlock"
)
var zabsListFlags struct {
Filter zabsFilterFlags
Json bool
}
var zabsCmdList = &cli.Subcommand{
Use: "list",
Short: `list zrepl ZFS abstractions`,
Run: doZabsList,
NoRequireConfig: true,
SetupFlags: func(f *pflag.FlagSet) {
zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
},
}
func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
var err error
if len(args) > 0 {
return errors.New("this subcommand takes no positional arguments")
}
q, err := zabsListFlags.Filter.Query()
if err != nil {
return errors.Wrap(err, "invalid filter specification on command line")
}
abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
if err != nil {
return err // context clear by invocation of command
}
defer drainDone()
var line chainlock.L
var wg sync.WaitGroup
defer wg.Wait()
// print results
wg.Add(1)
go func() {
defer wg.Done()
enc := json.NewEncoder(os.Stdout)
for a := range abstractions {
func() {
defer line.Lock().Unlock()
if zabsListFlags.Json {
enc.SetIndent("", " ")
if err := enc.Encode(a); err != nil {
panic(err)
}
fmt.Println()
} else {
fmt.Println(a)
}
}()
}
}()
// print errors to stderr
errorColor := color.New(color.FgRed)
var errorsSlice []endpoint.ListAbstractionsError
wg.Add(1)
go func() {
defer wg.Done()
for err := range errors {
func() {
defer line.Lock().Unlock()
errorsSlice = append(errorsSlice, err)
errorColor.Fprintf(os.Stderr, "%s\n", err)
}()
}
}()
wg.Wait()
if len(errorsSlice) > 0 {
errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
return fmt.Errorf("")
} else {
return nil
}
}
internal/client/zfsabstractions_release.go
+143
View File
@@ -0,0 +1,143 @@
package client
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/fatih/color"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/internal/cli"
"github.com/zrepl/zrepl/internal/endpoint"
)
// shared between release-all and release-step
var zabsReleaseFlags struct {
Filter zabsFilterFlags
Json bool
DryRun bool
}
func registerZabsReleaseFlags(s *pflag.FlagSet) {
zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
}
var zabsCmdReleaseAll = &cli.Subcommand{
Use: "release-all",
Run: doZabsReleaseAll,
NoRequireConfig: true,
Short: `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
SetupFlags: registerZabsReleaseFlags,
}
var zabsCmdReleaseStale = &cli.Subcommand{
Use: "release-stale",
Run: doZabsReleaseStale,
NoRequireConfig: true,
Short: `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
SetupFlags: registerZabsReleaseFlags,
}
func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
var err error
if len(args) > 0 {
return errors.New("this subcommand takes no positional arguments")
}
q, err := zabsReleaseFlags.Filter.Query()
if err != nil {
return errors.Wrap(err, "invalid filter specification on command line")
}
abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
if err != nil {
return err // context clear by invocation of command
}
if len(listErrors) > 0 {
color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
// proceed anyways with rest of abstractions
}
return doZabsRelease_Common(ctx, abstractions)
}
func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
var err error
if len(args) > 0 {
return errors.New("this subcommand takes no positional arguments")
}
q, err := zabsReleaseFlags.Filter.Query()
if err != nil {
return errors.Wrap(err, "invalid filter specification on command line")
}
stalenessInfo, err := endpoint.ListStale(ctx, q)
if err != nil {
return err // context clear by invocation of command
}
return doZabsRelease_Common(ctx, stalenessInfo.Stale)
}
func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
if zabsReleaseFlags.DryRun {
if zabsReleaseFlags.Json {
m, err := json.MarshalIndent(destroy, "", " ")
if err != nil {
panic(err)
}
if _, err := os.Stdout.Write(m); err != nil {
panic(err)
}
fmt.Println()
} else {
for _, a := range destroy {
fmt.Printf("would destroy %s\n", a)
}
}
return nil
}
outcome := endpoint.BatchDestroy(ctx, destroy)
hadErr := false
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
colorErr := color.New(color.FgRed)
printfSuccess := color.New(color.FgGreen).FprintfFunc()
printfSection := color.New(color.Bold).FprintfFunc()
for res := range outcome {
hadErr = hadErr || res.DestroyErr != nil
if zabsReleaseFlags.Json {
err := enc.Encode(res)
if err != nil {
colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
}
} else {
printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
if res.DestroyErr != nil {
colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
} else {
printfSuccess(os.Stdout, " OK\n")
}
}
}
if hadErr {
colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
return fmt.Errorf("")
} else {
return nil
}
}