adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: adminer/zrepl_patched:problame/fix-replication-order-deadlock-on-second-attempt
Branches Tags
adminer/zrepl_patched:master adminer/zrepl_patched:dependabot/uv/docs/idna-3.15 adminer/zrepl_patched:dependabot/uv/docs/urllib3-2.7.0 adminer/zrepl_patched:dependabot/uv/docs/pygments-2.20.0 adminer/zrepl_patched:dependabot/uv/docs/requests-2.33.0 adminer/zrepl_patched:dependabot/go_modules/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:dependabot/go_modules/build/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:releases/0.7.0-2 adminer/zrepl_patched:dependabot/go_modules/filippo.io/edwards25519-1.1.1 adminer/zrepl_patched:docs-user-delegation adminer/zrepl_patched:platformtest-simplify adminer/zrepl_patched:problame/go-1.24-tools adminer/zrepl_patched:problame/metric-for-running-jobs adminer/zrepl_patched:problame/triggers adminer/zrepl_patched:releases/0.6.1-2 adminer/zrepl_patched:problame/static-binaries adminer/zrepl_patched:problame/release-circleci-binaries adminer/zrepl_patched:problame/prep-release-0.6.1 adminer/zrepl_patched:problame/circleci-build adminer/zrepl_patched:iss-611-fix adminer/zrepl_patched:go1.19-update adminer/zrepl_patched:iss-486-duration-units adminer/zrepl_patched:iss-611-timezone-snapper adminer/zrepl_patched:problame/develop adminer/zrepl_patched:524-keep_not_replicated-for-pull-jobs adminer/zrepl_patched:517-tls-bad-cert adminer/zrepl_patched:495-fix-stuck-waiting-for-zfs-send-on-freebsd adminer/zrepl_patched:fix-encrypt-on-recv adminer/zrepl_patched:trigger-wait-reset-rearchitecture adminer/zrepl_patched:parallel-replication adminer/zrepl_patched:problame/golang-freebsd-investigation adminer/zrepl_patched:problame/zync adminer/zrepl_patched:problame/fix-replication-order-deadlock-on-second-attempt adminer/zrepl_patched:property_send_reformat_squash adminer/zrepl_patched:problame/platformtest-ci adminer/zrepl_patched:problame/322-fix-ssh-client-leaks-backport-0.2.1 adminer/zrepl_patched:problame/322-fix-ssh-client-leaks adminer/zrepl_patched:problame/pruner_rewrite_step_holds adminer/zrepl_patched:problame/307-tracing adminer/zrepl_patched:problame/pr-fix-replication-cursor-v1-crash-in-list-stale adminer/zrepl_patched:problame/fix-initial-replication-for-encrypted-send adminer/zrepl_patched:problame/ci-fixes-20200418 adminer/zrepl_patched:problame/308-zsh-completions-fixup adminer/zrepl_patched:problame/308-zsh-completions adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v3 adminer/zrepl_patched:problame/fix-zfscmd-nil-deref adminer/zrepl_patched:problame/cleanups-20200408 adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v2 adminer/zrepl_patched:problame/277-fix-receive-f adminer/zrepl_patched:problame/280-racy-holds-list adminer/zrepl_patched:problame/274-fix-incorrect-early-release-concurrent-send-sempahore adminer/zrepl_patched:problame/262-fix-status-eof-error adminer/zrepl_patched:problame/186-debug-dump-command adminer/zrepl_patched:problame/wip/tiered-snapshot-management adminer/zrepl_patched:problame/tracing adminer/zrepl_patched:problame/wip/keep-rule-most-recent-common-ancestor adminer/zrepl_patched:problame/experiment-runtime-controllable-concurrency adminer/zrepl_patched:parallel_zfs_ops adminer/zrepl_patched:resumable_send_recv
adminer/zrepl_patched:v0.7.0 adminer/zrepl_patched:v0.6.1 adminer/zrepl_patched:v0.6.0 adminer/zrepl_patched:v0.5.0 adminer/zrepl_patched:v0.5.0-rc1 adminer/zrepl_patched:v0.4.0 adminer/zrepl_patched:v0.4.0-rc2 adminer/zrepl_patched:v0.4.0-rc1 adminer/zrepl_patched:v0.3.1 adminer/zrepl_patched:v0.3.0 adminer/zrepl_patched:v0.3.0-rc2 adminer/zrepl_patched:v0.3.0-rc1 adminer/zrepl_patched:v0.2.1 adminer/zrepl_patched:v0.2.0 adminer/zrepl_patched:v0.2.0-rc1 adminer/zrepl_patched:v0.1.1 adminer/zrepl_patched:v0.1.0 adminer/zrepl_patched:v0.1.0-rc4 adminer/zrepl_patched:0.1.0-rc3 adminer/zrepl_patched:0.1.0-rc2 adminer/zrepl_patched:0.1.0-rc1 adminer/zrepl_patched:0.0.3 adminer/zrepl_patched:0.0.2 adminer/zrepl_patched:0.0.1
...
pull from: adminer/zrepl_patched:iss-611-fix
Branches Tags
adminer/zrepl_patched:master adminer/zrepl_patched:dependabot/uv/docs/idna-3.15 adminer/zrepl_patched:dependabot/uv/docs/urllib3-2.7.0 adminer/zrepl_patched:dependabot/uv/docs/pygments-2.20.0 adminer/zrepl_patched:dependabot/uv/docs/requests-2.33.0 adminer/zrepl_patched:dependabot/go_modules/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:dependabot/go_modules/build/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:releases/0.7.0-2 adminer/zrepl_patched:dependabot/go_modules/filippo.io/edwards25519-1.1.1 adminer/zrepl_patched:docs-user-delegation adminer/zrepl_patched:platformtest-simplify adminer/zrepl_patched:problame/go-1.24-tools adminer/zrepl_patched:problame/metric-for-running-jobs adminer/zrepl_patched:problame/triggers adminer/zrepl_patched:releases/0.6.1-2 adminer/zrepl_patched:problame/static-binaries adminer/zrepl_patched:problame/release-circleci-binaries adminer/zrepl_patched:problame/prep-release-0.6.1 adminer/zrepl_patched:problame/circleci-build adminer/zrepl_patched:iss-611-fix adminer/zrepl_patched:go1.19-update adminer/zrepl_patched:iss-486-duration-units adminer/zrepl_patched:iss-611-timezone-snapper adminer/zrepl_patched:problame/develop adminer/zrepl_patched:524-keep_not_replicated-for-pull-jobs adminer/zrepl_patched:517-tls-bad-cert adminer/zrepl_patched:495-fix-stuck-waiting-for-zfs-send-on-freebsd adminer/zrepl_patched:fix-encrypt-on-recv adminer/zrepl_patched:trigger-wait-reset-rearchitecture adminer/zrepl_patched:parallel-replication adminer/zrepl_patched:problame/golang-freebsd-investigation adminer/zrepl_patched:problame/zync adminer/zrepl_patched:problame/fix-replication-order-deadlock-on-second-attempt adminer/zrepl_patched:property_send_reformat_squash adminer/zrepl_patched:problame/platformtest-ci adminer/zrepl_patched:problame/322-fix-ssh-client-leaks-backport-0.2.1 adminer/zrepl_patched:problame/322-fix-ssh-client-leaks adminer/zrepl_patched:problame/pruner_rewrite_step_holds adminer/zrepl_patched:problame/307-tracing adminer/zrepl_patched:problame/pr-fix-replication-cursor-v1-crash-in-list-stale adminer/zrepl_patched:problame/fix-initial-replication-for-encrypted-send adminer/zrepl_patched:problame/ci-fixes-20200418 adminer/zrepl_patched:problame/308-zsh-completions-fixup adminer/zrepl_patched:problame/308-zsh-completions adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v3 adminer/zrepl_patched:problame/fix-zfscmd-nil-deref adminer/zrepl_patched:problame/cleanups-20200408 adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v2 adminer/zrepl_patched:problame/277-fix-receive-f adminer/zrepl_patched:problame/280-racy-holds-list adminer/zrepl_patched:problame/274-fix-incorrect-early-release-concurrent-send-sempahore adminer/zrepl_patched:problame/262-fix-status-eof-error adminer/zrepl_patched:problame/186-debug-dump-command adminer/zrepl_patched:problame/wip/tiered-snapshot-management adminer/zrepl_patched:problame/tracing adminer/zrepl_patched:problame/wip/keep-rule-most-recent-common-ancestor adminer/zrepl_patched:problame/experiment-runtime-controllable-concurrency adminer/zrepl_patched:parallel_zfs_ops adminer/zrepl_patched:resumable_send_recv
adminer/zrepl_patched:v0.7.0 adminer/zrepl_patched:v0.6.1 adminer/zrepl_patched:v0.6.0 adminer/zrepl_patched:v0.5.0 adminer/zrepl_patched:v0.5.0-rc1 adminer/zrepl_patched:v0.4.0 adminer/zrepl_patched:v0.4.0-rc2 adminer/zrepl_patched:v0.4.0-rc1 adminer/zrepl_patched:v0.3.1 adminer/zrepl_patched:v0.3.0 adminer/zrepl_patched:v0.3.0-rc2 adminer/zrepl_patched:v0.3.0-rc1 adminer/zrepl_patched:v0.2.1 adminer/zrepl_patched:v0.2.0 adminer/zrepl_patched:v0.2.0-rc1 adminer/zrepl_patched:v0.1.1 adminer/zrepl_patched:v0.1.0 adminer/zrepl_patched:v0.1.0-rc4 adminer/zrepl_patched:0.1.0-rc3 adminer/zrepl_patched:0.1.0-rc2 adminer/zrepl_patched:0.1.0-rc1 adminer/zrepl_patched:0.0.3 adminer/zrepl_patched:0.0.2 adminer/zrepl_patched:0.0.1

109 Commits
problame/fix-replication-order-deadlock-on-second-attempt ... iss-611-fix

Author SHA1 Message Date
Christian Schwarz a187fa3b09 snapper: fix delayed snapshots caused by system suspend/resume 
See explainer comment in periodic.go for details.

fixes https://github.com/zrepl/zrepl/issues/611
 2022-10-25 01:38:35 +02:00
Christian Schwarz a967986a18 fixup: fix hooks unit tests 
The previous commit c743c7b03f
broke the hooks unit tests.

GitHub was not configured to require passing tests for master merge.
Didn't notice it locally due to Go's test caching.
I amended this before pushing this change.
 2022-10-09 15:36:00 +02:00
Christian Schwarz c743c7b03f refactor snapper & support cron-based snapshotting 
fixes https://github.com/zrepl/zrepl/issues/554
refs https://github.com/zrepl/zrepl/discussions/547#discussioncomment-1936126
 2022-09-25 19:23:44 +02:00
Christian Schwarz a9c61b4b0b zrepl status UI: include w shortcut to wrap lines in help bar 2022-09-25 19:23:44 +02:00
Christian Schwarz 206d359dcd docs: sendrecvoptions: fix heading level for section on placeholders 2022-09-25 18:23:54 +02:00
Christian Schwarz 2d8c3692ec rework resume token validation to allow resuming from raw sends of unencrypted datasets 
Before this change, resuming from an unencrypted dataset with
send.raw=true specified wouldn't work with zrepl due to overly
restrictive resume token checking.

An initial PR to fix this was made in https://github.com/zrepl/zrepl/pull/503
but it didn't address the core of the problem.
The core of the problem was that zrepl assumed that if a resume token
contained `rawok=true, compressok=true`, the resulting send would be
encrypted. But if the sender dataset was unencrypted, such a resume would
actually result in an unencrypted send.
Which could be totally legitimate but zrepl failed to recognize that.

BACKGROUND
==========

The following snippets of OpenZFS code are insightful regarding how the
various ${X}ok values in the resume token are handled:

- https://github.com/openzfs/zfs/blob/6c3c5fcfbe27d9193cd131753cc7e47ee2784621/module/zfs/dmu_send.c#L1947-L2012
- https://github.com/openzfs/zfs/blob/6c3c5fcfbe27d9193cd131753cc7e47ee2784621/module/zfs/dmu_recv.c#L877-L891
- https://github.com/openzfs/zfs/blob/6c3c5fc/lib/libzfs/libzfs_sendrecv.c#L1663-L1672

Basically, some zfs send flags make the DMU send code set some DMU send
stream featureflags, although it's not a pure mapping, i.e, which DMU
send stream flags are used depends somewhat on the dataset (e.g., is it
encrypted or not, or, does it use zstd or not).

Then, the receiver looks at some (but not all) feature flags and maps
them to ${X}ok dataset zap attributes.

These are funnelled back to the sender 1:1 through the resume_token.

And the sender turns them into lzc flags.

As an example, let's look at zfs send --raw.
if the sender requests a raw send on an unencrypted dataset, the send
stream (and hence the resume token) will not have the raw stream
featureflag set, and hence the resume token will not have the rawok
field set. Instead, it will have compressok, embedok, and depending
on whether large blocks are present in the dataset, largeblockok set.

WHAT'S ZREPL'S ROLE IN THIS?
============================

zrepl provides a virtual encrypted sendflag that is like `raw`,
but further ensures that we only send encrypted datasets.

For any other resume token stuff, it shoudn't do any checking,
because it's a futile effort to keep up with ZFS send/recv features
that are orthogonal to encryption.

CHANGES MADE IN THIS COMMIT
===========================

- Rip out a bunch of needless checking that zrepl would do during
  planning. These checks were there to give better error messages,
  but actually, the error messages created by the endpoint.Sender.Send
  RPC upon send args validation failure are good enough.
- Add platformtests to validate all combinations of
  (Unencrypted/Encrypted FS) x (send.encrypted = true | false) x (send.raw = true | false)
  for cases both non-resuming and resuming send.

Additional manual testing done:
1. With zrepl 0.5, setup with unencrypted dataset, send.raw=true specified, no send.encrypted specified.
2. Observe that regular non-resuming send works, but resuming doesn't work.
3. Upgrade zrepl to this change.
4. Observe that both regular and resuming send works.

closes https://github.com/zrepl/zrepl/pull/613
 2022-09-25 17:32:02 +02:00
Christian Schwarz 7769263c2e platformtest: add QueueSubtest functionality 
Use it from a top-level test case to queue the
execution of sub-tests after this test case is complete.

Note that the testing harness executes the subtest
_after_ the current top-level test. Hence, the subtest
cannot use any ZFS state of the top-level test.
 2022-09-25 17:10:53 +02:00
Christian Schwarz 89f7c76c4e lint: allow empty else branches 2022-09-25 17:10:53 +02:00
jtagcat c7771f98f5 docs: improve overview 
There were and still is too many words. It's a very white paper vibe.
Docs needs to be more brief, exact, and on-point.

closes https://github.com/zrepl/zrepl/pull/618
 2022-07-31 15:50:53 +02:00
jtagcat 299f1c906e docs: overview: clarify configs _are_ ordered 
Previously with unordered list, and 'are considered'
left if unsure whether one or all files are 'considered'.
In reality, the first valid is used, so an ordered list and
perhaps better wording communicates this fact.

refs https://github.com/zrepl/zrepl/pull/618
 2022-07-31 15:33:23 +02:00
Kiss Károly d3f68ae4e8 replication: ignore bookmarks when computing incremental path 
fixes https://github.com/zrepl/zrepl/issues/490
closes https://github.com/zrepl/zrepl/pull/619

Co-authored-by: Christian Schwarz <me@cschwarz.com>
 2022-07-31 15:25:19 +02:00
Christian Schwarz 193abbe6b1 fix active child tasks panic with endpoint.ListAbstractionsStreamed 
The goroutine that does endTask() for
"list-abstractions-streamed-producer" can be preempted
after it has closed the out and outErrs channel,
but before it calls endTask().
If the parent ("handler") then gets scheduled and
and ends itself, it will observe an active child task
"list-abstractions-streamed-producer".

This is easy to demo by injecting a sleep here:

  --- a/endpoint/endpoint_zfs_abstraction.go
  +++ b/endpoint/endpoint_zfs_abstraction.go
  @@ -575,6 +576,7 @@ func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmark
          ctx, endTask := trace.WithTask(ctx, "list-abstractions-streamed-producer")
          go func() {
                  defer endTask()
  +               defer time.Sleep(10 * time.Second)
                  defer close(out)
                  defer close(outErrs)

fixes https://github.com/zrepl/zrepl/issues/607
 2022-07-17 21:44:03 +02:00
Goran Mekić 02b215128e build: consistently use $(MAKE) when invoking it recursively 
Not for the `docker run ... make ...` commands though!

closes https://github.com/zrepl/zrepl/pull/615
 2022-07-12 00:18:38 +02:00
Christian Schwarz dc03db7423 rpc/grpcclientidentity/authlistener_grpc_adaptor: don't assume peer.Addr is set 
On Illumos, getpeername doesn't work from Go on socketpair sockets.
That's why .RemoteAddr() returns nil on such a socket.
And that `nil` ultimately lands in the `p.Addr`.
So, `p.Addr.String()` would deref `nil`, leading to

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0xaea33e]

    goroutine 614 [running]:
    github.com/zrepl/zrepl/rpc/grpcclientidentity.NewInterceptors.func1({0xf1e158, 0xc000631200}, {0xd514c0, 0xc000631230}, 0xc000032740, 0xc000524348)
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/build/amd64/rpc/grpcclientidentity/authlistener_grpc_adaptor.go:121 +0x13e
    github.com/zrepl/zrepl/replication/logic/pdu._Replication_ListFilesystems_Handler({0xdb30c0, 0xc00001a630}, {0xf1e158, 0xc000631200}, 0xc00052b7a0, 0xc000522000)
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/build/amd64/replication/logic/pdu/pdu_grpc.pb.go:186 +0x16a
    google.golang.org/grpc.(*Server).processUnaryRPC(0xc00016e700, {0xf2bc00, 0xc0000f2780}, 0xc00011c200, 0xc000522150, 0x1497c78, 0x0)
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/zrepl-0.5.0/gopath/pkg/mod/google.golang.org/grpc@v1.35.0/server.go:1217 +0xe28
    google.golang.org/grpc.(*Server).handleStream(0xc00016e700, {0xf2bc00, 0xc0000f2780}, 0xc00011c200, 0x0)
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/zrepl-0.5.0/gopath/pkg/mod/google.golang.org/grpc@v1.35.0/server.go:1540 +0xcb3
    google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc000373b70, 0xc00016e700, {0xf2bc00, 0xc0000f2780}, 0xc00011c200)
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/zrepl-0.5.0/gopath/pkg/mod/google.golang.org/grpc@v1.35.0/server.go:878 +0xad
    created by google.golang.org/grpc.(*Server).serveStreams.func1
    	/dpool/export/home/mills/Downloads/code/oi-userland-gh/components/sysutils/zrepl/zrepl-0.5.0/gopath/pkg/mod/google.golang.org/grpc@v1.35.0/server.go:876 +0x1ec

fixes https://github.com/zrepl/zrepl/issues/598
 2022-07-10 23:59:40 +02:00
Cole Helbling 1df0f8912a Add --skip-cert-check flag to zrepl configcheck to prevent checking cert files 
It may be desirable to check that a config is valid without checking for
the existence of certificate files (e.g. when validating a config inside
a sandbox without access to the cert files).

This will be very useful for NixOS so that we can check the config file
at nix-build time (e.g. potentially without proper permissions to read cert
files for a TLS connection).

fixes https://github.com/zrepl/zrepl/issues/467
closes https://github.com/zrepl/zrepl/pull/587
 2022-07-08 20:18:41 +02:00
3nprob e4112d888c add ZREPL_DESTROY_MAX_BATCH_SIZE env var to control max batch destroy size 
fixes #508
closes https://github.com/zrepl/zrepl/pull/604
 2022-06-30 09:22:26 +02:00
Christian Schwarz 53f9bd6d88 docs: update CLI usage to --mode raw & remove outdated "Limitations" section 
fixes https://github.com/zrepl/zrepl/issues/609
 2022-06-28 00:17:34 +02:00
JMoVS 43c2a0d9b0 docs: clarity on the section that covers more complex setups 
closes https://github.com/zrepl/zrepl/pull/596
 2022-06-27 22:41:12 +02:00
Christian Schwarz e0c7ceedd5 prevent transient zrepl status error: Post "http://unix/status": EOF 
See the comment added to client.go in this commit.

fixes https://github.com/zrepl/zrepl/issues/483
fixes https://github.com/zrepl/zrepl/issues/262
fixes https://github.com/zrepl/zrepl/issues/379
fixes https://github.com/zrepl/zrepl/issues/379
 2022-06-26 14:39:35 +02:00
Christian Schwarz 2642c64303 make initial replication policy configurable (most_recent, all, fail) 
Config:

```
- type: push
  ...
  conflict_resolution:
    initial_replication: most_recent | all | fali
```

The ``initial_replication`` option determines which snapshots zrepl
replicates if the filesystem has not been replicated before.
If ``most_recent`` (the default), the initial replication will only
transfer the most recent snapshot, while ignoring previous snapshots.
If all snapshots should be replicated, specify ``all``.
Use ``fail`` to make replication of the filesystem fail in case
there is no corresponding fileystem on the receiver.

Code-Level Changes, apart from the obvious:
- Rework IncrementalPath()'s return signature.
  Now returns an error for initial replications as well.
- Rename & rework it's consumer, resolveConflict().

Co-authored-by: Graham Christensen <graham@grahamc.com>

Fixes https://github.com/zrepl/zrepl/issues/550
Fixes https://github.com/zrepl/zrepl/issues/187
Closes https://github.com/zrepl/zrepl/pull/592
 2022-06-26 14:36:59 +02:00
JMoVS 1acafabb5b docs: Fix typo in disjoing to disjoint 
Signed-off-by: Justin Scholz <git@justinscholz.de>
 2022-05-07 22:13:56 +02:00
Christian Schwarz 19b2deb2cf run go mod tidy; go version go1.17.2 linux/amd64 
Juding from the (now deleted) comments in go.mod, this might break 1.12
build. If it does, the CI will catch it as it currently build using
1.17 and 1.12.

fixes https://github.com/zrepl/zrepl/issues/586
 2022-05-07 21:59:51 +02:00
Christian Schwarz ce6701fb33 status: fix over-counted step when status != stepping 
This is a fixup of

  commit b00b61e967
  Author: Christian Schwarz <me@cschwarz.com>
  Date:   Sun Nov 21 15:15:23 2021 +0100

      status: user-visible replication step number should start at 1

fixes https://github.com/zrepl/zrepl/issues/589
refs https://github.com/zrepl/zrepl/issues/538
 2022-04-24 15:24:39 +02:00
Christian Schwarz 0121929164 build: use git+https to fix lazy.sh docdep failures 
CircleCI fails like so:

    #!/bin/bash -eo pipefail
    ./lazy.sh docdep

    pip3 is /home/circleci/.pyenv/shims/pip3
    Installing doc build dependencies
    Obtaining sphinxcontrib-versioning from git+git://github.com/rwblair/sphinxcontrib-versioning.git@7e3885a389a809e17ea55261316b7b0e98dbf98f#egg=sphinxcontrib-versioning (from -r ./docs/requirements.txt (line 28))
      Cloning git://github.com/rwblair/sphinxcontrib-versioning.git (to revision 7e3885a389a809e17ea55261316b7b0e98dbf98f) to ./src/sphinxcontrib-versioning
      Running command git clone --filter=blob:none --quiet git://github.com/rwblair/sphinxcontrib-versioning.git /home/circleci/project/src/sphinxcontrib-versioning
      fatal: remote error:
        The unauthenticated git protocol on port 9418 is no longer supported.
      Please see https://github.blog/2021-09-01-improving-git-protocol-security-github/ for more information.
      error: subprocess-exited-with-error

      × git clone --filter=blob:none --quiet git://github.com/rwblair/sphinxcontrib-versioning.git /home/circleci/project/src/sphinxcontrib-versioning did not run successfully.
      │ exit code: 128
      ╰─> See above for output.

      note: This error originates from a subprocess, and is likely not a problem with pip.
    error: subprocess-exited-with-error

    × git clone --filter=blob:none --quiet git://github.com/rwblair/sphinxcontrib-versioning.git /home/circleci/project/src/sphinxcontrib-versioning did not run successfully.
    │ exit code: 128
    ╰─> See above for output.

    note: This error originates from a subprocess, and is likely not a problem with pip.

    Exited with code exit status 1

    CircleCI received exit code 1
 2022-03-20 20:23:01 +01:00
Christian Schwarz bc96f8f212 build/circleci: update to Ubuntu 20.04 image for release-* jobs 
Background: `machine: true` is deprecated:

    https://circleci.com/docs/2.0/images/linux-vm/14.04-to-20.04-migration/
 2022-02-15 22:55:25 +01:00
Christian Schwarz 459508c9d9 docs: sendrecvoptions: placeholders: fix wrong link name and add summarizing config snippet for recv.placeholders 
fixes https://github.com/zrepl/zrepl/issues/573
 2022-02-05 10:59:33 +01:00
Lapo Luchini 4a27cc63a8 prometheus: convert zrepl_version_daemon to zrepl_start_time metric 
closes https://github.com/zrepl/zrepl/pull/556
fixes #553
 2022-01-20 19:33:18 +01:00
Christian Schwarz 0a6840273a build: add tag-release Make target 2022-01-20 19:25:22 +01:00
madbrain76 76ef84f83b docs: fix typo in backup_to_external_disk.rst 
closes https://github.com/zrepl/zrepl/pull/568
 2022-01-20 19:25:03 +01:00
Christian Schwarz 66946df756 docs: continous_server_backup: simplify by removing need for recv.placeholder 2022-01-09 12:51:00 +01:00
Andrew Gunnerson 556fac3002 docs: document fan-out replication & add quick-start guide 
closes https://github.com/zrepl/zrepl/pull/552
fixes https://github.com/zrepl/zrepl/issues/551

Signed-off-by: Andrew Gunnerson <chillermillerlong@hotmail.com>
Co-authored-by: Christian Schwarz <me@cschwarz.com>
 2022-01-09 12:45:09 +01:00
Christian Schwarz 1ad7df2df3 docs: badges & links to Matrix chat room 
fixes https://github.com/zrepl/zrepl/issues/488
 2022-01-09 12:05:19 +01:00
Christian Schwarz a3d010c5f0 util/optionaldeadline: disable scheduler latency-sensitive tests in CircleCI 2021-12-30 14:41:06 +01:00
Christian Schwarz 12503dc55a rpc/dataconn/timeoutconn: disable TestPartialWriteMockConn in CircleCI 2021-12-18 18:02:34 +01:00
Christian Schwarz 7d10a71cc0 0.5 changelog + front page update 2021-12-18 17:36:54 +01:00
Christian Schwarz 3d3d1b5679 quickstart: sample config uses placeholders, so provide sample value for recv.placeholder.encryption 2021-12-18 17:18:58 +01:00
Christian Schwarz 5240ab4949 docs: quickstart: make users aware that the example rules apply to all snaps, not just zrepl's 
fixes https://github.com/zrepl/zrepl/issues/540
 2021-12-18 16:30:15 +01:00
Christian Schwarz 19aebd399f docs: add a note that FreeBSD jail zfs userland needs to be kept in sync with kernel module 
fixes https://github.com/zrepl/zrepl/issues/500
 2021-12-18 16:06:26 +01:00
Christian Schwarz 04e03f4d06 platformtest: retry zpool export if 'pool is busy' 
On Ubuntu, something seems to be holding on to the pool for too long.
 2021-12-18 15:58:24 +01:00
Christian Schwarz 2e2a8a1d5d docs: add docs on how to run platform tests 
fixes https://github.com/zrepl/zrepl/issues/478
 2021-12-18 15:55:22 +01:00
Christian Schwarz a2b2e0fe34 daemon/control: make http server {Read,Write}Timeout envconst-configurable 
refs https://github.com/zrepl/zrepl/issues/379
 2021-12-18 15:14:33 +01:00
Christian Schwarz af2905d245 docs: apt repo: deploy gpg to /usr/share/keyrings and use 'signed-by' in repo definition 
gpg --dearmor because of note in https://wiki.debian.org/DebianRepository/UseThirdParty

fixes https://github.com/zrepl/zrepl/issues/529
 2021-12-18 15:14:33 +01:00
Christian Schwarz c3f0041efd zrepl test placeholder: fix panic if dataset does not exist 
fixes https://github.com/zrepl/zrepl/issues/406
 2021-12-18 15:14:33 +01:00
Christian Schwarz 083f6001eb build: freebsd armv7 and arm64 binaries 
fixes https://github.com/zrepl/zrepl/issues/539
 2021-12-18 15:14:33 +01:00
Christian Schwarz 2d57ec6ee0 docs: changelog: mention upstream ashift 9 => 12 send/recv bug 2021-12-18 15:14:33 +01:00
Christian Schwarz fb6a9be954 fix encrypt-on-receive with placeholders 
fixes https://github.com/zrepl/zrepl/issues/504

Problem:
  plain send + recv with root_fs encrypted + placeholders causes plain recvs
  whereas user would expect encrypt-on-recv
Reason:
  We create placeholder filesytems with -o encryption=off.
  Thus, children received below those placeholders won't inherit
  encryption of root_fs.
Fix:
  We'll have three values for `recv.placeholders.encryption: unspecified (default) | off | inherit`.
  When we create a placeholder, we will fail the operation if  `recv.placeholders.encryption = unspecified`.
  The exception is if the placeholder filesystem is to encode the client identity ($root_fs/$client_identity) in a pull job.
  Those are created in `inherit` mode if the config field is `unspecified` so that users who don't need
  placeholders are not bothered by these details.

Future Work:
  Automatically warn existing users of encrypt-on-recv about the problem
  if they are affected.
  The problem that I hit during implementation of this is that the
  `encryption` prop's `source` doesn't quite behave like other props:
  `source` is `default` for `encryption=off` and `-` when `encryption=on`.
  Hence, we can't use `source` to distinguish the following 2x2 cases:
  (1) placeholder created with explicit -o encryption=off
  (2) placeholder created without specifying -o encryption
  with
  (A) an encrypted parent at creation time
  (B) an unencrypted parent at creation time
 2021-12-18 15:12:47 +01:00
Christian Schwarz c1e2c9826f trace: hint debug env var in error when crashing due to active child tasks 
refs https://github.com/zrepl/zrepl/issues/542
 2021-12-05 18:57:43 +01:00
Christian Schwarz b00b61e967 status: user-visible replication step number should start at 1 
fixes https://github.com/zrepl/zrepl/issues/538
 2021-11-21 15:32:18 +01:00
Christian Schwarz ac147b5a6f replication: report a filesystem is active vs. blocked on something 
- `BlockedOn` prop in JSON report
- Bring back the `*` in front of the filesystem report as an activity indicator.

fixes https://github.com/zrepl/zrepl/issues/505
 2021-11-14 17:34:32 +01:00
Samy Mahmoudi 1850a332ed docs: prune: improve docs for 'grid' rule 
- Substitute full words for both string name 'gridspec' and short form 'grid spec'
- Fix alignment and make spacing more consistent
- Fix fall of snapshots into buckets for the example to really reflect right-exclusiveness

closes https://github.com/zrepl/zrepl/pull/535
 2021-11-14 17:34:32 +01:00
Christian Schwarz 20ff9717bc fix mis-spelled send option for embedded data 
fixes https://github.com/zrepl/zrepl/issues/522
 2021-11-14 17:34:32 +01:00
Christian Schwarz c2fbf93365 daemon: provide os.Environ() in zrepl status 
Useful for debugging.

fixes https://github.com/zrepl/zrepl/issues/534
 2021-11-14 17:34:32 +01:00
Christian Schwarz cf5e8e8f26 docs: add runbook on how to migrate sending side to new zpool 
fixes https://github.com/zrepl/zrepl/issues/525
 2021-11-14 17:34:32 +01:00
Christian Schwarz c600cc1f60 skip timing-sensitive tests on CircleCI 
We had too many spurious test failures in the past.
But on a developer machine, the tests don't usually fail because the
system isn't loaded as much.
So, only disable test on CircleCI.
 2021-11-14 17:34:32 +01:00
Lapo Luchini c6a9ebc71c job/active: add "last completed" metric for error reporting 
use case:

    So that I can use a more resilient alerting such as "last complete was sent more than 24h ago".

fixes https://github.com/zrepl/zrepl/issues/516
closes https://github.com/zrepl/zrepl/pull/530
 2021-11-10 17:35:12 +01:00
Christian Schwarz 1f0f2f8569 pruner + docs: less confusing type names, some comments, better docs for keep: not_replicated 
fixes https://github.com/zrepl/zrepl/issues/524
 2021-10-10 21:11:38 +02:00
Christian Schwarz 5104ad3d0b build: use go 1.17 for testing & release builds 2021-10-09 16:51:08 +02:00
Christian Schwarz a6dbda1ea8 go1.17: run goimports to supports the new //go:build lines 2021-10-09 16:51:08 +02:00
Christian Schwarz 1edb8014bc build: circleci: stop storing artifacts 
When we need artifacts, we use MinIO anyways.
And we have accumulated about 1TiB of (free) CircleCI artifact storage.
Don't need to waste space unnecessarily.
 2021-10-09 16:13:58 +02:00
Christian Schwarz 845195b7ed bandwidth limiting: fix crash with SnapJob 
zrepl daemon panics when the snap job triggers

fixup for f5f269bfd5 (bandwidth limiting)
fixes #521

Oct 01 16:14:56 cstp zrepl[56563]: panic: invalid config`BandwidthLimit` field invalid: BucketCapacity must not be zero
Oct 01 16:14:56 cstp zrepl[56563]:         panic: end span: span still has active child spans
Oct 01 16:14:56 cstp zrepl[56563]: goroutine 38 [running]:
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/daemon/logging/trace.WithSpan.func2()
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/logging/trace/trace.go:341 +0x2ea
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/daemon/logging/trace.WithTaskAndSpan.func1()
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/logging/trace/trace_convenience.go:40 +0x2e
Oct 01 16:14:56 cstp zrepl[56563]: panic(0xcee9c0, 0xc000676730)
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/go1.16.6/src/runtime/panic.go:965 +0x1b9
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/endpoint.NewSender(0xf5bbc0, 0xc0003840c0, 0xc0000b2c90, 0x4, 0xc0002c5958, 0x0, 0x0, 0x0, 0xc000068cf8)
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/endpoint/endpoint.go:68 +0x1ec
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/daemon/job.(*SnapJob).doPrune(0xc00039e000, 0xf6e3b8, 0xc0006541b0)
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/job/snapjob.go:179 +0x198
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/daemon/job.(*SnapJob).Run(0xc00039e000, 0xf6e3b8, 0xc0001d83c0)
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/job/snapjob.go:127 +0x329
Oct 01 16:14:56 cstp zrepl[56563]: github.com/zrepl/zrepl/daemon.(*jobs).start.func1(0xc0006a4100, 0xf6e3b8, 0xc00022a0f0, 0xf72d18, 0xc00039e000)
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/daemon.go:255 +0x15b
Oct 01 16:14:56 cstp zrepl[56563]: created by github.com/zrepl/zrepl/daemon.(*jobs).start
Oct 01 16:14:56 cstp zrepl[56563]:         /home/cs/zrepl/zrepl/daemon/daemon.go:251 +0x425
Oct 01 16:14:56 cstp systemd[1]: zrepl.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
Oct 01 16:14:56 cstp systemd[1]: zrepl.service: Failed with result 'exit-code'.
 2021-10-09 15:52:38 +02:00
Christian Schwarz 2c9fcd7c14 rpc/dataconn: always close send stream returned from Sender.Send() 
discovered while debugging #457
 2021-10-09 15:43:31 +02:00
Christian Schwarz 4f9b63aa09 rework size estimation & dry sends 
- use control connection (gRPC)
- use uint64 everywhere => fixes https://github.com/zrepl/zrepl/issues/463
- [BREAK] bump protocol version

closes https://github.com/zrepl/zrepl/pull/518
fixes https://github.com/zrepl/zrepl/issues/463
 2021-10-09 15:43:27 +02:00
Christian Schwarz a8e92971d0 zfs: rewrite SendStream, fix bug in Close() on FreeBSD, add platformtests 
This commit was motivated by https://github.com/zrepl/zrepl/issues/495
where, on FreeBSD with OpenZFS 2.0, a SendStream.Close() call might wait indefinitely for `zfs send` to exit.
The reason is that, due to the refactoring done for redacted send & recv
(https://github.com/openzfs/zfs/commit/30af21b02569ac192f52ce6e6511015f8a8d5729),
the `dump_bytes` function, which writes to the pipe, executes in a separate thread (synctask taskq) iff not `HAVE_LARGE_STACKS`.
The `zfs send` process/thread waits for that taskq thread using an uninterruptible primitive.
So when we SIGKILL `zfs send`, that signal doesn't reach the right thread to interrupt the pipe write.

Theoretically this affects both Linux and FreeBSD, but most Linux users `HAVE_LARGE_STACKS` and since https://github.com/penzfs/zfs/pull/12350/files OpenZFS on FreeBSD `HAVE_LARGE_STACKS` as well.
However, at least until FreeBSD 13.1, possibly for the entire 13 lifecycle, we're going to have to live with that oddity.

Measures taken in this commit:
- Report the behavior as an upstream bug https://github.com/openzfs/zfs/issues/12500
- Change SendStream code so that it closes zrepl's read-end of the pipe (see comment in code)
- Clean up and make explicit SendStream's state handling
- Write extensive platformtests for SendStream
    - They pass on my Linux install and on FreeBSD 12
    - FreeBSD 13 still needs testing.

fixes https://github.com/zrepl/zrepl/issues/495
 2021-09-19 20:11:31 +02:00
Christian Schwarz b54e477602 platformtest: fix 'active child tasks' panic for ReceiveForceRollbackWorksUnencrypted 
Revealed by rework of SendStream in a prior commit.
 2021-09-19 20:03:01 +02:00
Christian Schwarz 959fb08a89 platformtest: fix replication tests (SizeEstimationConcurrency field in PlannerPolicy was not set) 
fixup of 0ceea1b792 (parallel replication knobs)
 2021-09-19 20:03:01 +02:00
Christian Schwarz 6ac012aa3c platformtest: work around missing feature detection for test 'ReplicationPropertyReplicationWorks' 2021-09-19 20:03:01 +02:00
Christian Schwarz 3e93b31f75 platformtest: fix bandwidth-limiting-related panics (missing BucketCapacity in sender/receiver config) 
panic while running test: invalid config`Ratelimit` field invalid: BucketCapacity must not be zero
main.runTestCase.func1.1
	/home/cs/zrepl/zrepl/platformtest/harness/harness.go:190
runtime.gopanic
	/home/cs/go1.13/src/runtime/panic.go:679
github.com/zrepl/zrepl/endpoint.NewSender
	/home/cs/zrepl/zrepl/endpoint/endpoint.go:68
github.com/zrepl/zrepl/platformtest/tests.replicationInvocation.Do
	/home/cs/zrepl/zrepl/platformtest/tests/replication.go:87
github.com/zrepl/zrepl/platformtest/tests.ReplicationFailingInitialParentProhibitsChildReplication
	/home/cs/zrepl/zrepl/platformtest/tests/replication.go:925
main.runTestCase.func1
	/home/cs/zrepl/zrepl/platformtest/harness/harness.go:193
main.runTestCase
	/home/cs/zrepl/zrepl/platformtest/harness/harness.go:194
main.HarnessRun
	/home/cs/zrepl/zrepl/platformtest/harness/harness.go:107
main.main
	/home/cs/zrepl/zrepl/platformtest/harness/harness.go:42
runtime.main
	/home/cs/go1.13/src/runtime/proc.go:203
runtime.goexit
	/home/cs/go1.13/src/runtime/asm_amd64.s:1357

fixup for f5f269bfd5 (bandwidth limiting)
 2021-09-19 20:03:01 +02:00
Christian Schwarz 08df208149 bandwidth limiting: use correct field name on error 
fixup for f5f269bfd5 (bandwidth limiting)
 2021-09-19 20:03:01 +02:00
Christian Schwarz 936ed73a45 rpc/dataconn: fix log message in closeErr case 
refs #457
 2021-09-19 18:40:39 +02:00
Christian Schwarz 8fee536260 transport/tcp: ipmap tests: remove tests that cover CIDR normalization 
They broke on Go 1.17.
See
https://www.bleepingcomputer.com/news/security/go-rust-net-library-affected-by-critical-ip-address-validation-vulnerability/
for context.

fixes #514
 2021-09-15 08:43:08 +02:00
Christian Schwarz 01b4792974 fix make deb-docker for all platforms but amd64 2021-09-13 22:54:21 +02:00
Christian Schwarz ad80bb3735 status: byteprogresshistory: disable averaging as workaround for #497 
refs #497
 2021-09-12 20:08:44 +02:00
Christian Schwarz f5f269bfd5 send/recv: job-level bandwidth limiting 
Sponsored-by: Prominic.NET, Inc.

fixes #339
 2021-09-12 20:08:43 +02:00
Christian Schwarz 5b16769057 docs: update supporters 2021-08-30 11:01:25 +02:00
Christian Schwarz 009bd410af docs: prune: improve grid example 2021-07-08 19:46:24 +02:00
Christian Schwarz bcfcd7a134 docs / CI: stop creating churn with doc commits & commit as zreplbot@ 2021-07-08 17:07:24 +02:00
Matthias Freund bf1276f767 status: port status-v1 ETA calculation patch 
Must have forgotten to integrate it into the status-v2 branch at the
time.

refs https://github.com/zrepl/zrepl/issues/98#issuecomment-872154091

cc @dcdamien
 2021-07-08 15:00:26 +02:00
James W. Brinkerhoff IV 9fa7a18351 docs: quickstart: external_disk: fix typo in example 'derive -> drive' 
closes #472
 2021-04-19 23:36:03 +02:00
sre 50e8ee4549 docs: apt repo: use sudo in the snippet that sets up the repo 
I generally like when snippets are provided in a way which could be used without running as root, and uses sudo when applicable. This change allows for this.

It will, however print out one extra line, which is possible to remove by adding '>/dev/null' after '/etc/apt/sources.list.d/zrepl.list'.

closes #461
 2021-04-17 21:50:36 +02:00
Lapo Luchini 3b5a1a8b9a docs/monitoring: change suggested prometheus port to 9811 
Change to 9811 as registered with the prometheus project now.

Closes #444.
 2021-03-28 18:18:02 +02:00
Christian Schwarz f661d9429f pruning/keep_last_n: correctly handle the case where count > matching snaps 
fixes #446
 2021-03-25 22:42:01 +01:00
InsanePrawn ac4b109872 status/interactive: Revert to simple wakeup/reset signalling 
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>

closes #452
 2021-03-25 22:26:17 +01:00
InsanePrawn b2c6e51a43 client/signal: Revert "add signal 'snapshot', rename existing signal 'wakeup' to 'replication'" 
This was merged to master prematurely as the job components are not decoupled well enough
for these signals to be useful yet.

This reverts commit 2c8c2cfa14.

closes #452
 2021-03-25 22:26:17 +01:00
Lukas Schauer ee2336a24b zfs: pipe size: default to value of /proc/sys/fs/pipe-max-siz 
Addition by @problame: move getPipeCapacityHint() into platform-specific
code. This has the added benefit of not recognizing the envvar as an
envconst on platform that do not support resizing pipes.  => won't show
up in (zrepl status --raw).Global.Envconst

fixes #424
cloes #449
 2021-03-25 22:24:50 +01:00
Cole Helbling 1e85b1cb5f client/status: allow raw mode without a tty 
A fairly common pattern when presented with JSON is to pipe it to `jq`.

This also allows one to `zrepl status --mode raw > log` and operate on
the JSON there.

closes #442
 2021-03-22 23:58:32 +01:00
Christian Schwarz 5c6d69a69c zfs: PropertySource: set type to uint32 so that enumer-generated code is platform-independent 
make zrepl-bin test-platform-bin vet lint GOOS=freebsd   GOARCH=386
make[2]: Entering directory '/src'
GO111MODULE=on go build -mod=readonly  -ldflags "-X github.com/zrepl/zrepl/version.zreplVersion=v0.3.1-20-g07f2bff" -o "artifacts/zrepl-freebsd-386"
zfs/propertysource_enumer.go:41:9: constant 18446744073709551615 overflows PropertySource
zfs/propertysource_enumer.go:48:66: constant 18446744073709551615 overflows PropertySource
zfs/propertysource_enumer.go:57:23: constant 18446744073709551615 overflows PropertySource

fixes #429
 2021-03-14 22:32:45 +01:00
Christian Schwarz 299808aaaf docs: 0.4 changelog: mention the 0.3.1 fix to prune:grid 
fixes #400
refs 3a4e841c73
 2021-03-14 21:13:26 +01:00
Christian Schwarz 7071adc774 docs: 0.4 changelog 2021-03-14 20:56:02 +01:00
InsanePrawn 8d678eed19 docs: add note about zfs recv -x mountpoint with ZVOLs 
refs #430
 2021-03-14 20:26:39 +01:00
Calistoc ab7abd9686 status: show the current replication attempt's runtime 2021-03-14 18:24:28 +01:00
Christian Schwarz a58ce74ed0 implement new 'zrepl status' 
Primary goals:

- Scrollable output ( fixes #245 )
- Sending job signals from status view
- Filtering of output by filesystem

Implementation:

- original TUI framework: github.com/rivo/tview
- but: tview is quasi-unmaintained, didn't support some features
- => use fork https://gitlab.com/tslocum/cview
- however, don't buy into either too much to avoid lock-in

- instead: **port over the existing status UI drawing code
  and adjust it to produce strings instead of directly
  drawing into the termbox buffer**

Co-authored-by: Calistoc <calistoc@protonmail.com>
Co-authored-by: InsanePrawn <insane.prawny@gmail.com>

fixes #245
fixes #220
 2021-03-14 18:24:25 +01:00
Calistoc 2c8c2cfa14 add signal 'snapshot', rename existing signal 'wakeup' to 'replication' 2021-03-14 18:16:23 +01:00
Christian Schwarz 0ceea1b792 replication: simplify parallel replication variables & expose them in config 
closes #140
 2021-03-14 17:30:10 +01:00
Christian Schwarz 07f2bfff6a build: bump release build Go version + quickcheck default to Go 1.16 2021-02-20 17:29:17 +01:00
Christian Schwarz b9ee14f6d7 endpoint.ReceiverConfig: remove obsolete TODO comment 2021-02-20 17:20:58 +01:00
InsanePrawn 393fc10a69 [#285] support setting zfs send / recv flags in the config (send: -wLcepbS, recv: -ox) 
Co-authored-by: Christian Schwarz <me@cschwarz.com>
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>

closes #285
closes #276
closes #24
 2021-02-20 17:20:45 +01:00
Christian Schwarz 1c937e58f7 zfs.NilBool: document its purpose and move it to its own package 'nodefault' 2021-02-20 17:04:57 +01:00
Christian Schwarz 70bbdfe760 zfs: ResumeToken: parse embedok, largeblockok, savedok if available 
Developed for #285 but ultimately not used for it.
 2021-02-20 17:04:57 +01:00
Christian Schwarz efe7b17d21 Update to protobuf v1.25 and grpc 1.35; bump CI to go1.12 
From:
github.com/golang/protobuf v1.3.2
google.golang.org/grpc v1.17.0

To:
github.com/golang/protobuf v1.4.3
google.golang.org/grpc v1.35.0
google.golang.org/protobuf v1.25.0

About the two protobuf packages:
https://developers.google.com/protocol-buffers/docs/reference/go/faq
> Version v1.4.0 and higher of github.com/golang/protobuf wrap the new
implementation and permit programs to adopt the new API incrementally. For
example, the well-known types defined in github.com/golang/protobuf/ptypes are
simply aliases of those defined in the newer module. Thus,
google.golang.org/protobuf/types/known/emptypb and
github.com/golang/protobuf/ptypes/empty may be used interchangeably.

Notable Code Changes in zrepl:
- generate protobufs now contain a mutex so we can't copy them by value
  anymore
- grpc.WithDialer is deprecated => use grpc.WithContextDialer instead

Go1.12 is now actually required by some of the dependencies.
 2021-01-25 00:39:01 +01:00
Christian Schwarz 166e80bb96 lazy.sh: use 'go install' + import paths from build/tools.go 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 6ac537210b daemon: avoid math/rand race by using global source 
Unless we're using the global source for math/rand, (*rand.Rand).Read
should not be called concurrently. We seed the rng in daemon.Run to
avoid ambiguity or hiding global side effects inside packages.

closes #414
 2021-01-25 00:16:01 +01:00
Christian Schwarz 596a39c0f5 bump golangci-lint to 1.35.2 and fix resulting lint errors 
GO111MODULE=on golangci-lint run ./...
endpoint/endpoint.go:487:9: S1039: unnecessary use of fmt.Sprintf (gosimple)
                panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
                      ^
platformtest/platformtest_ops.go:259:41: S1039: unnecessary use of fmt.Sprintf (gosimple)
                                return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
                                                                    ^
platformtest/platformtest_ops.go:266:41: S1039: unnecessary use of fmt.Sprintf (gosimple)
                                return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
                                                                    ^
util/optionaldeadline/optionaldeadline_test.go:97:50: SA1029: should not use built-in type string as key for value; define your own type to avoid collisions (staticcheck)
        pctx := context.WithValue(context.Background(), "key", "value")
                                                        ^
rpc/rpc_debug.go:8:5: var `debugEnabled` is unused (unused)
rpc/dataconn/dataconn_debug.go:8:5: var `debugEnabled` is unused (unused)
rpc/dataconn/frameconn/frameconn.go:42:9: S1039: unnecessary use of fmt.Sprintf (gosimple)
                panic(fmt.Sprintf("frame header is 8 bytes long"))
                      ^
platformtest/platformtest_ops.go:322:40: S1039: unnecessary use of fmt.Sprintf (gosimple)
                        return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson d118bcc717 rpc: fix data race in timeoutconn 
- `timeoutconn` handles state, yet calls to Read/Write make a copy of
  that state (non-pointer receiver) so any outbound calls will not have
  the state updated

- Even without the copy issue, the renew methods can in edge cases set a
  new deadline _after_ DisableTimeouts have been called, consider the
  following racy behavior:
    1. `renewReadDeadline` is called, checks `renewDeadlinesDisabled`
       (not disabled)
    2. `DisableTimeouts` is called, sets `renewDeadlinesDisabled`
    3. `DisableTimeouts` invokes `c.SetDeadline`
    4. `renewReadDeadline` invokes `c.SetReadDeadline`

To fix the above, the `Conn` receiver was made to be a pointer
everywhere and access to renewDeadlinesDisabled is now guarded
by an RWMutex instead of using atomics.

closes #415
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 48be4032a2 daemon: fix data race in snapjob pruner report 
closes #416
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 2d545c87cd rpc: fix read mutex unlock in dataconn stream 
closes #413
 2021-01-25 00:16:01 +01:00
Christian Schwarz 0b48ba54f8 replication/driver: simplify second-attempt step correlation code & fix statekeeping 
Before this change, the step correlation code returned early in several cases:
- did not set f.planning.done in the cases where it was a no-op
- did not set f.planning.err in the cases where correlation did not
  succeed

Reported-by: InsanePrawn <insane.prawny@gmail.com>
 2021-01-25 00:16:01 +01:00
Christian Schwarz c3d87289bb [#388] util/semaphore: fix TestSemaphore test 
fixes #388
 2021-01-24 22:28:47 +01:00
Christian Schwarz 0d96627ffb [#388] trace: make WithTaskGroup actually concurrent 2021-01-24 22:28:47 +01:00
Christian Schwarz 61acc7494a [#388] endpoint: fix incorrect use of trace.WithTaskGroup in ListAbstractionsStreamed 
Capturing behavior was broken.
 2021-01-24 13:58:23 +01:00
175 changed files with 10676 additions and 4325 deletions
Expand all files Collapse all files
.circleci/config.yml
+14 -13
View File
@@ -150,7 +150,7 @@ parameters:
release_docker_baseimage_tag:
type: string
default: "1.15"
default: "1.17"
workflows:
version: 2
@@ -160,19 +160,19 @@ workflows:
jobs:
- quickcheck-docs
- quickcheck-go: &quickcheck-go-smoketest
name: quickcheck-go-amd64-linux-1.15
goversion: &latest-go-release "1.15"
name: quickcheck-go-amd64-linux-1.17
goversion: &latest-go-release "1.17"
goos: linux
goarch: amd64
- test-go-on-latest-go-release:
goversion: *latest-go-release
- quickcheck-go:
requires:
- quickcheck-go-amd64-linux-1.15 #quickcheck-go-smoketest.name
- quickcheck-go-amd64-linux-1.17 #quickcheck-go-smoketest.name
matrix: &quickcheck-go-matrix
alias: quickcheck-go-matrix
parameters:
goversion: [*latest-go-release, "1.11"]
goversion: [*latest-go-release, "1.12"]
goos: ["linux", "freebsd"]
goarch: ["amd64", "arm64"]
exclude:
@@ -180,8 +180,8 @@ workflows:
- goversion: *latest-go-release
goos: linux
goarch: amd64
# not supported by Go 1.11
- goversion: "1.11"
# not supported by Go 1.12
- goversion: "1.12"
goos: freebsd
goarch: arm64
@@ -231,8 +231,6 @@ jobs:
- install-docdep
- run: make docs
- store_artifacts:
path: artifacts
- download-and-install-minio-client
- upload-minio:
src: artifacts
@@ -296,7 +294,8 @@ jobs:
# don't save-cache-gomod here, test-go doesn't pull all the dependencies
release-build:
machine: true
machine:
image: ubuntu-2004:202201-02
steps:
- checkout
- run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
@@ -304,7 +303,8 @@ jobs:
root: .
paths: [.]
release-deb:
machine: true
machine:
image: ubuntu-2004:202201-02
steps:
- attach_workspace:
at: .
@@ -315,7 +315,8 @@ jobs:
- "artifacts/*.deb"
release-rpm:
machine: true
machine:
image: ubuntu-2004:202201-02
steps:
- attach_workspace:
at: .
@@ -358,7 +359,7 @@ jobs:
subcommand: docdep
- run:
command: |
git config --global user.email "me@cschwarz.com"
git config --global user.email "zreplbot@cschwarz.com"
git config --global user.name "zrepl-github-io-ci"
# https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
.golangci.yml
+6
View File
@@ -7,4 +7,10 @@ issues:
- path: _test\.go
linters:
- errcheck
# Disable staticcheck 'Empty body in an if or else branch' as it's useful
# to put a comment into an empty else-clause that explains why whatever
# is done in the if-caluse is not necessary if the condition is false.
- linters:
- staticcheck
text: "SA9003:"
Makefile
+12 -5
View File
@@ -28,7 +28,7 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
GOLANGCI_LINT := golangci-lint
GOCOVMERGE := gocovmerge
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.15
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.17
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
ifneq ($(GOARM),)
@@ -55,7 +55,7 @@ release: clean
$(MAKE) wrapup-and-checksum
$(MAKE) check-git-clean
ifeq (SIGN, 1)
$(make) sign
$(MAKE) sign
endif
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
@@ -173,6 +173,10 @@ check-git-clean:
fi; \
fi;
tag-release:
test -n "$(ZREPL_TAG_VERSION)" || exit 1
git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
sign:
gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \
--armor \
@@ -189,6 +193,8 @@ GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split
bins-all:
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=amd64
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=386
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=arm GOARM=7
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=arm64
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=amd64
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm64
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm GOARM=7
@@ -286,7 +292,8 @@ cover-full:
.PHONY: generate formatcheck format
generate: generate-platform-test-list
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
@@ -326,12 +333,12 @@ $(ARTIFACTDIR)/go_env.txt:
docs: $(ARTIFACTDIR)/docs
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
make -C docs \
$(MAKE) -C docs \
html \
BUILDDIR=../artifacts/docs \
SPHINXOPTS="-W --keep-going -n"
docs-clean:
make -C docs \
$(MAKE) -C docs \
clean \
BUILDDIR=../artifacts/docs
README.md
+4 -3
View File
@@ -1,11 +1,12 @@
[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
[![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
[![Language: Go](https://img.shields.io/badge/lang-Go-6ad7e5.svg)](https://golang.org/)
[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
[![Chat](https://img.shields.io/badge/chat-matrix-blue.svg)](https://matrix.to/#/#zrepl:matrix.org)
# zrepl
zrepl is a one-stop ZFS backup & replication solution.
@@ -95,7 +96,7 @@ Downstream packagers can read the changelog to determine whether they want to pu
### Additional Notes to Distro Package Maintainers
* Use `sudo make test-platform-bin && sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
* Ship a default config that adheres to your distro's `hier` and logging system.
* Ship a service manager file and _please_ try to upstream it to this repository.
* `dist/systemd` contains a Systemd unit template.
build/go.mod
+5 -41
View File
@@ -3,50 +3,14 @@ module github.com/zrepl/zrepl/build
go 1.12
require (
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
github.com/alvaroloes/enumer v1.1.1
github.com/fatih/color v1.7.0 // indirect
github.com/go-critic/go-critic v0.3.4 // indirect
github.com/gogo/protobuf v1.2.1 // indirect
github.com/golang/mock v1.2.0 // indirect
github.com/golang/protobuf v1.2.0
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
github.com/golangci/golangci-lint v1.17.1
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
github.com/golangci/golangci-lint v1.35.2
github.com/golangci/misspell v0.3.4 // indirect
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
github.com/google/go-cmp v0.3.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/pkg/errors v0.8.1 // indirect
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.3.2 // indirect
github.com/stretchr/testify v1.4.0 // indirect
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad // indirect
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
)
// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
replace (
github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
google.golang.org/protobuf v1.25.0
)
build/go.sum
+530 -114
View File
@@ -1,66 +1,133 @@
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw=
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk=
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/ashanbrown/forbidigo v1.0.0 h1:QdNXBduDUopc3GW+YVYZn8jzmIMklQiCfdN2N5+dQeE=
github.com/ashanbrown/forbidigo v1.0.0/go.mod h1:PH+zMRWE15yW69fYfe7Kn8nYR6yYyafc3ntEGh2BBAg=
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a h1:/U9tbJzDRof4fOR51vwzWdIBsIH6R2yU0KG1MBRM2Js=
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8=
github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg=
github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218=
github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-critic/go-critic v0.5.3 h1:xQEweNxzBNpSqI3wotXZAixRarETng3PTG4pkcrLCOA=
github.com/go-critic/go-critic v0.5.3/go.mod h1:2Lrs1m4jtOnnG/EdezbSpAoL0F2pRW+9HWJUZ+QaktY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=
github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
@@ -69,26 +136,16 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpc
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.35.2 h1:hD1999/sq3tCPXhhI4UpunxpAAdH9pK7kDIObqoGuWA=
github.com/golangci/golangci-lint v1.35.2/go.mod h1:Sg5fFp5oLLI1B8gXfUVUSePju8XF0uWefMkuZuGIHUo=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
@@ -101,182 +158,541 @@ github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gookit/color v1.3.1/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw=
github.com/gostaticanalysis/analysisutil v0.4.1 h1:/7clKqrVfiVwiBQLM0Uke4KvXnO6JcCTS7HwF2D6wG8=
github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0=
github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI=
github.com/gostaticanalysis/comment v1.4.1 h1:xHopR5L2lRz6OsjH4R2HG5wRhW9ySl3FsHIvi5pcXwc=
github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 h1:7nkB9fLPMwtn/R6qfPcHileL/x9ydlhw8XyDrLI1ZXg=
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kulti/thelper v0.2.1 h1:H4rSHiB3ALx//SXr+k9OPqKoOw2cAZpIQwVNH1RL5T4=
github.com/kulti/thelper v0.2.1/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
github.com/kunwardeep/paralleltest v1.0.2 h1:/jJRv0TiqPoEy/Y8dQxCFJhD56uS/pnvtatgTZBHokU=
github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30=
github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ=
github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mbilski/exhaustivestruct v1.1.0 h1:4ykwscnAFeHJruT+EY3M3vdeP8uXMh0VV2E61iR7XD8=
github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4=
github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw=
github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4=
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278 h1:5gcJ7tORNCNB2QjOJF+MYjzS9aiWpxhP3gntf7RVrOQ=
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryancurrah/gomodguard v1.2.0 h1:YWfhGOrXwLGiqcC/u5EqG6YeS8nh+1fw0HEc85CVZro=
github.com/ryancurrah/gomodguard v1.2.0/go.mod h1:rNqbC4TOIdUDcVMSIpNNAzTbzXAZa6W5lnUepvuMMgQ=
github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/securego/gosec/v2 v2.5.0 h1:kjfXLeKdk98gBe2+eYRFMpC4+mxmQQtbidpiiOQ69Qc=
github.com/securego/gosec/v2 v2.5.0/go.mod h1:L/CDXVntIff5ypVHIkqPXbtRpJiNCh6c6Amn68jXDjo=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI=
github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA=
github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
github.com/tetafro/godot v1.3.2 h1:HzWC3XjadkyeuBZxkfAFNY20UVvle0YD51I6zf6RKlU=
github.com/tetafro/godot v1.3.2/go.mod h1:ah7jjYmOMnIjS9ku2krapvGQrFNtTLo9Z/qB3dGU1eU=
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d h1:3EZyvNUMsGD1QA8cu0STNn1L7I77rvhf2IhOcHYQhSw=
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65 h1:Y0bLA422kvb32uZI4fy/Plop/Tbld0l9pSzl+j1FWok=
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65/go.mod h1:T22e7iRN4LsFPZGyRLRXeF+DWVXFuV9thsyO7NjbbTI=
github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs=
github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201030010431-2feb2bb1ff51/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130 h1:8qSBr5nyKsEgkP918Pu5FFDZpTtLIjXSo6mrtdVOFfk=
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc=
honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw=
mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
build/tools.go
+4 -1
View File
@@ -1,12 +1,15 @@
//go:build tools
// +build tools
package main
// the lines are parsed by lazy.sh, do not edit
import (
_ "github.com/alvaroloes/enumer"
_ "github.com/golang/protobuf/protoc-gen-go"
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
_ "github.com/wadey/gocovmerge"
_ "golang.org/x/tools/cmd/goimports"
_ "golang.org/x/tools/cmd/stringer"
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
_ "google.golang.org/protobuf/cmd/protoc-gen-go"
)
client/configcheck.go
+11 -1
View File
@@ -21,6 +21,7 @@ import (
var configcheckArgs struct {
format string
what string
skipCertCheck bool
}
var ConfigcheckCmd = &cli.Subcommand{
@@ -29,6 +30,7 @@ var ConfigcheckCmd = &cli.Subcommand{
SetupFlags: func(f *pflag.FlagSet) {
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
formatMap := map[string]func(interface{}){
@@ -56,8 +58,16 @@ var ConfigcheckCmd = &cli.Subcommand{
}
var hadErr bool
parseFlags := config.ParseFlagsNone
if configcheckArgs.skipCertCheck {
parseFlags |= config.ParseFlagsNoCertCheck
}
// further: try to build jobs
confJobs, err := job.JobsFromConfig(subcommand.Config())
confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
if err != nil {
err := errors.Wrap(err, "cannot build jobs from config")
if configcheckArgs.what == "jobs" {
client/migrate.go
+1 -1
View File
@@ -129,7 +129,7 @@ func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []
}
cfg := sc.Config()
jobs, err := job.JobsFromConfig(cfg)
jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
if err != nil {
fmt.Printf("cannot parse config:\n%s\n\n", err)
fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
client/status.go
-816
View File
@@ -1,816 +0,0 @@
package client
import (
"context"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strings"
"sync"
"time"
// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
// as of tcell#252, the number of default distributed terminals is relatively limited
// additional terminal definitions can be included via side-effect import
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
"github.com/gdamore/tcell/termbox"
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/replication/report"
)
type byteProgressMeasurement struct {
time time.Time
val int64
}
type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional
changeCount int
lastChange time.Time
bpsAvg float64
}
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
if p.last == nil {
p.last = &byteProgressMeasurement{
time: time.Now(),
val: currentVal,
}
return 0, 0
}
if p.last.val != currentVal {
p.changeCount++
p.lastChange = time.Now()
}
if time.Since(p.lastChange) > 3*time.Second {
p.last = nil
return 0, 0
}
deltaV := currentVal - p.last.val
deltaT := time.Since(p.last.time)
rate := float64(deltaV) / deltaT.Seconds()
factor := 0.3
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
p.last.time = time.Now()
p.last.val = currentVal
return int64(p.bpsAvg), p.changeCount
}
type tui struct {
x, y int
indent int
lock sync.Mutex //For report and error
report map[string]*job.Status
err error
jobFilter string
replicationProgress map[string]*bytesProgressHistory // by job name
}
func newTui() tui {
return tui{
replicationProgress: make(map[string]*bytesProgressHistory),
}
}
const INDENT_MULTIPLIER = 4
func (t *tui) moveLine(dl int, col int) {
t.y += dl
t.x = t.indent*INDENT_MULTIPLIER + col
}
func (t *tui) write(text string) {
for _, c := range text {
if c == '\n' {
t.newline()
continue
}
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
t.x += 1
}
}
func (t *tui) printf(text string, a ...interface{}) {
t.write(fmt.Sprintf(text, a...))
}
func wrap(s string, width int) string {
var b strings.Builder
for len(s) > 0 {
rem := width
if rem > len(s) {
rem = len(s)
}
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
rem = idx + 1
}
untilNewline := strings.TrimRight(s[:rem], "\n\r")
s = s[rem:]
if len(untilNewline) == 0 {
continue
}
b.WriteString(untilNewline)
b.WriteString("\n")
}
return strings.TrimRight(b.String(), "\n\r")
}
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
whole := fmt.Sprintf(format, a...)
width, _ := termbox.Size()
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
t.printf(format, a...)
} else {
t.addIndent(1)
t.newline()
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
t.addIndent(-1)
}
}
func (t *tui) newline() {
t.moveLine(1, 0)
}
func (t *tui) setIndent(indent int) {
t.indent = indent
t.moveLine(0, 0)
}
func (t *tui) addIndent(indent int) {
t.indent += indent
t.moveLine(0, 0)
}
var statusFlags struct {
Raw bool
Job string
}
var StatusCmd = &cli.Subcommand{
Use: "status",
Short: "show job activity or dump as JSON for monitoring",
SetupFlags: func(f *pflag.FlagSet) {
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
},
Run: runStatus,
}
func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
if err != nil {
return err
}
if statusFlags.Raw {
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Fprintf(os.Stderr, "Received error response:\n")
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
if err != nil {
return err
}
return errors.Errorf("exit")
}
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
return err
}
return nil
}
t := newTui()
t.lock.Lock()
t.err = errors.New("Got no report yet")
t.lock.Unlock()
t.jobFilter = statusFlags.Job
err = termbox.Init()
if err != nil {
return err
}
defer termbox.Close()
update := func() {
var m daemon.Status
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
struct{}{},
&m,
)
t.lock.Lock()
t.err = err2
t.report = m.Jobs
t.lock.Unlock()
t.draw()
}
update()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
go func() {
for range ticker.C {
update()
}
}()
termbox.HideCursor()
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
loop:
for {
switch ev := termbox.PollEvent(); ev.Type {
case termbox.EventKey:
switch ev.Key {
case termbox.KeyEsc:
break loop
case termbox.KeyCtrlC:
break loop
}
case termbox.EventResize:
t.draw()
}
}
return nil
}
func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
p, ok := t.replicationProgress[jobName]
if !ok {
p = &bytesProgressHistory{}
t.replicationProgress[jobName] = p
}
return p
}
func (t *tui) draw() {
t.lock.Lock()
defer t.lock.Unlock()
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
t.x = 0
t.y = 0
t.indent = 0
if t.err != nil {
t.write(t.err.Error())
} else {
//Iterate over map in alphabetical order
keys := make([]string, 0, len(t.report))
for k := range t.report {
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
continue
}
if t.jobFilter != "" && k != t.jobFilter {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
if len(keys) == 0 {
t.setIndent(0)
t.printf("no jobs to display")
t.newline()
termbox.Flush()
return
}
for _, k := range keys {
v := t.report[k]
t.setIndent(0)
t.printf("Job: %s", k)
t.setIndent(1)
t.newline()
t.printf("Type: %s", v.Type)
t.setIndent(1)
t.newline()
if v.Type == job.TypePush || v.Type == job.TypePull {
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
if !ok || activeStatus == nil {
t.printf("ActiveSideStatus is null")
t.newline()
continue
}
t.printf("Replication:")
t.newline()
t.addIndent(1)
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
t.addIndent(-1)
t.printf("Pruning Sender:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(activeStatus.PruningSender)
t.addIndent(-1)
t.printf("Pruning Receiver:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(activeStatus.PruningReceiver)
t.addIndent(-1)
if v.Type == job.TypePush {
t.printf("Snapshotting:")
t.newline()
t.addIndent(1)
t.renderSnapperReport(activeStatus.Snapshotting)
t.addIndent(-1)
}
} else if v.Type == job.TypeSnap {
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
if !ok || snapStatus == nil {
t.printf("SnapJobStatus is null")
t.newline()
continue
}
t.printf("Pruning snapshots:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(snapStatus.Pruning)
t.addIndent(-1)
t.printf("Snapshotting:")
t.newline()
t.addIndent(1)
t.renderSnapperReport(snapStatus.Snapshotting)
t.addIndent(-1)
} else if v.Type == job.TypeSource {
st := v.JobSpecific.(*job.PassiveStatus)
t.printf("Snapshotting:\n")
t.addIndent(1)
t.renderSnapperReport(st.Snapper)
t.addIndent(-1)
} else {
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
t.newline()
asYaml, err := yaml.Marshal(v.JobSpecific)
if err != nil {
t.printf("Error marshaling status to YAML: %s", err)
t.newline()
continue
}
t.write(string(asYaml))
t.newline()
continue
}
}
}
termbox.Flush()
}
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
if rep == nil {
t.printf("...\n")
return
}
if rep.WaitReconnectError != nil {
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
t.newline()
}
if !rep.WaitReconnectSince.IsZero() {
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
var until string
if rep.WaitReconnectUntil.IsZero() {
until = "waiting indefinitely"
} else {
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
}
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
rep.WaitReconnectSince, until)
} else {
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
}
t.newline()
}
// TODO visualize more than the latest attempt by folding all attempts into one
if len(rep.Attempts) == 0 {
t.printf("no attempts made yet")
return
} else {
t.printf("Attempt #%d", len(rep.Attempts))
if len(rep.Attempts) > 1 {
t.printf(". Previous attempts failed with the following statuses:")
t.newline()
t.addIndent(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
t.newline()
}
t.addIndent(-1)
} else {
t.newline()
}
}
latest := rep.Attempts[len(rep.Attempts)-1]
sort.Slice(latest.Filesystems, func(i, j int) bool {
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
})
t.printf("Status: %s", latest.State)
t.newline()
if latest.State == report.AttemptPlanningError {
t.printf("Problem: ")
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
t.newline()
} else if latest.State == report.AttemptFanOutError {
t.printf("Problem: one or more of the filesystems encountered errors")
t.newline()
}
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
// Draw global progress bar
// Progress: [---------------]
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
rate, changeCount := history.Update(replicated)
eta := time.Duration(0)
if rate > 0 {
eta = time.Duration((expected-replicated)/rate) * time.Second
}
t.write("Progress: ")
t.drawBar(50, replicated, expected, changeCount)
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
if eta != 0 {
t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
}
t.newline()
if containsInvalidSizeEstimates {
t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
t.newline()
}
if len(latest.Filesystems) == 0 {
t.write("NOTE: no filesystems were considered for replication!")
t.newline()
}
var maxFSLen int
for _, fs := range latest.Filesystems {
if len(fs.Info.Name) > maxFSLen {
maxFSLen = len(fs.Info.Name)
}
}
for _, fs := range latest.Filesystems {
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
}
}
}
func (t *tui) renderPrunerReport(r *pruner.Report) {
if r == nil {
t.printf("...\n")
return
}
state, err := pruner.StateString(r.State)
if err != nil {
t.printf("Status: %q (parse error: %q)\n", r.State, err)
return
}
t.printf("Status: %s", state)
t.newline()
if r.Error != "" {
t.printf("Error: %s\n", r.Error)
}
type commonFS struct {
*pruner.FSReport
completed bool
}
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false})
}
for i := range r.Completed {
all = append(all, commonFS{&r.Completed[i], true})
}
switch state {
case pruner.Plan:
fallthrough
case pruner.PlanErr:
return
}
if len(all) == 0 {
t.printf("nothing to do\n")
return
}
var totalDestroyCount, completedDestroyCount int
var maxFSname int
for _, fs := range all {
totalDestroyCount += len(fs.DestroyList)
if fs.completed {
completedDestroyCount += len(fs.DestroyList)
}
if maxFSname < len(fs.Filesystem) {
maxFSname = len(fs.Filesystem)
}
}
// global progress bar
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
t.write("Progress: ")
t.write("[")
t.write(times("=", progress))
t.write(">")
t.write(times("-", 80-progress))
t.write("]")
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.newline()
sort.SliceStable(all, func(i, j int) bool {
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
})
// Draw a table-like representation of 'all'
for _, fs := range all {
t.write(rightPad(fs.Filesystem, maxFSname, " "))
t.write(" ")
if !fs.SkipReason.NotSkipped() {
t.printf("skipped: %s\n", fs.SkipReason)
continue
}
if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") {
t.printf("ERROR:")
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else {
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
}
t.newline()
continue
}
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed {
t.printf("Completed %s\n", pruneRuleActionStr)
continue
}
t.write("Pending ") // whitespace is padding 10
if len(fs.DestroyList) == 1 {
t.write(fs.DestroyList[0].Name)
} else {
t.write(pruneRuleActionStr)
}
t.newline()
}
}
func (t *tui) renderSnapperReport(r *snapper.Report) {
if r == nil {
t.printf("<snapshot type does not have a report>\n")
return
}
t.printf("Status: %s", r.State)
t.newline()
if r.Error != "" {
t.printf("Error: %s\n", r.Error)
}
if !r.SleepUntil.IsZero() {
t.printf("Sleep until: %s\n", r.SleepUntil)
}
sort.Slice(r.Progress, func(i, j int) bool {
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
})
t.addIndent(1)
defer t.addIndent(-1)
dur := func(d time.Duration) string {
return d.Round(100 * time.Millisecond).String()
}
type row struct {
path, state, duration, remainder, hookReport string
}
var widths struct {
path, state, duration int
}
rows := make([]*row, len(r.Progress))
for i, fs := range r.Progress {
r := &row{
path: fs.Path,
state: fs.State.String(),
}
if fs.HooksHadError {
r.hookReport = fs.Hooks // FIXME render here, not in daemon
}
switch fs.State {
case snapper.SnapPending:
r.duration = "..."
r.remainder = ""
case snapper.SnapStarted:
r.duration = dur(time.Since(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
case snapper.SnapDone:
fallthrough
case snapper.SnapError:
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
}
rows[i] = r
if len(r.path) > widths.path {
widths.path = len(r.path)
}
if len(r.state) > widths.state {
widths.state = len(r.state)
}
if len(r.duration) > widths.duration {
widths.duration = len(r.duration)
}
}
for _, r := range rows {
path := rightPad(r.path, widths.path, " ")
state := rightPad(r.state, widths.state, " ")
duration := rightPad(r.duration, widths.duration, " ")
t.printf("%s %s %s", path, state, duration)
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
if r.hookReport != "" {
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
}
t.newline()
}
}
func times(str string, n int) (out string) {
for i := 0; i < n; i++ {
out += str
}
return
}
func rightPad(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
var arrowPositions = `>\|/`
// changeCount = 0 indicates stall / no progress
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
var completedLength int
if totalBytes > 0 {
completedLength = int(int64(length) * bytes / totalBytes)
if completedLength > length {
completedLength = length
}
} else if totalBytes == bytes {
completedLength = length
}
t.write("[")
t.write(times("=", completedLength))
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
t.write(times("-", length-completedLength))
t.write("]")
}
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
sizeEstimationImpreciseNotice := ""
if containsInvalidSizeEstimates {
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
}
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
}
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
strings.ToUpper(string(rep.State)),
rep.CurrentStep, len(rep.Steps),
ByteCountBinary(replicated), ByteCountBinary(expected),
sizeEstimationImpreciseNotice,
)
activeIndicator := " "
if active {
activeIndicator = "*"
}
t.printf("%s %s %s ",
activeIndicator,
rightPad(rep.Info.Name, maxFS, " "),
status)
next := ""
if err := rep.Error(); err != nil {
next = err.Err
} else if rep.State != report.FilesystemDone {
if nextStep := rep.NextStep(); nextStep != nil {
if nextStep.IsIncremental() {
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
} else {
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
}
attribs := []string{}
if nextStep.Info.Resumed {
attribs = append(attribs, "resumed")
}
attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
} else {
next = "" // individual FSes may still be in planning state
}
}
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
t.newline()
}
func ByteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
func humanizeDuration(duration time.Duration) string {
days := int64(duration.Hours() / 24)
hours := int64(math.Mod(duration.Hours(), 24))
minutes := int64(math.Mod(duration.Minutes(), 60))
seconds := int64(math.Mod(duration.Seconds(), 60))
var parts []string
force := false
chunks := []int64{days, hours, minutes, seconds}
for i, chunk := range chunks {
if force || chunk > 0 {
padding := 0
if force {
padding = 2
}
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
force = true
}
}
return strings.Join(parts, " ")
}
client/status/client/client.go
+111
View File
@@ -0,0 +1,111 @@
package client
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon"
)
type Client struct {
h *http.Client
}
func New(network, addr string) (*Client, error) {
httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
if err != nil {
return nil, err
}
return &Client{httpc}, nil
}
func (c *Client) Status() (s daemon.Status, _ error) {
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
struct{}{},
&s,
)
return s, err
}
func (c *Client) StatusRaw() ([]byte, error) {
var r json.RawMessage
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
if err != nil {
return nil, err
}
return r, nil
}
func (c *Client) signal(job, sig string) error {
return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
struct {
Name string
Op string
}{
Name: job,
Op: sig,
},
struct{}{},
)
}
func (c *Client) SignalWakeup(job string) error {
return c.signal(job, "wakeup")
}
func (c *Client) SignalReset(job string) error {
return c.signal(job, "reset")
}
func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
return &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return dialfunc(ctx)
},
},
}, nil
}
func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
var buf bytes.Buffer
encodeErr := json.NewEncoder(&buf).Encode(req)
if encodeErr != nil {
return encodeErr
}
hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
if err != nil {
return err
}
hreq.Header.Set("Content-Type", "application/json")
// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
// Found this by watching http.Server.ConnState changes, then found
// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
hreq.Close = true
resp, err := c.Do(hreq)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
var msg bytes.Buffer
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
return errors.Errorf("%s", msg.String())
}
decodeError := json.NewDecoder(resp.Body).Decode(&res)
if decodeError != nil {
return decodeError
}
return nil
}
client/status/status.go
+97
View File
@@ -0,0 +1,97 @@
package status
import (
"context"
"os"
"time"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/client/status/client"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/util/choices"
)
type Client interface {
Status() (daemon.Status, error)
StatusRaw() ([]byte, error)
SignalWakeup(job string) error
SignalReset(job string) error
}
type statusFlags struct {
Mode choices.Choices
Job string
Delay time.Duration
}
var statusv2Flags statusFlags
type statusv2Mode int
const (
StatusV2ModeInteractive statusv2Mode = 1 + iota
StatusV2ModeDump
StatusV2ModeRaw
StatusV2ModeLegacy
)
var Subcommand = &cli.Subcommand{
Use: "status",
Short: "retrieve & display daemon status information",
SetupFlags: func(f *pflag.FlagSet) {
statusv2Flags.Mode.Init(
"interactive", StatusV2ModeInteractive,
"dump", StatusV2ModeDump,
"raw", StatusV2ModeRaw,
"legacy", StatusV2ModeLegacy,
)
statusv2Flags.Mode.SetTypeString("mode")
statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runStatusV2Command(ctx, subcommand.Config(), args)
},
}
func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
c, err := client.New("unix", config.Global.Control.SockPath)
if err != nil {
return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
}
mode := statusv2Flags.Mode.Value().(statusv2Mode)
if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
if err != nil {
panic(err)
}
rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
if err != nil {
panic(err)
}
return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
}
switch mode {
case StatusV2ModeInteractive:
return interactive(c, statusv2Flags)
case StatusV2ModeDump:
return dump(c, statusv2Flags.Job)
case StatusV2ModeRaw:
return raw(c)
case StatusV2ModeLegacy:
return legacy(c, statusv2Flags)
default:
panic("unreachable")
}
}
client/status/status_dump.go
+70
View File
@@ -0,0 +1,70 @@
package status
import (
"fmt"
"os"
"strings"
"github.com/gdamore/tcell"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func dump(c Client, job string) error {
s, err := c.Status()
if err != nil {
return err
}
if job != "" {
if _, ok := s.Jobs[job]; !ok {
return errors.Errorf("job %q not found", job)
}
}
width := (1 << 31) - 1
wrap := false
hline := strings.Repeat("-", 80)
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
hline = strings.Repeat("-", width)
}
m := viewmodel.New()
params := viewmodel.Params{
Report: s.Jobs,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
m.Update(params)
for _, j := range m.Jobs() {
if job != "" && j.Name() != job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Println(m.SelectedJob().FullDescription())
if job != "" {
return nil
} else {
fmt.Println(hline)
}
}
return nil
}
client/status/status_interactive.go
+347
View File
@@ -0,0 +1,347 @@
package status
import (
"fmt"
"regexp"
"strings"
"sync"
"time"
"github.com/gdamore/tcell/v2"
tview "gitlab.com/tslocum/cview"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func interactive(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
jobDetailSplit := tview.NewFlex()
jobMenu := tview.NewTreeView()
jobMenuRoot := tview.NewTreeNode("jobs")
jobMenuRoot.SetSelectable(true)
jobMenu.SetRoot(jobMenuRoot)
jobMenu.SetCurrentNode(jobMenuRoot)
jobMenu.SetSelectedTextColor(tcell.ColorGreen)
jobTextDetail := tview.NewTextView()
jobTextDetail.SetWrap(false)
jobMenu.SetBorder(true)
jobTextDetail.SetBorder(true)
toolbarSplit := tview.NewFlex()
toolbarSplit.SetDirection(tview.FlexRow)
inputBarContainer := tview.NewFlex()
fsFilterInput := tview.NewInputField()
fsFilterInput.SetBorder(false)
fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
inputBarLabel := tview.NewTextView()
inputBarLabel.SetText("[::b]FILTER ")
inputBarLabel.SetDynamicColors(true)
inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
bottombar := tview.NewFlex()
bottombar.SetDirection(tview.FlexColumn)
bottombarDateView := tview.NewTextView()
bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
bottomBarStatus := tview.NewTextView()
bottomBarStatus.SetDynamicColors(true)
bottomBarStatus.SetTextAlign(tview.AlignRight)
bottombar.AddItem(bottomBarStatus, 0, 10, false)
toolbarSplit.AddItem(bottombar, 1, 0, false)
tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
var tabbable []tview.Primitive
tabbableActiveIndex := 0
tabbableRedraw := func() {
if len(tabbable) == 0 {
app.SetFocus(nil)
return
}
if tabbableActiveIndex >= len(tabbable) {
app.SetFocus(tabbable[0])
return
}
app.SetFocus(tabbable[tabbableActiveIndex])
}
tabbableCycle := func() {
if len(tabbable) == 0 {
return
}
tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
app.SetFocus(tabbable[tabbableActiveIndex])
tabbableRedraw()
}
jobMenuVisisble := false
reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
if jobMenuVisisble == setJobMenuVisible {
return
}
jobMenuVisisble = setJobMenuVisible
if setJobMenuVisible {
jobDetailSplit.RemoveItem(jobTextDetail)
jobDetailSplit.AddItem(jobMenu, 0, 1, true)
jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
tabbable = tabbableWithJobMenu
} else {
jobDetailSplit.RemoveItem(jobMenu)
tabbable = tabbableWithoutJobMenu
}
tabbableRedraw()
}
showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
preModalFocus := app.GetFocus()
m.SetDoneFunc(func(idx int, label string) {
if modalDoneFunc != nil {
modalDoneFunc(idx, label)
}
app.SetRoot(toolbarSplit, true)
app.SetFocus(preModalFocus)
app.Draw()
})
app.SetRoot(m, true)
app.Draw()
}
app.SetRoot(toolbarSplit, true)
// initial focus
tabbableActiveIndex = len(tabbable)
tabbableCycle()
reconfigureJobDetailSplit(true)
m := viewmodel.New()
params := &viewmodel.Params{
Report: nil,
SelectedJob: nil,
FSFilter: func(_ string) bool { return true },
DetailViewWidth: 100,
DetailViewWrap: false,
ShortKeybindingOverview: "[::b]Q[::-] quit [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines [::b]Shift+M[::-] toggle navbar [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
}
paramsMtx := &sync.Mutex{}
var redraw func()
viewmodelupdate := func(cb func(*viewmodel.Params)) {
paramsMtx.Lock()
defer paramsMtx.Unlock()
cb(params)
m.Update(*params)
}
redraw = func() {
jobs := m.Jobs()
if flag.Job != "" {
job_found := false
for _, job := range jobs {
if strings.Compare(flag.Job, job.Name()) == 0 {
jobs = []*viewmodel.Job{job}
job_found = true
break
}
}
if !job_found {
jobs = nil
}
}
redrawJobsList := false
var selectedJobN *tview.TreeNode
if len(jobMenuRoot.GetChildren()) == len(jobs) {
for i, jobN := range jobMenuRoot.GetChildren() {
if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
redrawJobsList = true
break
}
if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
selectedJobN = jobN
}
}
} else {
redrawJobsList = true
}
if redrawJobsList {
selectedJobN = nil
children := make([]*tview.TreeNode, len(jobs))
for i := range jobs {
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
jobN.SetReference(jobs[i])
jobN.SetSelectable(true)
children[i] = jobN
jobN.SetSelectedFunc(func() {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
})
})
if jobs[i] == m.SelectedJob() {
selectedJobN = jobN
}
}
jobMenuRoot.SetChildren(children)
}
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
jobMenu.SetCurrentNode(selectedJobN)
} else if selectedJobN == nil {
// select something, otherwise selection breaks (likely bug in tview)
jobMenu.SetCurrentNode(jobMenuRoot)
}
if selJ := m.SelectedJob(); selJ != nil {
jobTextDetail.SetText(selJ.FullDescription())
} else {
jobTextDetail.SetText("please select a job")
}
bottombardatestring := m.DateString()
bottombarDateView.SetText(bottombardatestring)
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
bottomBarStatus.SetText(m.BottomBarStatus())
app.Draw()
}
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
viewmodelupdate(func(p *viewmodel.Params) {
p.Report = st.Jobs
p.ReportFetchError = err
})
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
})
redraw()
jobTextDetail.ScrollToBeginning()
})
jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
app.SetFocus(jobTextDetail)
})
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
viewmodelupdate(func(p *viewmodel.Params) {
_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
})
return false
})
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
if e.Key() == tcell.KeyTab {
tabbableCycle()
return nil
}
if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == '/' {
if app.GetFocus() != fsFilterInput {
app.SetFocus(fsFilterInput)
}
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
reconfigureJobDetailSplit(!jobMenuVisisble)
return nil
}
if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
app.Stop()
}
if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
if !ok {
return nil
}
signals := []string{"wakeup", "reset"}
clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
sigMod := tview.NewModal()
sigMod.SetBackgroundColor(tcell.ColorDefault)
sigMod.SetBorder(true)
sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
sigMod.AddButtons(signals)
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
showModal(sigMod, func(idx int, _ string) {
go func() {
if idx == -1 {
return
}
err := clientFuncs[idx](job.Name())
if err != nil {
app.QueueUpdate(func() {
me := tview.NewModal()
me.SetText(fmt.Sprintf("signal error: %s", err))
me.AddButtons([]string{"Close"})
showModal(me, nil)
})
}
}()
})
}
return e
})
fsFilterInput.SetChangedFunc(func(searchterm string) {
viewmodelupdate(func(p *viewmodel.Params) {
p.FSFilter = func(fs string) bool {
r, err := regexp.Compile(searchterm)
if err != nil {
return true
}
return r.MatchString(fs)
}
})
redraw()
jobTextDetail.ScrollToBeginning()
})
fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEnter {
app.SetFocus(jobTextDetail)
return nil
}
return event
})
jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
// toggle wrapping
viewmodelupdate(func(p *viewmodel.Params) {
p.DetailViewWrap = !p.DetailViewWrap
})
redraw()
return nil
}
return event
})
return app.Run()
}
client/status/status_legacy.go
+128
View File
@@ -0,0 +1,128 @@
package status
import (
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gdamore/tcell/v2"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
tview "gitlab.com/tslocum/cview"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func legacy(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
textView := tview.NewTextView()
textView.SetWrap(true)
textView.SetScrollable(true) // so that it allows us to set scroll position
textView.SetScrollBarVisibility(tview.ScrollBarNever)
app.SetRoot(textView, true)
width := (1 << 31) - 1
wrap := false
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
}
paramsMtx := &sync.Mutex{}
params := viewmodel.Params{
Report: nil,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
redraw := func() {
textView.Clear()
paramsMtx.Lock()
defer paramsMtx.Unlock()
if params.ReportFetchError != nil {
fmt.Fprintln(textView, params.ReportFetchError.Error())
} else if params.Report != nil {
m := viewmodel.New()
m.Update(params)
for _, j := range m.Jobs() {
if flag.Job != "" && j.Name() != flag.Job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Fprintln(textView, m.SelectedJob().FullDescription())
if flag.Job != "" {
break
} else {
hline := strings.Repeat("-", params.DetailViewWidth)
fmt.Fprintln(textView, hline)
}
}
} else {
fmt.Fprintln(textView, "waiting for request results")
}
textView.ScrollToBeginning()
}
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
// sync resizes to `params`
paramsMtx.Lock()
_, _, newWidth, _ := textView.GetInnerRect()
if newWidth != params.DetailViewWidth {
params.DetailViewWidth = newWidth
app.QueueUpdateDraw(redraw)
}
paramsMtx.Unlock()
textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
return false
})
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
paramsMtx.Lock()
params.Report = st.Jobs
params.ReportFetchError = err
paramsMtx.Unlock()
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
return app.Run()
}
client/status/status_raw.go
+18
View File
@@ -0,0 +1,18 @@
package status
import (
"bytes"
"io"
"os"
)
func raw(c Client) error {
b, err := c.StatusRaw()
if err != nil {
return err
}
if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
return err
}
return nil
}
client/status/viewmodel/bytecountbinary.go
+26
View File
@@ -0,0 +1,26 @@
package viewmodel
import (
"fmt"
"math"
)
func ByteCountBinaryUint(b uint64) string {
if b > math.MaxInt64 {
panic(b)
}
return ByteCountBinary(int64(b))
}
func ByteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := unit, 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
client/status/viewmodel/bytesprogresshistory.go
+49
View File
@@ -0,0 +1,49 @@
package viewmodel
import "time"
type byteProgressMeasurement struct {
time time.Time
val uint64
}
type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional
changeCount int
lastChange time.Time
}
func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
if p.last == nil {
p.last = &byteProgressMeasurement{
time: time.Now(),
val: currentVal,
}
return 0, 0
}
if p.last.val != currentVal {
p.changeCount++
p.lastChange = time.Now()
}
if time.Since(p.lastChange) > 3*time.Second {
p.last = nil
return 0, 0
}
var deltaV int64
if currentVal >= p.last.val {
deltaV = int64(currentVal - p.last.val)
} else {
deltaV = -int64(p.last.val - currentVal)
}
deltaT := time.Since(p.last.time)
rate := float64(deltaV) / deltaT.Seconds()
p.last.time = time.Now()
p.last.val = currentVal
return int64(rate), p.changeCount
}
client/status/viewmodel/render.go
+656
View File
@@ -0,0 +1,656 @@
package viewmodel
import (
"fmt"
"math"
"sort"
"strings"
"time"
"github.com/go-playground/validator/v10"
yaml "github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/replication/report"
)
type M struct {
jobs map[string]*Job
jobsList []*Job
selectedJob *Job
dateString string
bottomBarStatus string
}
type Job struct {
// long-lived
name string
byteProgress *bytesProgressHistory
lastStatus *job.Status
fulldescription string
}
func New() *M {
return &M{
jobs: make(map[string]*Job),
jobsList: make([]*Job, 0),
selectedJob: nil,
}
}
type FilterFunc func(string) bool
type Params struct {
Report map[string]*job.Status
ReportFetchError error
SelectedJob *Job
FSFilter FilterFunc `validate:"required"`
DetailViewWidth int `validate:"gte=1"`
DetailViewWrap bool
ShortKeybindingOverview string
}
var validate = validator.New()
func (m *M) Update(p Params) {
if err := validate.Struct(p); err != nil {
panic(err)
}
if p.ReportFetchError != nil {
m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
} else {
m.bottomBarStatus = p.ShortKeybindingOverview
for jobname, st := range p.Report {
// TODO handle job renames & deletions
j, ok := m.jobs[jobname]
if !ok {
j = &Job{
name: jobname,
byteProgress: &bytesProgressHistory{},
}
m.jobs[jobname] = j
m.jobsList = append(m.jobsList, j)
}
j.lastStatus = st
}
}
// filter out internal jobs
var jobsList []*Job
for _, j := range m.jobsList {
if daemon.IsInternalJobName(j.name) {
continue
}
jobsList = append(jobsList, j)
}
m.jobsList = jobsList
// determinism!
sort.Slice(m.jobsList, func(i, j int) bool {
return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
})
// try to not lose the selected job
m.selectedJob = nil
for _, j := range m.jobsList {
j.updateFullDescription(p)
if j == p.SelectedJob {
m.selectedJob = j
}
}
m.dateString = time.Now().Format(time.RFC3339)
}
func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
func (m *M) Jobs() []*Job { return m.jobsList }
// may be nil
func (m *M) SelectedJob() *Job { return m.selectedJob }
func (m *M) DateString() string { return m.dateString }
func (j *Job) updateFullDescription(p Params) {
width := p.DetailViewWidth
if !p.DetailViewWrap {
width = 10000000 // FIXME
}
b := stringbuilder.New(stringbuilder.Config{
IndentMultiplier: 3,
Width: width,
})
drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
j.fulldescription = b.String()
}
func (j *Job) JobTreeTitle() string {
return j.name
}
func (j *Job) FullDescription() string {
return j.fulldescription
}
func (j *Job) Name() string {
return j.name
}
func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
t.Printf("Job: %s\n", name)
t.Printf("Type: %s\n\n", v.Type)
if v.Type == job.TypePush || v.Type == job.TypePull {
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
if !ok || activeStatus == nil {
t.Printf("ActiveSideStatus is null")
t.Newline()
return
}
t.Printf("Replication:")
t.AddIndentAndNewline(1)
renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Sender:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Receiver:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
t.AddIndentAndNewline(-1)
if v.Type == job.TypePush {
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
}
} else if v.Type == job.TypeSnap {
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
if !ok || snapStatus == nil {
t.Printf("SnapJobStatus is null")
t.Newline()
return
}
t.Printf("Pruning snapshots:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, snapStatus.Pruning, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
} else if v.Type == job.TypeSource {
st := v.JobSpecific.(*job.PassiveStatus)
t.Printf("Snapshotting:\n")
t.AddIndent(1)
renderSnapperReport(t, st.Snapper, fsfilter)
t.AddIndentAndNewline(-1)
} else {
t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
t.Newline()
asYaml, err := yaml.Marshal(v.JobSpecific)
if err != nil {
t.Printf("Error marshaling status to YAML: %s", err)
t.Newline()
return
}
t.Write(string(asYaml))
t.Newline()
}
}
func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
sizeEstimationImpreciseNotice := ""
if containsInvalidSizeEstimates {
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
}
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
}
userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
// `.CurrentStep` is == len(rep.Steps) if all steps are done.
// Until then, it's an index into .Steps that starts at 0.
// For the user, we want it to start at 1.
if rep.CurrentStep >= len(rep.Steps) {
// rep.CurrentStep is what we want to show.
// We check for >= and not == for robustness.
} else {
// We're not done yet, so, make step count start at 1
// (The `.State` is included in the output, indicating we're not done yet)
userVisisbleCurrentStep = rep.CurrentStep + 1
}
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
strings.ToUpper(string(rep.State)),
userVisisbleCurrentStep, userVisibleTotalSteps,
ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
sizeEstimationImpreciseNotice,
)
activeIndicator := " "
if rep.BlockedOn == report.FsBlockedOnNothing &&
(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
activeIndicator = "*"
}
t.AddIndent(1)
t.Printf("%s %s %s ",
activeIndicator,
stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
status)
next := ""
if err := rep.Error(); err != nil {
next = err.Err
} else if rep.State != report.FilesystemDone {
if nextStep := rep.NextStep(); nextStep != nil {
if nextStep.IsIncremental() {
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
} else {
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
}
attribs := []string{}
if nextStep.Info.Resumed {
attribs = append(attribs, "resumed")
}
if len(attribs) > 0 {
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
}
} else {
next = "" // individual FSes may still be in planning state
}
}
t.Printf("%s", next)
t.AddIndent(-1)
t.Newline()
}
func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
if rep == nil {
t.Printf("...\n")
return
}
if rep.WaitReconnectError != nil {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
t.Newline()
}
if !rep.WaitReconnectSince.IsZero() {
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
var until string
if rep.WaitReconnectUntil.IsZero() {
until = "waiting indefinitely"
} else {
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
}
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
rep.WaitReconnectSince, until)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
}
t.Newline()
}
// TODO visualize more than the latest attempt by folding all attempts into one
if len(rep.Attempts) == 0 {
t.Printf("no attempts made yet")
return
} else {
t.Printf("Attempt #%d", len(rep.Attempts))
if len(rep.Attempts) > 1 {
t.Printf(". Previous attempts failed with the following statuses:")
t.AddIndentAndNewline(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
}
t.AddIndentAndNewline(-1)
} else {
t.Newline()
}
}
latest := rep.Attempts[len(rep.Attempts)-1]
sort.Slice(latest.Filesystems, func(i, j int) bool {
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
})
// apply filter
filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
for _, fs := range latest.Filesystems {
if !fsfilter(fs.Info.Name) {
continue
}
filtered = append(filtered, fs)
}
latest.Filesystems = filtered
t.Printf("Status: %s", latest.State)
t.Newline()
if !latest.FinishAt.IsZero() {
t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
} else {
t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
}
if latest.State == report.AttemptPlanningError {
t.Printf("Problem: ")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
t.Newline()
} else if latest.State == report.AttemptFanOutError {
t.Printf("Problem: one or more of the filesystems encountered errors")
t.Newline()
}
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
// Draw global progress bar
// Progress: [---------------]
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
rate, changeCount := history.Update(replicated)
eta := time.Duration(0)
if rate > 0 {
eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
}
t.Write("Progress: ")
t.DrawBar(50, replicated, expected, changeCount)
t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
if eta != 0 {
t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
}
t.Newline()
if containsInvalidSizeEstimates {
t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
t.Newline()
}
if len(latest.Filesystems) == 0 {
t.Write("NOTE: no filesystems were considered for replication!")
t.Newline()
}
var maxFSLen int
for _, fs := range latest.Filesystems {
if len(fs.Info.Name) > maxFSLen {
maxFSLen = len(fs.Info.Name)
}
}
for _, fs := range latest.Filesystems {
printFilesystemStatus(t, fs, maxFSLen)
}
}
}
func humanizeDuration(duration time.Duration) string {
days := int64(duration.Hours() / 24)
hours := int64(math.Mod(duration.Hours(), 24))
minutes := int64(math.Mod(duration.Minutes(), 60))
seconds := int64(math.Mod(duration.Seconds(), 60))
var parts []string
force := false
chunks := []int64{days, hours, minutes, seconds}
for i, chunk := range chunks {
if force || chunk > 0 {
padding := 0
if force {
padding = 2
}
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
force = true
}
}
return strings.Join(parts, " ")
}
func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("...\n")
return
}
state, err := pruner.StateString(r.State)
if err != nil {
t.Printf("Status: %q (parse error: %q)\n", r.State, err)
return
}
t.Printf("Status: %s", state)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
type commonFS struct {
*pruner.FSReport
completed bool
}
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false})
}
for i := range r.Completed {
all = append(all, commonFS{&r.Completed[i], true})
}
// filter all
filtered := make([]commonFS, 0, len(all))
for _, fs := range all {
if fsfilter(fs.FSReport.Filesystem) {
filtered = append(filtered, fs)
}
}
all = filtered
switch state {
case pruner.Plan:
fallthrough
case pruner.PlanErr:
return
}
if len(all) == 0 {
t.Printf("nothing to do\n")
return
}
var totalDestroyCount, completedDestroyCount int
var maxFSname int
for _, fs := range all {
totalDestroyCount += len(fs.DestroyList)
if fs.completed {
completedDestroyCount += len(fs.DestroyList)
}
if maxFSname < len(fs.Filesystem) {
maxFSname = len(fs.Filesystem)
}
}
// global progress bar
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
t.Write("Progress: ")
t.Write("[")
t.Write(stringbuilder.Times("=", progress))
t.Write(">")
t.Write(stringbuilder.Times("-", 80-progress))
t.Write("]")
t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.Newline()
sort.SliceStable(all, func(i, j int) bool {
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
})
// Draw a table-like representation of 'all'
for _, fs := range all {
t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
t.Write(" ")
if !fs.SkipReason.NotSkipped() {
t.Printf("skipped: %s\n", fs.SkipReason)
continue
}
if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") {
t.Printf("ERROR:")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
}
t.Newline()
continue
}
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed {
t.Printf("Completed %s\n", pruneRuleActionStr)
continue
}
t.Write("Pending ") // whitespace is padding 10
if len(fs.DestroyList) == 1 {
t.Write(fs.DestroyList[0].Name)
} else {
t.Write(pruneRuleActionStr)
}
t.Newline()
}
}
func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("<no snapshotting report available>\n")
return
}
t.Printf("Type: %s\n", r.Type)
if r.Periodic != nil {
renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
} else if r.Cron != nil {
renderSnapperReportCron(t, r.Cron, fsfilter)
} else {
t.Printf("<no details available>")
}
}
func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
t.Printf("Status: %s", r.State)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
if !r.SleepUntil.IsZero() {
t.Printf("Sleep until: %s\n", r.SleepUntil)
}
renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
}
func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
t.Printf("State: %s\n", r.State)
now := time.Now()
if r.WakeupTime.After(now) {
t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
} else {
t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
}
renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
}
func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
sort.Slice(fss, func(i, j int) bool {
return strings.Compare(fss[i].Path, fss[j].Path) == -1
})
dur := func(d time.Duration) string {
return d.Round(100 * time.Millisecond).String()
}
type row struct {
path, state, duration, remainder, hookReport string
}
var widths struct {
path, state, duration int
}
rows := make([]*row, 0, len(fss))
for _, fs := range fss {
if !fsfilter(fs.Path) {
continue
}
r := &row{
path: fs.Path,
state: fs.State.String(),
}
if fs.HooksHadError {
r.hookReport = fs.Hooks // FIXME render here, not in daemon
}
switch fs.State {
case snapper.SnapPending:
r.duration = "..."
r.remainder = ""
case snapper.SnapStarted:
r.duration = dur(time.Since(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
case snapper.SnapDone:
fallthrough
case snapper.SnapError:
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
}
rows = append(rows, r)
if len(r.path) > widths.path {
widths.path = len(r.path)
}
if len(r.state) > widths.state {
widths.state = len(r.state)
}
if len(r.duration) > widths.duration {
widths.duration = len(r.duration)
}
}
for _, r := range rows {
path := stringbuilder.RightPad(r.path, widths.path, " ")
state := stringbuilder.RightPad(r.state, widths.state, " ")
duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
t.Printf("%s %s %s", path, state, duration)
t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
if r.hookReport != "" {
t.AddIndent(1)
t.Newline()
t.Printf("%s", r.hookReport)
t.AddIndent(-1)
}
t.Newline()
}
}
client/status/viewmodel/stringbuilder/stringbuilder.go
+119
View File
@@ -0,0 +1,119 @@
package stringbuilder
import (
"fmt"
"strings"
"github.com/go-playground/validator/v10"
)
type B struct {
// const
indentMultiplier int
// mut
sb *strings.Builder
indent int
width int
x, y int
}
type Config struct {
IndentMultiplier int `validate:"gte=1"`
Width int `validate:"gte=1"`
}
var validate = validator.New()
func New(config Config) *B {
if err := validate.Struct(config); err != nil {
panic(err)
}
return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
}
func (b *B) String() string { return b.sb.String() }
func (w *B) Newline() {
w.Write("\n")
}
func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
if strings.ContainsAny(whole, "\n\r") {
w.AddIndent(1)
defer w.AddIndent(-1)
}
w.Write(whole)
}
func (w *B) Printf(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
w.Write(whole)
}
func (t *B) AddIndent(delta int) {
t.indent += delta * t.indentMultiplier
}
func (t *B) AddIndentAndNewline(delta int) {
t.indent += delta * t.indentMultiplier
t.Write("\n")
}
func (w *B) Write(s string) {
for _, c := range s {
if c == '\n' {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
w.y++
continue
}
if w.x >= w.width {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
}
fmt.Fprintf(w.sb, "%c", c)
w.x++
}
}
func Times(str string, n int) (out string) {
for i := 0; i < n; i++ {
out += str
}
return
}
func RightPad(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
// changeCount = 0 indicates stall / no progress
func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
const arrowPositions = `>\|/`
var completedLength int
if totalBytes > 0 {
completedLength = int(uint64(length) * bytes / totalBytes)
if completedLength > length {
completedLength = length
}
} else if totalBytes == bytes {
completedLength = length
}
w.Write("[")
w.Write(Times("=", completedLength))
w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
w.Write(Times("-", length-completedLength))
w.Write("]")
}
client/testcmd.go
+9 -1
View File
@@ -139,9 +139,11 @@ var testPlaceholder = &cli.Subcommand{
func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
var checkDPs []*zfs.DatasetPath
var datasetWasExplicitArgument bool
// all actions first
if testPlaceholderArgs.all {
datasetWasExplicitArgument = false
out, err := zfs.ZFSList(ctx, []string{"name"})
if err != nil {
return errors.Wrap(err, "could not list ZFS filesystems")
@@ -154,6 +156,7 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
checkDPs = append(checkDPs, dp)
}
} else {
datasetWasExplicitArgument = true
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
if err != nil {
return err
@@ -171,7 +174,12 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
return errors.Wrap(err, "cannot get placeholder state")
}
if !ph.FSExists {
panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
if datasetWasExplicitArgument {
return errors.Errorf("filesystem %q does not exist", ph.FS)
} else {
// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
continue
}
}
is := "yes"
if !ph.IsPlaceholder {
client/zfsabstractions_list.go
+3 -2
View File
@@ -44,17 +44,18 @@ func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
return errors.Wrap(err, "invalid filter specification on command line")
}
abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
if err != nil {
return err // context clear by invocation of command
}
defer drainDone()
var line chainlock.L
var wg sync.WaitGroup
defer wg.Wait()
wg.Add(1)
// print results
wg.Add(1)
go func() {
defer wg.Done()
enc := json.NewEncoder(os.Stdout)
config/config.go
+86 -1
View File
@@ -11,7 +11,18 @@ import (
"time"
"github.com/pkg/errors"
"github.com/robfig/cron/v3"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/util/datasizeunit"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
type ParseFlags uint
const (
ParseFlagsNone ParseFlags = 0
ParseFlagsNoCertCheck ParseFlags = 1 << iota
)
type Config struct {
@@ -58,6 +69,11 @@ type ActiveJob struct {
Pruning PruningSenderReceiver `yaml:"pruning"`
Debug JobDebugSettings `yaml:"debug,optional"`
Replication *Replication `yaml:"replication,optional,fromdefaults"`
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
}
type ConflictResolution struct {
InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
}
type PassiveJob struct {
@@ -78,18 +94,40 @@ type SnapJob struct {
type SendOptions struct {
Encrypted bool `yaml:"encrypted,optional,default=false"`
Raw bool `yaml:"raw,optional,default=false"`
SendProperties bool `yaml:"send_properties,optional,default=false"`
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
Compressed bool `yaml:"compressed,optional,default=false"`
EmbeddedData bool `yaml:"embedded_data,optional,default=false"`
Saved bool `yaml:"saved,optional,default=false"`
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
}
type RecvOptions struct {
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
// Encrypted bool `yaml:"may_encrypted"`
// Future:
// Reencrypt bool `yaml:"reencrypt"`
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
}
var _ yaml.Unmarshaler = &datasizeunit.Bits{}
type BandwidthLimit struct {
Max datasizeunit.Bits `yaml:"max,default=-1 B"`
BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
}
type Replication struct {
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
}
type ReplicationOptionsProtection struct {
@@ -97,6 +135,20 @@ type ReplicationOptionsProtection struct {
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
}
type ReplicationOptionsConcurrency struct {
Steps int `yaml:"steps,optional,default=1"`
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
}
type PropertyRecvOptions struct {
Inherit []zfsprop.Property `yaml:"inherit,optional"`
Override map[zfsprop.Property]string `yaml:"override,optional"`
}
type PlaceholderRecvOptions struct {
Encryption string `yaml:"encryption,default=unspecified"`
}
type PushJob struct {
ActiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
@@ -182,6 +234,38 @@ type SnapshottingPeriodic struct {
Hooks HookList `yaml:"hooks,optional"`
}
type CronSpec struct {
Schedule cron.Schedule
}
var _ yaml.Unmarshaler = &CronSpec{}
func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
var specString string
if err := unmarshal(&specString, false); err != nil {
return err
}
// Use standard cron format.
// Disable the various "descriptors" (@daily, etc)
// They are just aliases to "top of hour", "midnight", etc.
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
sched, err := parser.Parse(specString)
if err != nil {
return errors.Wrap(err, "cron syntax invalid")
}
s.Schedule = sched
return nil
}
type SnapshottingCron struct {
Type string `yaml:"type"`
Prefix string `yaml:"prefix"`
Cron CronSpec `yaml:"cron"`
Hooks HookList `yaml:"hooks,optional"`
}
type SnapshottingManual struct {
Type string `yaml:"type"`
}
@@ -505,6 +589,7 @@ func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err e
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"periodic": &SnapshottingPeriodic{},
"manual": &SnapshottingManual{},
"cron": &SnapshottingCron{},
})
return
}
config/config_global_test.go
+2 -2
View File
@@ -70,9 +70,9 @@ func TestPrometheusMonitoring(t *testing.T) {
global:
monitoring:
- type: prometheus
listen: ':9091'
listen: ':9811'
`)
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
}
func TestSyslogLoggingOutletFacility(t *testing.T) {
config/config_recv_test.go
+134
View File
@@ -0,0 +1,134 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
func TestRecvOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: pull
connect:
type: local
listener_name: foo
client_identity: bar
root_fs: "zreplplatformtest"
%s
interval: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
recv_properties_empty := `
recv:
properties:
`
recv_inherit_empty := `
recv:
properties:
inherit:
`
recv_inherit := `
recv:
properties:
inherit:
- testprop
`
recv_override_empty := `
recv:
properties:
override:
`
recv_override := `
recv:
properties:
override:
testprop2: "test123"
`
recv_override_and_inherit := `
recv:
properties:
inherit:
- testprop
override:
testprop2: "test123"
`
recv_empty := `
recv: {}
`
recv_not_specified := `
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
t.Run("recv_inherit_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit_empty))
assert.NotNil(t, c)
})
t.Run("recv_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
})
t.Run("recv_override_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_empty))
assert.NotNil(t, c)
})
t.Run("recv_override", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override))
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
require.Len(t, override, 1)
require.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_override_and_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_and_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
assert.NotEmpty(t, override)
assert.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_properties_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_properties_empty))
assert.NotNil(t, c)
})
t.Run("recv_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_empty))
assert.NotNil(t, c)
})
t.Run("send_not_specified", func(t *testing.T) {
c := testValidConfig(t, fill(recv_not_specified))
assert.NotNil(t, c)
})
}
config/config_send_test.go
+80 -8
View File
@@ -38,7 +38,41 @@ jobs:
encrypted: true
`
encrypted_unspecified := `
raw_true := `
send:
raw: true
`
raw_false := `
send:
raw: false
`
raw_and_encrypted := `
send:
encrypted: true
raw: true
`
properties_and_encrypted := `
send:
encrypted: true
send_properties: true
`
properties_true := `
send:
send_properties: true
`
properties_false := `
send:
send_properties: false
`
send_empty := `
send: {}
`
@@ -46,28 +80,66 @@ jobs:
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
var c *Config
t.Run("encrypted_false", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_false))
c := testValidConfig(t, fill(encrypted_false))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("encrypted_true", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_true))
c := testValidConfig(t, fill(encrypted_true))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, encrypted)
})
t.Run("encrypted_unspecified", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_unspecified))
t.Run("send_empty", func(t *testing.T) {
c := testValidConfig(t, fill(send_empty))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("properties_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(properties_and_encrypted))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, encrypted)
assert.Equal(t, true, properties)
})
t.Run("properties_false", func(t *testing.T) {
c := testValidConfig(t, fill(properties_false))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, false, properties)
})
t.Run("properties_true", func(t *testing.T) {
c := testValidConfig(t, fill(properties_true))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, properties)
})
t.Run("raw_true", func(t *testing.T) {
c := testValidConfig(t, fill(raw_true))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, true, raw)
})
t.Run("raw_false", func(t *testing.T) {
c := testValidConfig(t, fill(raw_false))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, false, raw)
})
t.Run("raw_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(raw_and_encrypted))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, raw)
assert.Equal(t, true, encrypted)
})
t.Run("send_not_specified", func(t *testing.T) {
c, err := testConfig(t, fill(send_not_specified))
assert.NoError(t, err)
c := testValidConfig(t, fill(send_not_specified))
assert.NotNil(t, c)
})
config/config_test.go
+51
View File
@@ -13,6 +13,7 @@ import (
"github.com/kr/pretty"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
@@ -86,3 +87,53 @@ func TestTrimSpaceEachLineAndPad(t *testing.T) {
`
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
}
func TestCronSpec(t *testing.T) {
expectAccept := []string{
`"* * * * *"`,
`"0-10 * * * *"`,
`"* 0-5,8,12 * * *"`,
}
expectFail := []string{
`* * * *`,
``,
`23`,
`"@reboot"`,
`"@every 1h30m"`,
`"@daily"`,
`* * * * * *`,
}
for _, input := range expectAccept {
t.Run(input, func(t *testing.T) {
s := fmt.Sprintf("spec: %s\n", input)
var v struct {
Spec CronSpec
}
v.Spec.Schedule = nil
t.Logf("input:\n%s", s)
err := yaml.UnmarshalStrict([]byte(s), &v)
t.Logf("error: %T %s", err, err)
require.NoError(t, err)
require.NotNil(t, v.Spec.Schedule)
})
}
for _, input := range expectFail {
t.Run(input, func(t *testing.T) {
s := fmt.Sprintf("spec: %s\n", input)
var v struct {
Spec CronSpec
}
v.Spec.Schedule = nil
t.Logf("input: %q", s)
err := yaml.UnmarshalStrict([]byte(s), &v)
t.Logf("error: %T %s", err, err)
require.Error(t, err)
require.Nil(t, v.Spec.Schedule)
})
}
}
config/samples/bandwidth_limit.yml
+41
View File
@@ -0,0 +1,41 @@
jobs:
- type: sink
name: "limited_sink"
root_fs: "fs0"
recv:
bandwidth_limit:
max: 12345 B
serve:
type: local
listener_name: localsink
- type: push
name: "limited_push"
connect:
type: local
listener_name: localsink
client_identity: local_backup
filesystems: {
"root<": true,
}
send:
bandwidth_limit:
max: 54321 B
bucket_capacity: 1024 B
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 1
keep_receiver:
- type: last_n
count: 1
- type: sink
name: "nolimit_sink"
root_fs: "fs1"
serve:
type: local
listener_name: localsink
config/samples/quickstart_backup_to_external_disk.yml
+1 -1
View File
@@ -5,7 +5,7 @@
# quick start section which inlines this example.
#
# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_derive`
# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
config/samples/quickstart_continuous_server_backup_sender.yml
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
key: /etc/zrepl/prod.key
server_cn: "backups"
filesystems: {
"zroot/var/db": true,
"zroot/usr/home<": true,
"zroot<": true,
"zroot/var/tmp<": false,
"zroot/usr/home/paranoid": false
}
snapshotting:
config/samples/quickstart_fan_out_replication_source.yml
+59
View File
@@ -0,0 +1,59 @@
jobs:
# Separate job for snapshots and pruning
- name: snapshots
type: snap
filesystems:
'tank<': true # all filesystems
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
pruning:
keep:
# Keep non-zrepl snapshots
- type: regex
negate: true
regex: '^zrepl_'
# Time-based snapshot retention
- type: grid
grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
regex: '^zrepl_'
# Source job for target B
- name: target_b
type: source
serve:
type: tls
listen: :8888
ca: /etc/zrepl/b.example.com.crt
cert: /etc/zrepl/a.example.com.crt
key: /etc/zrepl/a.example.com.key
client_cns:
- b.example.com
filesystems:
'tank<': true # all filesystems
# Snapshots are handled by the separate snap job
snapshotting:
type: manual
# Source job for target C
- name: target_c
type: source
serve:
type: tls
listen: :8889
ca: /etc/zrepl/c.example.com.crt
cert: /etc/zrepl/a.example.com.crt
key: /etc/zrepl/a.example.com.key
client_cns:
- c.example.com
filesystems:
'tank<': true # all filesystems
# Snapshots are handled by the separate snap job
snapshotting:
type: manual
# Source jobs for remaining targets. Each one should listen on a different port
# and reference the correct certificate and client CN.
# - name: target_c
# ...
config/samples/quickstart_fan_out_replication_target.yml
+30
View File
@@ -0,0 +1,30 @@
jobs:
# Pull from source server A
- name: source_a
type: pull
connect:
type: tls
# Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
address: a.example.com:8888
ca: /etc/zrepl/a.example.com.crt
# Use the correct key pair for this specific client
cert: /etc/zrepl/b.example.com.crt
key: /etc/zrepl/b.example.com.key
server_cn: a.example.com
root_fs: pool0/backup
interval: 10m
pruning:
keep_sender:
# Source does the pruning in its snap job
- type: regex
regex: '.*'
# Receiver-side pruning can be configured as desired on each target server
keep_receiver:
# Keep non-zrepl snapshots
- type: regex
negate: true
regex: '^zrepl_'
# Time-based snapshot retention
- type: grid
grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
regex: '^zrepl_'
config/samples/snap_cron.yml
+14
View File
@@ -0,0 +1,14 @@
jobs:
- name: snapjob
type: snap
filesystems: {
"tank<": true,
}
snapshotting:
type: cron
prefix: zrepl_snapjob_
cron: "*/5 * * * *"
pruning:
keep:
- type: last_n
count: 60
daemon/control.go
+4 -2
View File
@@ -8,6 +8,7 @@ import (
"io"
"net"
"net/http"
"os"
"time"
"github.com/pkg/errors"
@@ -126,6 +127,7 @@ func (j *controlJob) Run(ctx context.Context) {
Global: GlobalStatus{
ZFSCmds: globalZFS,
Envconst: envconstReport,
OsEnviron: os.Environ(),
}}
return s, nil
}})
@@ -156,8 +158,8 @@ func (j *controlJob) Run(ctx context.Context) {
server := http.Server{
Handler: mux,
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
WriteTimeout: 1 * time.Second,
ReadTimeout: 1 * time.Second,
WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
ReadTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
}
outer:
daemon/daemon.go
+9 -1
View File
@@ -3,6 +3,7 @@ package daemon
import (
"context"
"fmt"
"math/rand"
"os"
"os/signal"
"strings"
@@ -38,13 +39,19 @@ func Run(ctx context.Context, conf *config.Config) error {
cancel()
}()
// The math/rand package is used presently for generating trace IDs, we
// seed it with the current time and pid so that the IDs are mostly
// unique.
rand.Seed(time.Now().UnixNano())
rand.Seed(int64(os.Getpid()))
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
if err != nil {
return errors.Wrap(err, "cannot build logging from config")
}
outlets.Add(newPrometheusLogOutlet(), logger.Debug)
confJobs, err := job.JobsFromConfig(conf)
confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
if err != nil {
return errors.Wrap(err, "cannot build jobs from config")
}
@@ -155,6 +162,7 @@ type Status struct {
type GlobalStatus struct {
ZFSCmds *zfscmd.Report
Envconst *envconst.Report
OsEnviron []string
}
func (s *jobs) status() map[string]*job.Status {
daemon/hooks/hook_type_command.go
+8 -1
View File
@@ -93,7 +93,14 @@ func (r *CommandHookReport) String() string {
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
}
return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
var msg string
if r.Err == nil {
msg = "command hook"
} else {
msg = fmt.Sprintf("command hook failed with %q", r.Err)
}
return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
}
func (r *CommandHookReport) Error() string {
if r.Err == nil {
daemon/hooks/hooks_test.go
+5 -5
View File
@@ -161,7 +161,7 @@ jobs:
ExpectedEdge: hooks.Pre,
ExpectStatus: hooks.StepErr,
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
},
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
expectStep{
@@ -185,7 +185,7 @@ jobs:
ExpectedEdge: hooks.Pre,
ExpectStatus: hooks.StepErr,
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
},
expectStep{
ExpectedEdge: hooks.Pre,
@@ -234,7 +234,7 @@ jobs:
ExpectedEdge: hooks.Post,
ExpectStatus: hooks.StepErr,
OutputTest: containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
},
},
},
@@ -267,7 +267,7 @@ jobs:
ExpectedEdge: hooks.Pre,
ExpectStatus: hooks.StepErr,
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
},
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
expectStep{
@@ -295,7 +295,7 @@ jobs:
ExpectedEdge: hooks.Pre,
ExpectStatus: hooks.StepErr,
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
},
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
expectStep{
daemon/job/active.go
+60 -11
View File
@@ -11,6 +11,7 @@ import (
"github.com/prometheus/common/log"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job/reset"
@@ -33,12 +34,15 @@ type ActiveSide struct {
name endpoint.JobID
connecter transport.Connecter
replicationDriverConfig driver.Config
prunerFactory *pruner.PrunerFactory
promRepStateSecs *prometheus.HistogramVec // labels: state
promPruneSecs *prometheus.HistogramVec // labels: prune_side
promBytesReplicated *prometheus.CounterVec // labels: filesystem
promReplicationErrors prometheus.Gauge
promLastSuccessful prometheus.Gauge
tasksMtx sync.Mutex
tasks activeSideTasks
@@ -97,7 +101,7 @@ type modePush struct {
receiver *rpc.Client
senderConfig *endpoint.SenderConfig
plannerPolicy *logic.PlannerPolicy
snapper *snapper.PeriodicOrManual
snapper snapper.Snapper
}
func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
@@ -133,7 +137,8 @@ func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
}
func (m *modePush) SnapperReport() *snapper.Report {
return m.snapper.Report()
r := m.snapper.Report()
return &r
}
func (m *modePush) ResetConnectBackoff() {
@@ -158,9 +163,18 @@ func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.Job
return nil, errors.Wrap(err, "field `replication`")
}
conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
if err != nil {
return nil, errors.Wrap(err, "field `conflict_resolution`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
ReplicationConfig: *replicationConfig,
ConflictResolution: conflictResolution,
ReplicationConfig: replicationConfig,
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
}
if err := m.plannerPolicy.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build planner policy")
}
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
@@ -253,9 +267,18 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
return nil, errors.Wrap(err, "field `replication`")
}
conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
if err != nil {
return nil, errors.Wrap(err, "field `conflict_resolution`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.DontCare,
ReplicationConfig: *replicationConfig,
ConflictResolution: conflictResolution,
ReplicationConfig: replicationConfig,
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
}
if err := m.plannerPolicy.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build planner policy")
}
m.receiverConfig, err = buildReceiverConfig(in, jobID)
@@ -266,7 +289,17 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
return m, nil
}
func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
func replicationDriverConfigFromConfig(in *config.Replication) (c driver.Config, err error) {
c = driver.Config{
StepQueueConcurrency: in.Concurrency.Steps,
MaxAttempts: envconst.Int("ZREPL_REPLICATION_MAX_ATTEMPTS", 3),
ReconnectHardFailTimeout: envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute),
}
err = c.Validate()
return c, err
}
func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}, parseFlags config.ParseFlags) (j *ActiveSide, err error) {
j = &ActiveSide{}
j.name, err = endpoint.MakeJobID(in.Name)
@@ -300,7 +333,6 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
Help: "number of bytes replicated from sender to receiver per filesystem",
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
}, []string{"filesystem"})
j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "zrepl",
Subsystem: "replication",
@@ -308,8 +340,15 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
Help: "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
})
j.promLastSuccessful = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "zrepl",
Subsystem: "replication",
Name: "last_successful",
Help: "timestamp of last successful replication",
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
})
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect, parseFlags)
if err != nil {
return nil, errors.Wrap(err, "cannot build client")
}
@@ -326,6 +365,11 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
return nil, err
}
j.replicationDriverConfig, err = replicationDriverConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.Wrap(err, "cannot build replication driver config")
}
return j, nil
}
@@ -334,6 +378,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
registerer.MustRegister(j.promPruneSecs)
registerer.MustRegister(j.promBytesReplicated)
registerer.MustRegister(j.promReplicationErrors)
registerer.MustRegister(j.promLastSuccessful)
}
func (j *ActiveSide) Name() string { return j.name.String() }
@@ -459,7 +504,7 @@ func (j *ActiveSide) do(ctx context.Context) {
*tasks = activeSideTasks{}
tasks.replicationCancel = func() { repCancel(); endSpan() }
tasks.replicationReport, repWait = replication.Do(
ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
)
tasks.state = ActiveSideReplicating
})
@@ -468,7 +513,11 @@ func (j *ActiveSide) do(ctx context.Context) {
repCancel() // always cancel to free up context resources
replicationReport := j.tasks.replicationReport()
j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
var numErrors = replicationReport.GetFailedFilesystemsCountInLatestAttempt()
j.promReplicationErrors.Set(float64(numErrors))
if numErrors == 0 {
j.promLastSuccessful.SetToCurrentTime()
}
endSpan()
}
daemon/job/build_jobs.go
+18 -7
View File
@@ -8,12 +8,13 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/util/bandwidthlimit"
)
func JobsFromConfig(c *config.Config) ([]Job, error) {
func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
js := make([]Job, len(c.Jobs))
for i := range c.Jobs {
j, err := buildJob(c.Global, c.Jobs[i])
j, err := buildJob(c.Global, c.Jobs[i], parseFlags)
if err != nil {
return nil, err
}
@@ -41,19 +42,19 @@ func JobsFromConfig(c *config.Config) ([]Job, error) {
return js, nil
}
func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags) (j Job, err error) {
cannotBuildJob := func(e error, name string) (Job, error) {
return nil, errors.Wrapf(e, "cannot build job %q", name)
}
// FIXME prettify this
switch v := in.Ret.(type) {
case *config.SinkJob:
j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
if err != nil {
return cannotBuildJob(err, v.Name)
}
case *config.SourceJob:
j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
if err != nil {
return cannotBuildJob(err, v.Name)
}
@@ -63,12 +64,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
return cannotBuildJob(err, v.Name)
}
case *config.PushJob:
j, err = activeSide(c, &v.ActiveJob, v)
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
if err != nil {
return cannotBuildJob(err, v.Name)
}
case *config.PullJob:
j, err = activeSide(c, &v.ActiveJob, v)
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
if err != nil {
return cannotBuildJob(err, v.Name)
}
@@ -107,3 +108,13 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
}
return nil
}
func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
}
return bandwidthlimit.Config{
Max: int64(in.Max.ToBytes()),
BucketCapacity: int64(in.BucketCapacity.ToBytes()),
}, nil
}
daemon/job/build_jobs_sendrecvoptions.go
+49 -3
View File
@@ -6,6 +6,7 @@ import (
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -20,12 +21,33 @@ func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.Sen
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
sendOpts := in.GetSendOptions()
bwlim, err := buildBandwidthLimitConfig(sendOpts.BandwidthLimit)
if err != nil {
return nil, errors.Wrap(err, "cannot build bandwith limit config")
}
return &endpoint.SenderConfig{
sc := &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.GetSendOptions().Encrypted},
JobID: jobID,
}, nil
Encrypt: &nodefault.Bool{B: sendOpts.Encrypted},
SendRaw: sendOpts.Raw,
SendProperties: sendOpts.SendProperties,
SendBackupProperties: sendOpts.BackupProperties,
SendLargeBlocks: sendOpts.LargeBlocks,
SendCompressed: sendOpts.Compressed,
SendEmbeddedData: sendOpts.EmbeddedData,
SendSaved: sendOpts.Saved,
BandwidthLimit: bwlim,
}
if err := sc.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build sender config")
}
return sc, nil
}
type ReceivingJobConfig interface {
@@ -43,10 +65,34 @@ func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoi
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
}
recvOpts := in.GetRecvOptions()
bwlim, err := buildBandwidthLimitConfig(recvOpts.BandwidthLimit)
if err != nil {
return rc, errors.Wrap(err, "cannot build bandwith limit config")
}
placeholderEncryption, err := endpoint.PlaceholderCreationEncryptionPropertyString(recvOpts.Placeholder.Encryption)
if err != nil {
options := []string{}
for _, v := range endpoint.PlaceholderCreationEncryptionPropertyValues() {
options = append(options, endpoint.PlaceholderCreationEncryptionProperty(v).String())
}
return rc, errors.Errorf("placeholder encryption value %q is invalid, must be one of %s",
recvOpts.Placeholder.Encryption, options)
}
rc = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootFs,
AppendClientIdentity: in.GetAppendClientIdentity(),
InheritProperties: recvOpts.Properties.Inherit,
OverrideProperties: recvOpts.Properties.Override,
BandwidthLimit: bwlim,
PlaceholderEncryption: placeholderEncryption,
}
if err := rc.Validate(); err != nil {
return rc, errors.Wrap(err, "cannot build receiver config")
daemon/job/build_jobs_test.go
+178 -6
View File
@@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport/tls"
)
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
@@ -96,7 +95,7 @@ jobs:
conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
require.NoError(t, err, "not expecting yaml-config to know about job ids")
require.NotNil(t, conf)
jobs, err := JobsFromConfig(conf)
jobs, err := JobsFromConfig(conf, config.ParseFlagsNone)
if c.valid {
assert.NoError(t, err)
@@ -119,6 +118,14 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
t.Errorf("glob failed: %+v", err)
}
type additionalCheck struct {
state int
test func(t *testing.T, jobs []Job)
}
additionalChecks := map[string]*additionalCheck{
"bandwidth_limit.yml": {test: testSampleConfig_BandwidthLimit},
}
for _, p := range paths {
if path.Ext(p) != ".yml" {
@@ -126,21 +133,186 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
continue
}
filename := path.Base(p)
t.Logf("checking for presence additonal checks for file %q", filename)
additionalCheck := additionalChecks[filename]
if additionalCheck == nil {
t.Logf("no additional checks")
} else {
t.Logf("additional check present")
additionalCheck.state = 1
}
t.Run(p, func(t *testing.T) {
c, err := config.ParseConfig(p)
if err != nil {
t.Errorf("error parsing %s:\n%+v", p, err)
t.Fatalf("error parsing %s:\n%+v", p, err)
}
t.Logf("file: %s", p)
t.Log(pretty.Sprint(c))
tls.FakeCertificateLoading(t)
jobs, err := JobsFromConfig(c)
jobs, err := JobsFromConfig(c, config.ParseFlagsNoCertCheck)
t.Logf("jobs: %#v", jobs)
assert.NoError(t, err)
require.NoError(t, err)
if additionalCheck != nil {
additionalCheck.test(t, jobs)
additionalCheck.state = 2
}
})
}
for basename, c := range additionalChecks {
if c.state == 0 {
panic("univisited additional check " + basename)
}
}
}
func testSampleConfig_BandwidthLimit(t *testing.T, jobs []Job) {
require.Len(t, jobs, 3)
{
limitedSink, ok := jobs[0].(*PassiveSide)
require.True(t, ok, "%T", jobs[0])
limitedSinkMode, ok := limitedSink.mode.(*modeSink)
require.True(t, ok, "%T", limitedSink)
assert.Equal(t, int64(12345), limitedSinkMode.receiverConfig.BandwidthLimit.Max)
assert.Equal(t, int64(1<<17), limitedSinkMode.receiverConfig.BandwidthLimit.BucketCapacity)
}
{
limitedPush, ok := jobs[1].(*ActiveSide)
require.True(t, ok, "%T", jobs[1])
limitedPushMode, ok := limitedPush.mode.(*modePush)
require.True(t, ok, "%T", limitedPush)
assert.Equal(t, int64(54321), limitedPushMode.senderConfig.BandwidthLimit.Max)
assert.Equal(t, int64(1024), limitedPushMode.senderConfig.BandwidthLimit.BucketCapacity)
}
{
unlimitedSink, ok := jobs[2].(*PassiveSide)
require.True(t, ok, "%T", jobs[2])
unlimitedSinkMode, ok := unlimitedSink.mode.(*modeSink)
require.True(t, ok, "%T", unlimitedSink)
max := unlimitedSinkMode.receiverConfig.BandwidthLimit.Max
assert.Less(t, max, int64(0), max, "unlimited mode <=> negative value for .Max, see bandwidthlimit.Config")
}
}
func TestReplicationOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
type: local
listener_name: foo
client_identity: bar
filesystems: {"<": true}
%s
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
type Test struct {
name string
input string
expectOk func(t *testing.T, a *ActiveSide, m *modePush)
expectError bool
}
tests := []Test{
{
name: "defaults",
input: `
replication: {}
`,
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {},
},
{
name: "steps_zero",
input: `
replication:
concurrency:
steps: 0
`,
expectError: true,
},
{
name: "size_estimates_zero",
input: `
replication:
concurrency:
size_estimates: 0
`,
expectError: true,
},
{
name: "custom_values",
input: `
replication:
concurrency:
steps: 23
size_estimates: 42
`,
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {
assert.Equal(t, 23, a.replicationDriverConfig.StepQueueConcurrency)
assert.Equal(t, 42, m.plannerPolicy.SizeEstimationConcurrency)
},
},
{
name: "negative_values_forbidden",
input: `
replication:
concurrency:
steps: -23
size_estimates: -42
`,
expectError: true,
},
}
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
for _, ts := range tests {
t.Run(ts.name, func(t *testing.T) {
assert.True(t, (ts.expectError) != (ts.expectOk != nil))
cstr := fill(ts.input)
t.Logf("testing config:\n%s", cstr)
c, err := config.ParseConfigBytes([]byte(cstr))
require.NoError(t, err)
jobs, err := JobsFromConfig(c, config.ParseFlagsNone)
if ts.expectOk != nil {
require.NoError(t, err)
require.NotNil(t, c)
require.NoError(t, err)
require.Len(t, jobs, 1)
a := jobs[0].(*ActiveSide)
m := a.mode.(*modePush)
ts.expectOk(t, a, m)
} else if ts.expectError {
require.Error(t, err)
} else {
t.Fatalf("test must define expectOk or expectError")
}
})
}
}
daemon/job/passive.go
+5 -4
View File
@@ -58,7 +58,7 @@ func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.Job
type modeSource struct {
senderConfig *endpoint.SenderConfig
snapper *snapper.PeriodicOrManual
snapper snapper.Snapper
}
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
@@ -88,10 +88,11 @@ func (m *modeSource) RunPeriodic(ctx context.Context) {
}
func (m *modeSource) SnapperReport() *snapper.Report {
return m.snapper.Report()
r := m.snapper.Report()
return &r
}
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}, parseFlags config.ParseFlags) (s *PassiveSide, err error) {
s = &PassiveSide{}
@@ -110,7 +111,7 @@ func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob in
return nil, err // no wrapping necessary
}
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve, parseFlags); err != nil {
return nil, errors.Wrap(err, "cannot build listener factory")
}
daemon/job/snapjob.go
+17 -5
View File
@@ -4,11 +4,14 @@ import (
"context"
"fmt"
"sort"
"sync"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/bandwidthlimit"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
@@ -23,12 +26,13 @@ import (
type SnapJob struct {
name endpoint.JobID
fsfilter zfs.DatasetFilter
snapper *snapper.PeriodicOrManual
snapper snapper.Snapper
prunerFactory *pruner.LocalPrunerFactory
promPruneSecs *prometheus.HistogramVec // labels: prune_side
prunerMtx sync.Mutex
pruner *pruner.Pruner
}
@@ -77,10 +81,13 @@ type SnapJobStatus struct {
func (j *SnapJob) Status() *Status {
s := &SnapJobStatus{}
t := j.Type()
j.prunerMtx.Lock()
if j.pruner != nil {
s.Pruning = j.pruner.Report()
}
s.Snapshotting = j.snapper.Report()
j.prunerMtx.Unlock()
r := j.snapper.Report()
s.Snapshotting = &r
return &Status{Type: t, JobSpecific: s}
}
@@ -141,7 +148,7 @@ type alwaysUpToDateReplicationCursorHistory struct {
target pruner.Target
}
var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
fsvReq := &pdu.ListFilesystemVersionsReq{
@@ -174,10 +181,15 @@ func (j *SnapJob) doPrune(ctx context.Context) {
sender := endpoint.NewSender(endpoint.SenderConfig{
JobID: j.name,
FSF: j.fsfilter,
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
Encrypt: &zfs.NilBool{B: true},
// FIXME the following config fields are irrelevant for SnapJob
// because the endpoint is only used as pruner.Target.
// However, the implementation requires them to be set.
Encrypt: &nodefault.Bool{B: true},
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
})
j.prunerMtx.Lock()
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
j.prunerMtx.Unlock()
log.Info("start pruning")
j.pruner.Prune()
log.Info("finished pruning")
daemon/logging/trace/trace.go
+1 -1
View File
@@ -246,7 +246,7 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
// the debugString can be quite long and panic won't print it completely
fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks:\n%s\n", this.debugString())
}
panic(errors.WithMessagef(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks\n", this.activeChildTasks))
panic(errors.WithMessagef(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks (run daemon with env var %s=1 for more details)\n", this.activeChildTasks, debugEnabledEnvVar))
}
// support idempotent task ends
daemon/logging/trace/trace_convenience.go
+4
View File
@@ -43,15 +43,19 @@ func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Con
}
// create a span during which several child tasks are spawned using the `add` function
//
// IMPORTANT FOR USERS: Caller must ensure that the capturing behavior is correct, the Go linter doesn't catch this.
func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
var wg sync.WaitGroup
ctx, endSpan := WithSpan(ctx, taskGroup)
add = func(f func(context.Context)) {
wg.Add(1)
go func() {
defer wg.Done()
ctx, endTask := WithTask(ctx, taskGroup)
defer endTask()
f(ctx)
}()
}
waitEnd = func() {
wg.Wait()
daemon/logging/trace/trace_convenience_test.go
+46
View File
@@ -1,7 +1,10 @@
package trace
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
@@ -14,3 +17,46 @@ func TestGetCallerOrPanic(t *testing.T) {
// zrepl prefix is stripped
assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
}
func TestWithTaskGroupRunTasksConcurrently(t *testing.T) {
// spawn a task group where each task waits for the other to start
// => without concurrency, they would hang
rootCtx, endRoot := WithTaskFromStack(context.Background())
defer endRoot()
_, add, waitEnd := WithTaskGroup(rootCtx, "test-task-group")
schedulerTimeout := 2 * time.Second
timeout := time.After(schedulerTimeout)
var hadTimeout uint32
started0, started1 := make(chan struct{}), make(chan struct{})
for i := 0; i < 2; i++ {
i := i // capture by copy
add(func(ctx context.Context) {
switch i {
case 0:
close(started0)
select {
case <-started1:
case <-timeout:
atomic.AddUint32(&hadTimeout, 1)
}
case 1:
close(started1)
select {
case <-started0:
case <-timeout:
atomic.AddUint32(&hadTimeout, 1)
}
default:
panic("unreachable")
}
})
}
waitEnd()
assert.Zero(t, hadTimeout, "either bad impl or scheduler timeout (which is %v)", schedulerTimeout)
}
daemon/logging/trace/trace_debug.go
+3 -1
View File
@@ -7,7 +7,9 @@ import (
"github.com/zrepl/zrepl/util/envconst"
)
var debugEnabled = envconst.Bool("ZREPL_TRACE_DEBUG_ENABLED", false)
const debugEnabledEnvVar = "ZREPL_TRACE_DEBUG_ENABLED"
var debugEnabled = envconst.Bool(debugEnabledEnvVar, false)
func debug(format string, args ...interface{}) {
if !debugEnabled {
daemon/logging/trace/trace_genID.go
+1 -10
View File
@@ -3,20 +3,11 @@ package trace
import (
"encoding/base64"
"math/rand"
"os"
"strings"
"time"
"github.com/zrepl/zrepl/util/envconst"
)
var genIdPRNG = rand.New(rand.NewSource(1))
func init() {
genIdPRNG.Seed(time.Now().UnixNano())
genIdPRNG.Seed(int64(os.Getpid()))
}
var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
func init() {
@@ -30,7 +21,7 @@ func genID() string {
enc := base64.NewEncoder(base64.RawStdEncoding, &out)
buf := make([]byte, genIdNumBytes)
for i := 0; i < len(buf); {
n, err := genIdPRNG.Read(buf[i:])
n, err := rand.Read(buf[i:])
if err != nil {
panic(err)
}
daemon/pruner/pruner.go
+20 -12
View File
@@ -19,12 +19,20 @@ import (
"github.com/zrepl/zrepl/util/envconst"
)
// The sender in the replication setup.
// The pruner uses the Sender to determine which of the Target's filesystems need to be pruned.
// Also, it asks the Sender about the replication cursor of each filesystem
// to enable the 'not_replicated' pruning rule.
//
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
type History interface {
type Sender interface {
ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
}
// The pruning target, i.e., on which snapshots are destroyed.
// This can be a replication sender or receiver.
//
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
type Target interface {
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
@@ -48,7 +56,7 @@ func GetLogger(ctx context.Context) Logger {
type args struct {
ctx context.Context
target Target
receiver History
sender Sender
rules []pruning.KeepRule
retryWait time.Duration
considerSnapAtCursorReplicated bool
@@ -132,12 +140,12 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
return f, nil
}
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, sender Sender) *Pruner {
p := &Pruner{
args: args{
context.WithValue(ctx, contextKeyPruneSide, "sender"),
target,
receiver,
sender,
f.senderRules,
f.retryWait,
f.considerSnapAtCursorReplicated,
@@ -148,12 +156,12 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, re
return p
}
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, sender Sender) *Pruner {
p := &Pruner{
args: args{
context.WithValue(ctx, contextKeyPruneSide, "receiver"),
target,
receiver,
sender,
f.receiverRules,
f.retryWait,
false, // senseless here anyways
@@ -164,12 +172,12 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
return p
}
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, history Sender) *Pruner {
p := &Pruner{
args: args{
context.WithValue(ctx, contextKeyPruneSide, "local"),
target,
receiver,
history,
f.keepRules,
f.retryWait,
false, // considerSnapAtCursorReplicated is not relevant for local pruning
@@ -343,9 +351,9 @@ func (s snapshot) Date() time.Time { return s.date }
func doOneAttempt(a *args, u updater) {
ctx, target, receiver := a.ctx, a.target, a.receiver
ctx, target, sender := a.ctx, a.target, a.sender
sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
sfssres, err := sender.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
if err != nil {
u(func(p *Pruner) {
p.state = PlanErr
@@ -410,7 +418,7 @@ tfss_loop:
rcReq := &pdu.ReplicationCursorReq{
Filesystem: tfs.Path,
}
rc, err := receiver.ReplicationCursor(ctx, rcReq)
rc, err := sender.ReplicationCursor(ctx, rcReq)
if err != nil {
pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
continue tfss_loop
@@ -456,7 +464,7 @@ tfss_loop:
})
}
if preCursor {
pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
pfsPlanErrAndLog(fmt.Errorf("prune target has no snapshot that corresponds to sender replication cursor bookmark"), "")
continue tfss_loop
}
daemon/snapper/cron.go
+151
View File
@@ -0,0 +1,151 @@
package snapper
import (
"context"
"fmt"
"sync"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/util/suspendresumesafetimer"
"github.com/zrepl/zrepl/zfs"
)
func cronFromConfig(fsf zfs.DatasetFilter, in config.SnapshottingCron) (*Cron, error) {
hooksList, err := hooks.ListFromConfig(&in.Hooks)
if err != nil {
return nil, errors.Wrap(err, "hook config error")
}
planArgs := planArgs{
prefix: in.Prefix,
hooks: hooksList,
}
return &Cron{config: in, fsf: fsf, planArgs: planArgs}, nil
}
type Cron struct {
config config.SnapshottingCron
fsf zfs.DatasetFilter
planArgs planArgs
mtx sync.RWMutex
running bool
wakeupTime time.Time // zero value means uninit
lastError error
lastPlan *plan
wakeupWhileRunningCount int
}
func (s *Cron) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
for {
now := time.Now()
s.mtx.Lock()
s.wakeupTime = s.config.Cron.Schedule.Next(now)
s.mtx.Unlock()
ctxDone := suspendresumesafetimer.SleepUntil(ctx, s.wakeupTime)
if ctxDone != nil {
return
}
getLogger(ctx).Debug("cron timer fired")
s.mtx.Lock()
if s.running {
getLogger(ctx).Warn("snapshotting triggered according to cron rules but previous snapshotting is not done; not taking a snapshot this time")
s.wakeupWhileRunningCount++
s.mtx.Unlock()
continue
}
s.lastError = nil
s.lastPlan = nil
s.wakeupWhileRunningCount = 0
s.running = true
s.mtx.Unlock()
go func() {
err := s.do(ctx)
s.mtx.Lock()
s.lastError = err
s.running = false
s.mtx.Unlock()
select {
case snapshotsTaken <- struct{}{}:
default:
if snapshotsTaken != nil {
getLogger(ctx).Warn("callback channel is full, discarding snapshot update event")
}
}
}()
}
}
func (s *Cron) do(ctx context.Context) error {
fss, err := zfs.ZFSListMapping(ctx, s.fsf)
if err != nil {
return errors.Wrap(err, "cannot list filesystems")
}
p := makePlan(s.planArgs, fss)
s.mtx.Lock()
s.lastPlan = p
s.lastError = nil
s.mtx.Unlock()
ok := p.execute(ctx, false)
if !ok {
return errors.New("one or more snapshots could not be created, check logs for details")
} else {
return nil
}
}
type CronState string
const (
CronStateRunning CronState = "running"
CronStateWaiting CronState = "waiting"
)
type CronReport struct {
State CronState
WakeupTime time.Time
Errors []string
Progress []*ReportFilesystem
}
func (s *Cron) Report() Report {
s.mtx.Lock()
defer s.mtx.Unlock()
r := CronReport{}
r.WakeupTime = s.wakeupTime
if s.running {
r.State = CronStateRunning
} else {
r.State = CronStateWaiting
}
if s.lastError != nil {
r.Errors = append(r.Errors, s.lastError.Error())
}
if s.wakeupWhileRunningCount > 0 {
r.Errors = append(r.Errors, fmt.Sprintf("cron frequency is too high; snapshots were not taken %d times", s.wakeupWhileRunningCount))
}
r.Progress = nil
if s.lastPlan != nil {
r.Progress = s.lastPlan.report()
}
return Report{Type: TypeCron, Cron: &r}
}
daemon/snapper/cron_test.go
+68
View File
@@ -0,0 +1,68 @@
package snapper
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/config"
)
func TestCronLibraryWorks(t *testing.T) {
type testCase struct {
spec string
in time.Time
expect time.Time
}
dhm := func(day, hour, minutes int) time.Time {
return time.Date(2022, 7, day, hour, minutes, 0, 0, time.UTC)
}
hm := func(hour, minutes int) time.Time {
return dhm(23, hour, minutes)
}
tcs := []testCase{
{"0-10 * * * *", dhm(17, 1, 10), dhm(17, 2, 0)},
{"0-10 * * * *", dhm(17, 23, 10), dhm(18, 0, 0)},
{"0-10 * * * *", hm(1, 9), hm(1, 10)},
{"0-10 * * * *", hm(1, 9), hm(1, 10)},
{"1,3,5 * * * *", hm(1, 1), hm(1, 3)},
{"1,3,5 * * * *", hm(1, 2), hm(1, 3)},
{"1,3,5 * * * *", hm(1, 3), hm(1, 5)},
{"1,3,5 * * * *", hm(1, 5), hm(2, 1)},
{"* 0-5,8,12 * * *", hm(0, 0), hm(0, 1)},
{"* 0-5,8,12 * * *", hm(4, 59), hm(5, 0)},
{"* 0-5,8,12 * * *", hm(5, 0), hm(5, 1)},
{"* 0-5,8,12 * * *", hm(5, 59), hm(8, 0)},
{"* 0-5,8,12 * * *", hm(8, 59), hm(12, 0)},
// https://github.com/zrepl/zrepl/pull/614#issuecomment-1188358989
{"53 17,18,19 * * *", dhm(23, 17, 52), dhm(23, 17, 53)},
{"53 17,18,19 * * *", dhm(23, 17, 53), dhm(23, 18, 53)},
{"53 17,18,19 * * *", dhm(23, 18, 53), dhm(23, 19, 53)},
{"53 17,18,19 * * *", dhm(23, 19, 53), dhm(24 /* ! */, 17, 53)},
}
for i, tc := range tcs {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var s struct {
Cron config.CronSpec `yaml:"cron"`
}
inp := fmt.Sprintf("cron: %q", tc.spec)
fmt.Println("spec is ", inp)
err := yaml.UnmarshalStrict([]byte(inp), &s)
require.NoError(t, err)
actual := s.Cron.Schedule.Next(tc.in)
assert.Equal(t, tc.expect, actual)
})
}
}
daemon/snapper/impl.go
+251
View File
@@ -0,0 +1,251 @@
package snapper
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/zfs"
)
type planArgs struct {
prefix string
hooks *hooks.List
}
type plan struct {
mtx chainlock.L
args planArgs
snaps map[*zfs.DatasetPath]*snapProgress
}
func makePlan(args planArgs, fss []*zfs.DatasetPath) *plan {
snaps := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
for _, fs := range fss {
snaps[fs] = &snapProgress{state: SnapPending}
}
return &plan{snaps: snaps, args: args}
}
//go:generate stringer -type=SnapState
type SnapState uint
const (
SnapPending SnapState = 1 << iota
SnapStarted
SnapDone
SnapError
)
// All fields protected by Snapper.mtx
type snapProgress struct {
state SnapState
// SnapStarted, SnapDone, SnapError
name string
startAt time.Time
hookPlan *hooks.Plan
// SnapDone
doneAt time.Time
// SnapErr TODO disambiguate state
runResults hooks.PlanReport
}
func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
hookMatchCount := make(map[hooks.Hook]int, len(*plan.args.hooks))
for _, h := range *plan.args.hooks {
hookMatchCount[h] = 0
}
anyFsHadErr := false
// TODO channel programs -> allow a little jitter?
for fs, progress := range plan.snaps {
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
snapname := fmt.Sprintf("%s%s", plan.args.prefix, suffix)
ctx := logging.WithInjectedField(ctx, "fs", fs.ToString())
ctx = logging.WithInjectedField(ctx, "snap", snapname)
hookEnvExtra := hooks.Env{
hooks.EnvFS: fs.ToString(),
hooks.EnvSnapshot: snapname,
}
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
l := getLogger(ctx)
l.Debug("create snapshot")
err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
if err != nil {
l.WithError(err).Error("cannot create snapshot")
}
return
})
fsHadErr := false
var hookPlanReport hooks.PlanReport
var hookPlan *hooks.Plan
{
filteredHooks, err := plan.args.hooks.CopyFilteredForFilesystem(fs)
if err != nil {
getLogger(ctx).WithError(err).Error("unexpected filter error")
fsHadErr = true
goto updateFSState
}
// account for running hooks
for _, h := range filteredHooks {
hookMatchCount[h] = hookMatchCount[h] + 1
}
var planErr error
hookPlan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
if planErr != nil {
fsHadErr = true
getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
goto updateFSState
}
}
plan.mtx.HoldWhile(func() {
progress.name = snapname
progress.startAt = time.Now()
progress.hookPlan = hookPlan
progress.state = SnapStarted
})
{
getLogger(ctx).WithField("report", hookPlan.Report().String()).Debug("begin run job plan")
hookPlan.Run(ctx, dryRun)
hookPlanReport = hookPlan.Report()
fsHadErr = hookPlanReport.HadError() // not just fatal errors
if fsHadErr {
getLogger(ctx).WithField("report", hookPlanReport.String()).Error("end run job plan with error")
} else {
getLogger(ctx).WithField("report", hookPlanReport.String()).Info("end run job plan successful")
}
}
updateFSState:
anyFsHadErr = anyFsHadErr || fsHadErr
plan.mtx.HoldWhile(func() {
progress.doneAt = time.Now()
progress.state = SnapDone
if fsHadErr {
progress.state = SnapError
}
progress.runResults = hookPlanReport
})
}
for h, mc := range hookMatchCount {
if mc == 0 {
hookIdx := -1
for idx, ah := range *plan.args.hooks {
if ah == h {
hookIdx = idx
break
}
}
getLogger(ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
}
}
return !anyFsHadErr
}
type ReportFilesystem struct {
Path string
State SnapState
// Valid in SnapStarted and later
SnapName string
StartAt time.Time
Hooks string
HooksHadError bool
// Valid in SnapDone | SnapError
DoneAt time.Time
}
func (plan *plan) report() []*ReportFilesystem {
plan.mtx.Lock()
defer plan.mtx.Unlock()
pReps := make([]*ReportFilesystem, 0, len(plan.snaps))
for fs, p := range plan.snaps {
var hooksStr string
var hooksHadError bool
if p.hookPlan != nil {
hooksStr, hooksHadError = p.report()
}
pReps = append(pReps, &ReportFilesystem{
Path: fs.ToString(),
State: p.state,
SnapName: p.name,
StartAt: p.startAt,
DoneAt: p.doneAt,
Hooks: hooksStr,
HooksHadError: hooksHadError,
})
}
sort.Slice(pReps, func(i, j int) bool {
return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
})
return pReps
}
func (p *snapProgress) report() (hooksStr string, hooksHadError bool) {
hr := p.hookPlan.Report()
// FIXME: technically this belongs into client
// but we can't serialize hooks.Step ATM
rightPad := func(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
hooksHadError = hr.HadError()
rows := make([][]string, len(hr))
const numCols = 4
lens := make([]int, numCols)
for i, e := range hr {
rows[i] = make([]string, numCols)
rows[i][0] = fmt.Sprintf("%d", i+1)
rows[i][1] = e.Status.String()
runTime := "..."
if e.Status != hooks.StepPending {
runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
}
rows[i][2] = runTime
rows[i][3] = ""
if e.Report != nil {
rows[i][3] = e.Report.String()
}
for j, col := range lens {
if len(rows[i][j]) > col {
lens[j] = len(rows[i][j])
}
}
}
rowsFlat := make([]string, len(hr))
for i, r := range rows {
colsPadded := make([]string, len(r))
for j, c := range r[:len(r)-1] {
colsPadded[j] = rightPad(c, lens[j], " ")
}
colsPadded[len(r)-1] = r[len(r)-1]
rowsFlat[i] = strings.Join(colsPadded, " ")
}
hooksStr = strings.Join(rowsFlat, "\n")
return hooksStr, hooksHadError
}
daemon/snapper/manual.go
+15
View File
@@ -0,0 +1,15 @@
package snapper
import (
"context"
)
type manual struct{}
func (s *manual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
// nothing to do
}
func (s *manual) Report() Report {
return Report{Type: TypeManual, Manual: &struct{}{}}
}
daemon/snapper/periodic.go
+393
View File
@@ -0,0 +1,393 @@
package snapper
import (
"context"
"fmt"
"sort"
"sync"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/suspendresumesafetimer"
"github.com/zrepl/zrepl/zfs"
)
func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.SnapshottingPeriodic) (*Periodic, error) {
if in.Prefix == "" {
return nil, errors.New("prefix must not be empty")
}
if in.Interval <= 0 {
return nil, errors.New("interval must be positive")
}
hookList, err := hooks.ListFromConfig(&in.Hooks)
if err != nil {
return nil, errors.Wrap(err, "hook config error")
}
args := periodicArgs{
interval: in.Interval,
fsf: fsf,
planArgs: planArgs{
prefix: in.Prefix,
hooks: hookList,
},
// ctx and log is set in Run()
}
return &Periodic{state: SyncUp, args: args}, nil
}
type periodicArgs struct {
ctx context.Context
interval time.Duration
fsf zfs.DatasetFilter
planArgs planArgs
snapshotsTaken chan<- struct{}
dryRun bool
}
type Periodic struct {
args periodicArgs
mtx sync.Mutex
state State
// set in state Plan, used in Waiting
lastInvocation time.Time
// valid for state Snapshotting
plan *plan
// valid for state SyncUp and Waiting
sleepUntil time.Time
// valid for state Err
err error
}
//go:generate stringer -type=State
type State uint
const (
SyncUp State = 1 << iota
SyncUpErrWait
Planning
Snapshotting
Waiting
ErrorWait
Stopped
)
func (s State) sf() state {
m := map[State]state{
SyncUp: periodicStateSyncUp,
SyncUpErrWait: periodicStateWait,
Planning: periodicStatePlan,
Snapshotting: periodicStateSnapshot,
Waiting: periodicStateWait,
ErrorWait: periodicStateWait,
Stopped: nil,
}
return m[s]
}
type updater func(u func(*Periodic)) State
type state func(a periodicArgs, u updater) state
func (s *Periodic) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
getLogger(ctx).Debug("start")
defer getLogger(ctx).Debug("stop")
s.args.snapshotsTaken = snapshotsTaken
s.args.ctx = ctx
s.args.dryRun = false // for future expansion
u := func(u func(*Periodic)) State {
s.mtx.Lock()
defer s.mtx.Unlock()
if u != nil {
u(s)
}
return s.state
}
var st state = periodicStateSyncUp
for st != nil {
pre := u(nil)
st = st(s.args, u)
post := u(nil)
getLogger(ctx).
WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
Debug("state transition")
}
}
func onErr(err error, u updater) state {
return u(func(s *Periodic) {
s.err = err
preState := s.state
switch s.state {
case SyncUp:
s.state = SyncUpErrWait
case Planning:
fallthrough
case Snapshotting:
s.state = ErrorWait
}
getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
}).sf()
}
func onMainCtxDone(ctx context.Context, u updater) state {
return u(func(s *Periodic) {
s.err = ctx.Err()
s.state = Stopped
}).sf()
}
func periodicStateSyncUp(a periodicArgs, u updater) state {
u(func(snapper *Periodic) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
syncPoint, err := findSyncPoint(a.ctx, fss, a.planArgs.prefix, a.interval)
if err != nil {
return onErr(err, u)
}
u(func(s *Periodic) {
s.sleepUntil = syncPoint
})
ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, syncPoint)
if ctxDone != nil {
return onMainCtxDone(a.ctx, u)
}
return u(func(s *Periodic) {
s.state = Planning
}).sf()
}
func periodicStatePlan(a periodicArgs, u updater) state {
u(func(snapper *Periodic) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
p := makePlan(a.planArgs, fss)
return u(func(s *Periodic) {
s.state = Snapshotting
s.plan = p
s.err = nil
}).sf()
}
func periodicStateSnapshot(a periodicArgs, u updater) state {
var plan *plan
u(func(snapper *Periodic) {
plan = snapper.plan
})
ok := plan.execute(a.ctx, false)
select {
case a.snapshotsTaken <- struct{}{}:
default:
if a.snapshotsTaken != nil {
getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
}
}
return u(func(snapper *Periodic) {
if !ok {
snapper.state = ErrorWait
snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
} else {
snapper.state = Waiting
snapper.err = nil
}
}).sf()
}
func periodicStateWait(a periodicArgs, u updater) state {
var sleepUntil time.Time
u(func(snapper *Periodic) {
lastTick := snapper.lastInvocation
snapper.sleepUntil = lastTick.Add(a.interval)
sleepUntil = snapper.sleepUntil
log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
logFunc := log.Debug
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
logFunc = log.Error
}
logFunc("enter wait-state after error")
})
ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, sleepUntil)
if ctxDone != nil {
return onMainCtxDone(a.ctx, u)
}
return u(func(snapper *Periodic) {
snapper.state = Planning
}).sf()
}
func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {
return zfs.ZFSListMapping(ctx, mf)
}
var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
// see docs/snapshotting.rst
func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
const (
prioHasVersions int = iota
prioNoVersions
)
type snapTime struct {
ds *zfs.DatasetPath
prio int // lower is higher
time time.Time
}
if len(fss) == 0 {
return time.Now(), nil
}
snaptimes := make([]snapTime, 0, len(fss))
hardErrs := 0
now := time.Now()
getLogger(ctx).Debug("examine filesystem state to find sync point")
for _, d := range fss {
ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
if err == findSyncPointFSNoFilesystemVersionsErr {
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioNoVersions,
time: now,
})
} else if err != nil {
hardErrs++
getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
} else {
getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioHasVersions,
time: syncPoint,
})
}
}
if hardErrs == len(fss) {
return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
}
if len(snaptimes) == 0 {
panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
}
// sort ascending by (prio,time)
// => those filesystems with versions win over those without any
sort.Slice(snaptimes, func(i, j int) bool {
if snaptimes[i].prio == snaptimes[j].prio {
return snaptimes[i].time.Before(snaptimes[j].time)
}
return snaptimes[i].prio < snaptimes[j].prio
})
winnerSyncPoint := snaptimes[0].time
l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
l.Info("determined sync point")
if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
for _, st := range snaptimes {
if st.prio == prioNoVersions {
l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
}
}
}
return snaptimes[0].time, nil
}
var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
Types: zfs.Snapshots,
ShortnamePrefix: prefix,
})
if err != nil {
return time.Time{}, errors.Wrap(err, "list filesystem versions")
}
if len(fsvs) <= 0 {
return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
}
// Sort versions by creation
sort.SliceStable(fsvs, func(i, j int) bool {
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
})
latest := fsvs[len(fsvs)-1]
getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
since := now.Sub(latest.Creation)
if since < 0 {
return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
}
return latest.Creation.Add(interval), nil
}
type PeriodicReport struct {
State State
// valid in state SyncUp and Waiting
SleepUntil time.Time
// valid in state Err
Error string
// valid in state Snapshotting
Progress []*ReportFilesystem
}
func (s *Periodic) Report() Report {
s.mtx.Lock()
defer s.mtx.Unlock()
var progress []*ReportFilesystem = nil
if s.plan != nil {
progress = s.plan.report()
}
r := &PeriodicReport{
State: s.state,
SleepUntil: s.sleepUntil,
Error: errOrEmptyString(s.err),
Progress: progress,
}
return Report{Type: TypePeriodic, Periodic: r}
}
daemon/snapper/snapper.go
+21 -478
View File
@@ -3,497 +3,40 @@ package snapper
import (
"context"
"fmt"
"sort"
"sync"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/zfs"
)
//go:generate stringer -type=SnapState
type SnapState uint
type Type string
const (
SnapPending SnapState = 1 << iota
SnapStarted
SnapDone
SnapError
TypePeriodic Type = "periodic"
TypeCron Type = "cron"
TypeManual Type = "manual"
)
// All fields protected by Snapper.mtx
type snapProgress struct {
state SnapState
// SnapStarted, SnapDone, SnapError
name string
startAt time.Time
hookPlan *hooks.Plan
// SnapDone
doneAt time.Time
// SnapErr TODO disambiguate state
runResults hooks.PlanReport
type Snapper interface {
Run(ctx context.Context, snapshotsTaken chan<- struct{})
Report() Report
}
type args struct {
ctx context.Context
prefix string
interval time.Duration
fsf zfs.DatasetFilter
snapshotsTaken chan<- struct{}
hooks *hooks.List
dryRun bool
type Report struct {
Type Type
Periodic *PeriodicReport
Cron *CronReport
Manual *struct{}
}
type Snapper struct {
args args
mtx sync.Mutex
state State
// set in state Plan, used in Waiting
lastInvocation time.Time
// valid for state Snapshotting
plan map[*zfs.DatasetPath]*snapProgress
// valid for state SyncUp and Waiting
sleepUntil time.Time
// valid for state Err
err error
}
//go:generate stringer -type=State
type State uint
const (
SyncUp State = 1 << iota
SyncUpErrWait
Planning
Snapshotting
Waiting
ErrorWait
Stopped
)
func (s State) sf() state {
m := map[State]state{
SyncUp: syncUp,
SyncUpErrWait: wait,
Planning: plan,
Snapshotting: snapshot,
Waiting: wait,
ErrorWait: wait,
Stopped: nil,
}
return m[s]
}
type updater func(u func(*Snapper)) State
type state func(a args, u updater) state
type Logger = logger.Logger
func getLogger(ctx context.Context) Logger {
return logging.GetLogger(ctx, logging.SubsysSnapshot)
}
func PeriodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
if in.Prefix == "" {
return nil, errors.New("prefix must not be empty")
}
if in.Interval <= 0 {
return nil, errors.New("interval must be positive")
}
hookList, err := hooks.ListFromConfig(&in.Hooks)
if err != nil {
return nil, errors.Wrap(err, "hook config error")
}
args := args{
prefix: in.Prefix,
interval: in.Interval,
fsf: fsf,
hooks: hookList,
// ctx and log is set in Run()
}
return &Snapper{state: SyncUp, args: args}, nil
}
func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
getLogger(ctx).Debug("start")
defer getLogger(ctx).Debug("stop")
s.args.snapshotsTaken = snapshotsTaken
s.args.ctx = ctx
s.args.dryRun = false // for future expansion
u := func(u func(*Snapper)) State {
s.mtx.Lock()
defer s.mtx.Unlock()
if u != nil {
u(s)
}
return s.state
}
var st state = syncUp
for st != nil {
pre := u(nil)
st = st(s.args, u)
post := u(nil)
getLogger(ctx).
WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
Debug("state transition")
}
}
func onErr(err error, u updater) state {
return u(func(s *Snapper) {
s.err = err
preState := s.state
switch s.state {
case SyncUp:
s.state = SyncUpErrWait
case Planning:
fallthrough
case Snapshotting:
s.state = ErrorWait
}
getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
}).sf()
}
func onMainCtxDone(ctx context.Context, u updater) state {
return u(func(s *Snapper) {
s.err = ctx.Err()
s.state = Stopped
}).sf()
}
func syncUp(a args, u updater) state {
u(func(snapper *Snapper) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
syncPoint, err := findSyncPoint(a.ctx, fss, a.prefix, a.interval)
if err != nil {
return onErr(err, u)
}
u(func(s *Snapper) {
s.sleepUntil = syncPoint
})
t := time.NewTimer(time.Until(syncPoint))
defer t.Stop()
select {
case <-t.C:
return u(func(s *Snapper) {
s.state = Planning
}).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}
}
func plan(a args, u updater) state {
u(func(snapper *Snapper) {
snapper.lastInvocation = time.Now()
})
fss, err := listFSes(a.ctx, a.fsf)
if err != nil {
return onErr(err, u)
}
plan := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
for _, fs := range fss {
plan[fs] = &snapProgress{state: SnapPending}
}
return u(func(s *Snapper) {
s.state = Snapshotting
s.plan = plan
s.err = nil
}).sf()
}
func snapshot(a args, u updater) state {
var plan map[*zfs.DatasetPath]*snapProgress
u(func(snapper *Snapper) {
plan = snapper.plan
})
hookMatchCount := make(map[hooks.Hook]int, len(*a.hooks))
for _, h := range *a.hooks {
hookMatchCount[h] = 0
}
anyFsHadErr := false
// TODO channel programs -> allow a little jitter?
for fs, progress := range plan {
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
ctx := logging.WithInjectedField(a.ctx, "fs", fs.ToString())
ctx = logging.WithInjectedField(ctx, "snap", snapname)
hookEnvExtra := hooks.Env{
hooks.EnvFS: fs.ToString(),
hooks.EnvSnapshot: snapname,
}
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
l := getLogger(ctx)
l.Debug("create snapshot")
err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
if err != nil {
l.WithError(err).Error("cannot create snapshot")
}
return
})
fsHadErr := false
var planReport hooks.PlanReport
var plan *hooks.Plan
{
filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
if err != nil {
getLogger(ctx).WithError(err).Error("unexpected filter error")
fsHadErr = true
goto updateFSState
}
// account for running hooks
for _, h := range filteredHooks {
hookMatchCount[h] = hookMatchCount[h] + 1
}
var planErr error
plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
if planErr != nil {
fsHadErr = true
getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
goto updateFSState
}
}
u(func(snapper *Snapper) {
progress.name = snapname
progress.startAt = time.Now()
progress.hookPlan = plan
progress.state = SnapStarted
})
{
getLogger(ctx).WithField("report", plan.Report().String()).Debug("begin run job plan")
plan.Run(ctx, a.dryRun)
planReport = plan.Report()
fsHadErr = planReport.HadError() // not just fatal errors
if fsHadErr {
getLogger(ctx).WithField("report", planReport.String()).Error("end run job plan with error")
} else {
getLogger(ctx).WithField("report", planReport.String()).Info("end run job plan successful")
}
}
updateFSState:
anyFsHadErr = anyFsHadErr || fsHadErr
u(func(snapper *Snapper) {
progress.doneAt = time.Now()
progress.state = SnapDone
if fsHadErr {
progress.state = SnapError
}
progress.runResults = planReport
})
}
select {
case a.snapshotsTaken <- struct{}{}:
func FromConfig(g *config.Global, fsf zfs.DatasetFilter, in config.SnapshottingEnum) (Snapper, error) {
switch v := in.Ret.(type) {
case *config.SnapshottingPeriodic:
return periodicFromConfig(g, fsf, v)
case *config.SnapshottingCron:
return cronFromConfig(fsf, *v)
case *config.SnapshottingManual:
return &manual{}, nil
default:
if a.snapshotsTaken != nil {
getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
return nil, fmt.Errorf("unknown snapshotting type %T", v)
}
}
for h, mc := range hookMatchCount {
if mc == 0 {
hookIdx := -1
for idx, ah := range *a.hooks {
if ah == h {
hookIdx = idx
break
}
}
getLogger(a.ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
}
}
return u(func(snapper *Snapper) {
if anyFsHadErr {
snapper.state = ErrorWait
snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
} else {
snapper.state = Waiting
snapper.err = nil
}
}).sf()
}
func wait(a args, u updater) state {
var sleepUntil time.Time
u(func(snapper *Snapper) {
lastTick := snapper.lastInvocation
snapper.sleepUntil = lastTick.Add(a.interval)
sleepUntil = snapper.sleepUntil
log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
logFunc := log.Debug
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
logFunc = log.Error
}
logFunc("enter wait-state after error")
})
t := time.NewTimer(time.Until(sleepUntil))
defer t.Stop()
select {
case <-t.C:
return u(func(snapper *Snapper) {
snapper.state = Planning
}).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}
}
func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {
return zfs.ZFSListMapping(ctx, mf)
}
var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
// see docs/snapshotting.rst
func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
const (
prioHasVersions int = iota
prioNoVersions
)
type snapTime struct {
ds *zfs.DatasetPath
prio int // lower is higher
time time.Time
}
if len(fss) == 0 {
return time.Now(), nil
}
snaptimes := make([]snapTime, 0, len(fss))
hardErrs := 0
now := time.Now()
getLogger(ctx).Debug("examine filesystem state to find sync point")
for _, d := range fss {
ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
if err == findSyncPointFSNoFilesystemVersionsErr {
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioNoVersions,
time: now,
})
} else if err != nil {
hardErrs++
getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
} else {
getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioHasVersions,
time: syncPoint,
})
}
}
if hardErrs == len(fss) {
return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
}
if len(snaptimes) == 0 {
panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
}
// sort ascending by (prio,time)
// => those filesystems with versions win over those without any
sort.Slice(snaptimes, func(i, j int) bool {
if snaptimes[i].prio == snaptimes[j].prio {
return snaptimes[i].time.Before(snaptimes[j].time)
}
return snaptimes[i].prio < snaptimes[j].prio
})
winnerSyncPoint := snaptimes[0].time
l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
l.Info("determined sync point")
if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
for _, st := range snaptimes {
if st.prio == prioNoVersions {
l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
}
}
}
return snaptimes[0].time, nil
}
var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
Types: zfs.Snapshots,
ShortnamePrefix: prefix,
})
if err != nil {
return time.Time{}, errors.Wrap(err, "list filesystem versions")
}
if len(fsvs) <= 0 {
return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
}
// Sort versions by creation
sort.SliceStable(fsvs, func(i, j int) bool {
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
})
latest := fsvs[len(fsvs)-1]
getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
since := now.Sub(latest.Creation)
if since < 0 {
return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
}
return latest.Creation.Add(interval), nil
}
daemon/snapper/snapper_all.go
-48
View File
@@ -1,48 +0,0 @@
package snapper
import (
"context"
"fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/zfs"
)
// FIXME: properly abstract snapshotting:
// - split up things that trigger snapshotting from the mechanism
// - timer-based trigger (periodic)
// - call from control socket (manual)
// - mixed modes?
// - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
type PeriodicOrManual struct {
s *Snapper
}
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
if s.s != nil {
s.s.Run(ctx, wakeUpCommon)
}
}
// Returns nil if manual
func (s *PeriodicOrManual) Report() *Report {
if s.s != nil {
return s.s.Report()
}
return nil
}
func FromConfig(g *config.Global, fsf zfs.DatasetFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
switch v := in.Ret.(type) {
case *config.SnapshottingPeriodic:
snapper, err := PeriodicFromConfig(g, fsf, v)
if err != nil {
return nil, err
}
return &PeriodicOrManual{snapper}, nil
case *config.SnapshottingManual:
return &PeriodicOrManual{}, nil
default:
return nil, fmt.Errorf("unknown snapshotting type %T", v)
}
}
daemon/snapper/snapper_report.go
-118
View File
@@ -1,118 +0,0 @@
package snapper
import (
"fmt"
"sort"
"strings"
"time"
"github.com/zrepl/zrepl/daemon/hooks"
)
type Report struct {
State State
// valid in state SyncUp and Waiting
SleepUntil time.Time
// valid in state Err
Error string
// valid in state Snapshotting
Progress []*ReportFilesystem
}
type ReportFilesystem struct {
Path string
State SnapState
// Valid in SnapStarted and later
SnapName string
StartAt time.Time
Hooks string
HooksHadError bool
// Valid in SnapDone | SnapError
DoneAt time.Time
}
func errOrEmptyString(e error) string {
if e != nil {
return e.Error()
}
return ""
}
func (s *Snapper) Report() *Report {
s.mtx.Lock()
defer s.mtx.Unlock()
pReps := make([]*ReportFilesystem, 0, len(s.plan))
for fs, p := range s.plan {
var hooksStr string
var hooksHadError bool
if p.hookPlan != nil {
hr := p.hookPlan.Report()
// FIXME: technically this belongs into client
// but we can't serialize hooks.Step ATM
rightPad := func(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
hooksHadError = hr.HadError()
rows := make([][]string, len(hr))
const numCols = 4
lens := make([]int, numCols)
for i, e := range hr {
rows[i] = make([]string, numCols)
rows[i][0] = fmt.Sprintf("%d", i+1)
rows[i][1] = e.Status.String()
runTime := "..."
if e.Status != hooks.StepPending {
runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
}
rows[i][2] = runTime
rows[i][3] = ""
if e.Report != nil {
rows[i][3] = e.Report.String()
}
for j, col := range lens {
if len(rows[i][j]) > col {
lens[j] = len(rows[i][j])
}
}
}
rowsFlat := make([]string, len(hr))
for i, r := range rows {
colsPadded := make([]string, len(r))
for j, c := range r[:len(r)-1] {
colsPadded[j] = rightPad(c, lens[j], " ")
}
colsPadded[len(r)-1] = r[len(r)-1]
rowsFlat[i] = strings.Join(colsPadded, " ")
}
hooksStr = strings.Join(rowsFlat, "\n")
}
pReps = append(pReps, &ReportFilesystem{
Path: fs.ToString(),
State: p.state,
SnapName: p.name,
StartAt: p.startAt,
DoneAt: p.doneAt,
Hooks: hooksStr,
HooksHadError: hooksHadError,
})
}
sort.Slice(pReps, func(i, j int) bool {
return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
})
r := &Report{
State: s.state,
SleepUntil: s.sleepUntil,
Error: errOrEmptyString(s.err),
Progress: pReps,
}
return r
}
daemon/snapper/util.go
+21
View File
@@ -0,0 +1,21 @@
package snapper
import (
"context"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
)
type Logger = logger.Logger
func getLogger(ctx context.Context) Logger {
return logging.GetLogger(ctx, logging.SubsysSnapshot)
}
func errOrEmptyString(e error) string {
if e != nil {
return e.Error()
}
return ""
}
dist/grafana/grafana-prometheus-zrepl.json
Vendored
+1 -1
View File
@@ -249,7 +249,7 @@
"stack": true,
"steppedLine": false,
"targets": [{
"expr": "zrepl_version_daemon{job='$prom_job_name'}",
"expr": "sgn(zrepl_start_time{job='$prom_job_name'})",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
docs/changelog.rst
+90 -17
View File
@@ -6,6 +6,7 @@
.. |docs| replace:: [DOCS]
.. |feature| replace:: [FEATURE]
.. |mig| replace:: **[MIGRATION]**
.. |maint| replace:: [MAINT]
.. _changelog:
@@ -15,17 +16,96 @@ Changelog
The changelog summarizes bugfixes that are deemed relevant for users and package maintainers.
Developers should consult the git commit log or GitHub issue tracker.
We use the following annotations for classifying changes:
0.6 (Unreleased)
----------------
* |break_config| Change that breaks the config.
As a package maintainer, make sure to warn your users about config breakage somehow.
* |break| Change that breaks interoperability or persistent state representation with previous releases.
As a package maintainer, make sure to warn your users about config breakage somehow.
Note that even updating the package on both sides might not be sufficient, e.g. if persistent state needs to be migrated to a new format.
* |mig| Migration that must be run by the user.
* |feature| Change that introduces new functionality.
* |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
* |docs| Change to the documentation.
* `Feature Wishlist on GitHub <https://github.com/zrepl/zrepl/discussions/547>`_
* |feature| :ref:`Schedule-based snapshotting<job-snapshotting--cron>` using ``cron`` syntax instead of an interval.
* |feature| Add ``ZREPL_DESTROY_MAX_BATCH_SIZE`` env var (default 0=unlimited).
* |bugfix| Fix resuming from interrupted replications that use ``send.raw`` on unencrypted datasets.
* The send options introduced in zrepl 0.4 allow users to specify additional zfs send flags for zrepl to use.
Before this fix, when setting ``send.raw=true`` on a job that replicates unencrypted datasets,
zrepl would not allow an interrupted replication to resume.
The reason were overly cautious checks to support the ``send.encrypted`` option.
* This bugfix removes these checks from the replication planner.
This makes ``send.encrypted`` a sender-side-only concern, much like all other ``send.*`` flags.
* However, this means that the ``zrepl status`` UI no longer indicates whether a replication step uses encrypted sends or not.
The setting is still effective though.
* |break| |feature| convert Prometheus metric ``zrepl_version_daemon`` to ``zrepl_start_time`` metric
* The metric still reports the zrepl version in a label.
But the metric *value* is now the Unix timestamp at the time the daemon was started.
The Grafana dashboard in :repomasterlink:`dist/grafana` has been updated.
* |bugfix| transient zrepl status error: ``Post "http://unix/status": EOF``
* |bugfix| don't treat receive-side bookmarks as a replication conflict.
This facilitates chaining of replication jobs. See :issue:`490`.
0.5
---
* |feature| :ref:`Bandwidth limiting <job-send-recv-options--bandwidth-limit>` (Thanks, Prominic.NET, Inc.)
* |feature| zrepl status: use a ``*`` to indicate which filesystem is currently replicating
* |feature| include daemon environment variables in zrepl status (currently only in ``--raw``)
* |bugfix| **fix encrypt-on-receive + placeholders use case** (:issue:`504`)
* Before this fix, **plain sends** to a receiver with an encrypted ``root_fs`` **could be received unencrypted** if zrepl needed to create placeholders on the receiver.
* Existing zrepl users should :ref:`read the docs <job-recv-options--placeholder>` and check ``zfs get -r encryption,zrepl:placeholder PATH_TO_ROOTFS`` on the receiver.
* Thanks to `@mologie <https://github.com/mologie>`_ and `@razielgn <https://github.com/razielgn>`_ for reporting and testing!
* |bugfix| Rename mis-spelled :ref:`send option <job-send-options>` ``embbeded_data`` to ``embedded_data``.
* |bugfix| zrepl status: replication step numbers should start at 1
* |bugfix| incorrect bandwidth averaging in ``zrepl status``.
* |bugfix| FreeBSD with OpenZFS 2.0: zrepl would wait indefinitely for zfs send to exit on timeouts.
* |bugfix| fix ``strconv.ParseInt: value out of range`` bug (and use the control RPCs).
* |docs| improve description of multiple pruning rules.
* |docs| document :ref:`platform tests <usage-platform-tests>`.
* |docs| quickstart: make users aware that prune rules apply to all snapshots.
* |maint| some platformtests were broken.
* |maint| FreeBSD: release armv7 and arm64 binaries.
* |maint| apt repo: update instructions due to ``apt-key`` deprecation.
Note to all users: please read up on the following OpenZFS bugs, as you might be affected:
* `ZFS send/recv with ashift 9->12 leads to data corruption <https://github.com/openzfs/zfs/issues/12762>`_.
* Various bugs with encrypted send/recv (`Leadership meeting notes <https://openzfs.topicbox.com/groups/developer/T24bdaa2886c6cbf5-Mc039a11c3f1507ea0664817b/december-openzfs-leadership-meeting>`_)
Finally, I'd like to point you to the `GitHub discussion <https://github.com/zrepl/zrepl/discussions/547>`_ about which bugfixes and features should be prioritized in zrepl 0.6 and beyond!
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
0.4.0
-----
* |feature| support setting zfs send / recv flags in the config (send: ``-wLcepbS`` , recv: ``-ox`` ).
Config docs :ref:`here <job-send-options>` and :ref:`here <job-recv-options>` .
* |feature| parallel replication is now configurable (disabled by default, :ref:`config docs here <replication-option-concurrency>` ).
* |feature| New ``zrepl status`` UI:
* Interactive job selection.
* Interactively ``zrepl signal`` jobs.
* Filter filesystems in the job view by name.
* An approximation of the old UI is still included as `--mode legacy` but will be removed in a future release of zrepl.
* |bugfix| Actually use concurrency when listing zrepl abstractions & doing size estimation.
These operations were accidentally made sequential in zrepl 0.3.
* |bugfix| Job hang-up during second replication attempt.
* |bugfix| Data races conditions in the dataconn rpc stack.
* |maint| Update to protobuf v1.25 and grpc 1.35.
For users who skipped the 0.3.1 update: please make sure your pruning grid config is correct.
The following bugfix in 0.3.1 :issue:`caused problems for some users <400>`:
* |bugfix| pruning: ``grid``: add all snapshots that do not match the regex to the rule's destroy list.
0.3.1
-----
@@ -45,13 +125,6 @@ Mostly a bugfix release for :ref:`zrepl 0.3 <release-0.3>`.
* CI infrastructure rework
* Continuous deployment of that new `stable` branch to zrepl.github.io.
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
.. _release-0.3:
0.3
docs/configuration.rst
+1
View File
@@ -13,6 +13,7 @@ Configuration
configuration/filter_syntax
configuration/sendrecvoptions
configuration/replication
configuration/conflict_resolution
configuration/snapshotting
configuration/prune
configuration/logging
docs/configuration/conflict_resolution.rst
+36
View File
@@ -0,0 +1,36 @@
.. include:: ../global.rst.inc
Conflict Resolution Options
===========================
::
jobs:
- type: push
filesystems: ...
conflict_resolution:
initial_replication: most_recent | all | fail # default: most_recent
...
.. _conflict_resolution-initial_replication-option-send_all_snapshots:
``initial_replication`` option
------------------------------
The ``initial_replication`` option determines how many snapshots zrepl replicates if the filesystem has not been replicated before.
If ``most_recent`` (the default), the initial replication will only transfer the most recent snapshot, while ignoring previous snapshots.
If all snapshots should be replicated, specify ``all``.
Use ``fail`` to make replication of the filesystem fail in case there is no corresponding fileystem on the receiver.
For example, suppose there are snapshosts ``tank@1``, ``tank@2``, ``tank@3`` on a sender.
Then ``most_recent`` will replicate just ``@3``, but ``all`` will replicate ``@1``, ``@2``, and ``@3``.
If initial replication is interrupted, and there is at least one (maybe partial) snapshot on the receiver, zrepl will always resume in **incremental mode**.
And that is regardless of where the initial replication was interrupted.
For example, if ``initial_replication: all`` and the transfer of ``@1`` is interrupted, zrepl would retry/resume at ``@1``.
And even if the user changes the config to ``initial_replication: most_recent`` before resuming, **incremental mode** will still resume at ``@1``.
docs/configuration/misc.rst
+2
View File
@@ -1,3 +1,5 @@
.. _miscellaneous:
Miscellaneous
=============
docs/configuration/monitoring.rst
+2 -2
View File
@@ -13,7 +13,7 @@ Prometheus & Grafana
--------------------
zrepl can expose `Prometheus metrics <https://prometheus.io/docs/instrumenting/exposition_formats/>`_ via HTTP.
The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_ string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``.
The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_ string for tcp, e.g. ``:9811`` or ``127.0.0.1:9811`` (port 9811 was reserved to zrepl `on the official list <https://github.com/prometheus/prometheus/wiki/Default-port-allocations/_compare/43e495dd251ee328ac0d08b58084665b5c0f7a7e...459195059b55b414193ebeb80c5ba463d2606951>`_).
The ``listen_freebind`` attribute is :ref:`explained here <listen-freebind-explanation>`.
The Prometheus monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
@@ -30,7 +30,7 @@ The dashboard also contains some advice on which metrics are important to monito
global:
monitoring:
- type: prometheus
listen: ':9091'
listen: ':9811'
listen_freebind: true # optional, default false
docs/configuration/overview.rst
+82 -44
View File
@@ -3,16 +3,17 @@ Overview & Terminology
======================
All work zrepl does is performed by the zrepl daemon which is configured in a single YAML configuration file loaded on startup.
The following paths are considered:
The following paths are searched, in this order:
* If set, the location specified via the global ``--config`` flag
* ``/etc/zrepl/zrepl.yml``
* ``/usr/local/etc/zrepl/zrepl.yml``
1. The path specified via the global ``--config`` flag
2. ``/etc/zrepl/zrepl.yml``
3. ``/usr/local/etc/zrepl/zrepl.yml``
The ``zrepl configcheck`` subcommand can be used to validate the configuration.
The command will output nothing and exit with zero status code if the configuration is valid.
``zrepl configcheck`` can be used to validate the configuration.
If the configuration is valid, it will output nothing and exit with code ``0``.
The error messages vary in quality and usefulness: please report confusing config errors to the tracking :issue:`155`.
Full example configs such as in the :ref:`quick-start guides <quickstart-toc>` or the :sampleconf:`/` directory might also be helpful.
Full example configs are available at :ref:`quick-start guides <quickstart-toc>` and :sampleconf:`/`.
However, copy-pasting examples is no substitute for reading documentation!
Config File Structure
@@ -26,9 +27,8 @@ Config File Structure
type: push
- ...
zrepl is configured using a single YAML configuration file with two main sections: ``global`` and ``jobs``.
The ``global`` section is filled with sensible defaults and is covered later in this chapter.
The ``jobs`` section is a list of jobs which we are going to explain now.
A zrepl configuration file is divided in to two main sections: ``global`` and ``jobs``.
``global`` has sensible defaults. It is covered in :ref:`logging <logging>`, :ref:`monitoring <monitoring>` \& :ref:`miscellaneous <miscellaneous>`.
.. _job-overview:
@@ -42,8 +42,7 @@ Jobs are identified by their ``name``, both in log files and the ``zrepl status`
.. NOTE::
The job name is persisted in several places on disk and thus :issue:`cannot be changed easily<327>`.
Replication always happens between a pair of jobs: one is the **active side**, and one the **passive side**.
Replication always happens between a pair of jobs: one **active side** and one **passive side**.
The active side connects to the passive side using a :ref:`transport <transport>` and starts executing the replication logic.
The passive side responds to requests from the active side after checking its permissions.
@@ -72,30 +71,29 @@ How the Active Side Works
The active side (:ref:`push <job-push>` and :ref:`pull <job-pull>` job) executes the replication and pruning logic:
* Wakeup because of finished snapshotting (``push`` job) or pull interval ticker (``pull`` job).
* Connect to the corresponding passive side using a :ref:`transport <transport>` and instantiate an RPC client.
* Replicate data from the sending to the receiving side (see below).
* Prune on sender & receiver.
1. Wakeup after snapshotting (``push`` job) or pull interval ticker (``pull`` job).
2. Connect to the passive side and instantiate an RPC client.
3. Replicate data from the sender to the receiver.
4. Prune on sender & receiver.
.. TIP::
The progress of the active side can be watched live using the ``zrepl status`` subcommand.
The progress of the active side can be watched live using ``zrepl status``.
.. _overview-passive-side--client-identity:
How the Passive Side Works
--------------------------
The passive side (:ref:`sink <job-sink>` and :ref:`source <job-source>`) waits for connections from the corresponding active side,
using the transport listener type specified in the ``serve`` field of the job configuration.
When a client connects, the transport listener performS listener-specific access control (cert validation, IP ACLs, etc)
and determines the *client identity*.
The passive side job then uses this client identity as follows:
The passive side (:ref:`sink <job-sink>` and :ref:`source <job-source>`) waits for connections from the active side,
on the :ref:`transport <transport>` specified with ``serve`` in the job configuration.
The respective transport then perfoms authentication & authorization, resulting in a stable *client identity*.
The passive side job uses this *client identity* as follows:
* The ``sink`` job maps requests from different client identities to their respective sub-filesystem tree ``root_fs/${client_identity}``.
* The ``source`` might, in the future, embed the client identity in :ref:`zrepl's ZFS abstraction names <zrepl-zfs-abstractions>` in order to support multi-host replication.
* In ``sink`` jobs, to map requests from different *client identities* to their respective sub-filesystem tree ``root_fs/${client_identity}``.
* *In the future, ``source`` might embed the client identity in :ref:`zrepl's ZFS abstraction names <zrepl-zfs-abstractions>`, to support multi-host replication.*
.. TIP::
The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
The use of the client identity in the ``sink`` job implies that it must be usable as a ZFS ZFS filesystem name component.
.. _overview-how-replication-works:
@@ -106,7 +104,7 @@ One of the major design goals of the replication module is to avoid any duplicat
As such, the code works on abstract senders and receiver **endpoints**, where typically one will be implemented by a local program object and the other is an RPC client instance.
Regardless of push- or pull-style setup, the logic executes on the active side, i.e. in the ``push`` or ``pull`` job.
The following high-level steps take place during replication and can be monitored using the ``zrepl status`` subcommand:
The following high-level steps take place during replication and can be monitored using ``zrepl status``:
* Plan the replication:
@@ -148,6 +146,22 @@ Incremental sends require that ``@from`` be present on the receiving side when r
Incremental sends can also use a ZFS bookmark as *from* on the sending side (``zfs send -i #bm_from fs@to``), where ``#bm_from`` was created using ``zfs bookmark fs@from fs#bm_from``.
The receiving side must always have the actual snapshot ``@from``, regardless of whether the sending side uses ``@from`` or a bookmark of it.
.. _zfs-background-knowledge-plain-vs-raw-sends:
**Plain and raw sends**
By default, ``zfs send`` sends the most generic, backwards-compatible data stream format (so-called 'plain send').
If the sent uses newer features, e.g. compression or encryption, ``zfs send`` has to un-do these operations on the fly to produce the plain send stream.
If the receiver uses newer features (e.g. compression or encryption inherited from the parent FS), it applies the necessary transformations again on the fly during ``zfs recv``.
Flags such as ``-e``, ``-c`` and ``-L`` tell ZFS to produce a send stream that is closer to how the data is stored on disk.
Sending with those flags removes computational overhead from sender and receiver.
However, the receiver will not apply certain transformations, e.g., it will not compress with the receive-side ``compression`` algorithm.
The ``-w`` (``--raw``) flag produces a send stream that is as *raw* as possible.
For unencrypted datasets, its current effect is the same as ``-Lce``.
Encrypted datasets can only be sent plain (unencrypted) or raw (encrypted) using the ``-w`` flag.
**Resumable Send & Recv**
The ``-s`` flag for ``zfs recv`` tells zfs to save the partially received send stream in case it is interrupted.
To resume the replication, the receiving side filesystem's ``receive_resume_token`` must be passed to a new ``zfs send -t <value> | zfs recv`` command.
@@ -217,22 +231,11 @@ The ``zrepl zfs-abstraction list`` command provides a listing of all bookmarks a
More details can be found in the design document :repomasterlink:`replication/design.md`.
Limitations
^^^^^^^^^^^
.. ATTENTION::
Caveats With Complex Setups (More Than 2 Jobs or Machines)
----------------------------------------------------------
Currently, zrepl does not replicate filesystem properties.
When receiving a filesystem, it is never mounted (`-u` flag) and `mountpoint=none` is set.
This is temporary and being worked on :issue:`24`.
.. _jobs-multiple-jobs:
Multiple Jobs & More than 2 Machines
------------------------------------
The quick-start guides focus on simple setups with a single sender and a single receiver.
Most users are served well with a single sender and a single receiver job.
This section documents considerations for more complex setups.
.. ATTENTION::
@@ -246,7 +249,7 @@ This section documents considerations for more complex setups.
If you can't find your desired configuration, have questions or would like to see improvements to multi-job setups, please `open an issue on GitHub <https://github.com/zrepl/zrepl/issues/new>`_.
Multiple Jobs on one Machine
Multiple Jobs on One Machine
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
As a general rule, multiple jobs configured on one machine **must operate on disjoint sets of filesystems**.
Otherwise, concurrently running jobs might interfere when operating on the same filesystem.
@@ -265,18 +268,53 @@ On your setup, ensure that
This scenario is detailed in one of the :ref:`quick-start guides <quickstart-backup-to-external-disk>`.
More Than 2 Machines
Two Or More Machines
^^^^^^^^^^^^^^^^^^^^
This section might be relevant to users who wish to *fan-in* (N machines replicate to 1) or *fan-out* (replicate 1 machine to N machines).
**Working setups**:
* N ``push`` identities, 1 ``sink`` (as long as the different push jobs have a different :ref:`client identity <overview-passive-side--client-identity>`)
* **Fan-in: N servers replicated to one receiver, disjoint dataset trees.**
* ``sink`` constrains each client to a disjoint sub-tree of the sink-side dataset hierarchy ``${root_fs}/${client_identity}``.
* This is the common use case of a centralized backup server.
* Implementation:
* N ``push`` jobs (one per sender server), 1 ``sink`` (as long as the different push jobs have a different :ref:`client identity <overview-passive-side--client-identity>`)
* N ``source`` jobs (one per sender server), N ``pull`` on the receiver server (unique names, disjoint ``root_fs``)
* The ``sink`` job automatically constrains each client to a disjoint sub-tree of the sink-side dataset hierarchy ``${root_fs}/${client_identity}``.
Therefore, the different clients cannot interfere.
* The ``pull`` job only pulls from one host, so it's up to the zrepl user to ensure that the different ``pull`` jobs don't interfere.
.. _fan-out-replication:
* **Fan-out: 1 server replicated to N receivers**
* Can be implemented either in a pull or push fashion.
* **pull setup**: 1 ``pull`` job on each receiver server, each with a corresponding **unique** ``source`` job on the sender server.
* **push setup**: 1 ``sink`` job on each receiver server, each with a corresponding **unique** ``push`` job on the sender server.
* It is critical that we have one sending-side job (``source``, ``push``) per receiver.
The reason is that :ref:`zrepl's ZFS abstractions <zrepl-zfs-abstractions>` (``zrepl zfs-abstraction list``) include the name of the ``source``/``push`` job, but not the receive-side job name or client identity (see :issue:`380`).
As a counter-example, suppose we used multiple ``pull`` jobs with only one ``source`` job.
All ``pull`` jobs would share the same :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` and trip over each other, breaking incremental replication guarantees quickly.
The anlogous problem exists for 1 ``push`` to N ``sink`` jobs.
* The ``filesystems`` matched by the sending side jobs (``source``, ``push``) need not necessarily be disjoint.
For this to work, we need to avoid interference between snapshotting and pruning of the different sending jobs.
The solution is to centralize sender-side snapshot management in a separate ``snap`` job.
Snapshotting in the ``source``/``push`` job should then be disabled (``type: manual``).
And sender-side pruning (``keep_sender``) needs to be disabled in the active side (``pull`` / ``push``), since that'll be done by the ``snap job``.
* **Restore limitations**: when restoring from one of the ``pull`` targets (e.g., using ``zfs send -R``), the replication cursor bookmarks don't exist on the restored system.
This can break incremental replication to all other receive-sides after restore.
* See :ref:`the fan-out replication quick-start guide <quickstart-fan-out-replication>` for an example of this setup.
**Setups that do not work**:
docs/configuration/prune.rst
+32 -23
View File
@@ -67,8 +67,9 @@ Policy ``not_replicated``
...
``not_replicated`` keeps all snapshots that have not been replicated to the receiving side.
It only makes sense to specify this rule on a sender (source or push job).
The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` on the sending side.
It only makes sense to specify this rule for the ``keep_sender``.
The reason is that, by definition, all snapshots on the receiver have already been replicated to there from the sender.
To determine whether a sender-side snapshot has already been replicated, zrepl uses the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` which corresponds to the most recent successfully replicated snapshot.
.. _prune-keep-retention-grid:
@@ -106,7 +107,7 @@ The following procedure happens during pruning:
#. All subsequent buckets are placed adjacent to their predecessor bucket.
#. Now each snapshot on the axis either falls into one bucket or it is older than our rightmost bucket.
Buckets are left-inclusive and right-exclusive which means that a snapshot on the edge of bucket will always 'fall into the right one'.
#. Snapshots older than the rightmost bucket **not kept** by this gridspec.
#. Snapshots older than the rightmost bucket are **not kept** by the grid specification.
#. For each bucket, we only keep the ``keep`` oldest snapshots.
The syntax to describe the bucket list is as follows:
@@ -124,43 +125,51 @@ The syntax to describe the bucket list is as follows:
::
This grid spec produces the following list of adjacent buckets. For the sake of simplicity,
we subject all snapshots to the grid pruning policy by settings `regex: .*`.
Assume the following grid specification:
`
grid: 1x1h(keep=all) | 2x2h | 1x3h
regex: .*
`
This grid specification produces the following constellation of buckets:
0h 1h 2h 3h 4h 5h 6h 7h 8h 9h
| | | | | | | | | |
|-Bucket1-|-----Bucket 2------|------Bucket 3-----|-----------Bucket 4----------|
|-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
| keep=all| keep=1 | keep=1 | keep=1 |
Let us consider the following set of snapshots @a-zA-C:
Now assume that we have a set of snapshots @a, @b, ..., @D.
Snapshot @a is the most recent snapshot.
Snapshot @D is the oldest snapshot, it is almost 9 hours older than snapshot @a.
We place the snapshots on the same timeline as the buckets:
| a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D |
0h 1h 2h 3h 4h 5h 6h 7h 8h 9h
| | | | | | | | | |
|-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
| keep=all| keep=1 | keep=1 | keep=1 |
| | | | |
| a b c | d e f g h i j k l m n o p |q r s t u v w x y z |A B C D
The `grid` algorithm maps them to their respective buckets:
We obtain the following mapping of snapshots to buckets:
Bucket1: a,b,c
Bucket 2: d,e,f,g,h,i,j
Bucket 3: k,l,m,n,o,p
Bucket 4: q,r, q,r,s,t,u,v,w,x,y,z
None: A,B,C,D
Bucket2: d,e,f,g,h,i
Bucket3: j,k,l,m,n,o,p
Bucket4: q,r,s,t,u,v,w,x,y,z
No bucket: A,B,C,D
It then applies the per-bucket pruning logic described above which resulting in the
following list of remaining snapshots.
For each bucket, we now prune snapshots until it only contains `keep` snapshots.
Newer snapshots are destroyed first.
Snapshots that do not fall into a bucket are always destroyed.
| a b c j p z |
Note that it only makes sense to grow (not shorten) the interval duration for buckets
further in the past since each bucket acts like a low-pass filter for incoming snapshots
and adding a less-low-pass-filter after a low-pass one has no effect.
Result after pruning:
0h 1h 2h 3h 4h 5h 6h 7h 8h 9h
| | | | | | | | | |
|-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
| | | | |
| a b c | i | p | z |
.. _prune-keep-last-n:
docs/configuration/replication.rst
+27
View File
@@ -14,6 +14,10 @@ Replication Options
protection:
initial: guarantee_resumability # guarantee_{resumability,incremental,nothing}
incremental: guarantee_resumability # guarantee_{resumability,incremental,nothing}
concurrency:
size_estimates: 4
steps: 1
...
.. _replication-option-protection:
@@ -45,3 +49,26 @@ which is useful if replication happens so rarely (or fails so frequently) that t
When changing this flag, obsoleted zrepl-managed bookmarks and holds will be destroyed on the next replication step that is attempted for each filesystem.
.. _replication-option-concurrency:
``concurrency`` option
----------------------
The ``concurrency`` options control the maximum amount of concurrency during replication.
The default values allow some concurrency during size estimation but no parallelism for the actual replication.
* ``concurrency.steps`` (default = 1) controls the maximum number of concurrently executed :ref:`replication steps <overview-how-replication-works>`.
The planning step for each file system is counted as a single step.
* ``concurrency.size_estimates`` (default = 4) controls the maximum number of concurrent step size estimations done by the job.
Note that initial replication cannot start replicating child filesystems before the parent filesystem's initial replication step has completed.
Some notes on tuning these values:
* Disk: Size estimation is less I/O intensive than step execution because it does not need to access the data blocks.
* CPU: Size estimation is usually a dense CPU burst whereas step execution CPU utilization is stretched out over time because of disk IO.
Faster disks, sending a compressed dataset in :ref:`plain mode <zfs-background-knowledge-plain-vs-raw-sends>` and the zrepl transport mode all contribute to higher CPU requirements.
* Network bandwidth: Size estimation does not consume meaningful amounts of bandwidth, step execution does.
* :ref:`zrepl ZFS abstractions <zrepl-zfs-abstractions>`: for each replication step zrepl needs to update its ZFS abstractions through the ``zfs`` command which often waits multiple seconds for the zpool to sync.
Thus, if the actual send & recv time of a step is small compared to the time spent on zrepl ZFS abstractions then increasing step execution concurrency will result in a lower overall turnaround time.
docs/configuration/sendrecvoptions.rst
+246 -9
View File
@@ -9,22 +9,65 @@ Send & Recv Options
Send Options
~~~~~~~~~~~~
:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
::
jobs:
- type: push
filesystems: ...
send:
encrypted: true
# flags from the table below go here
...
:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
The following table specifies the list of (boolean) options.
Flags with an entry in the ``zfs send`` column map directly to the zfs send CLI flags.
zrepl does not perform feature checks for these flags.
If you enable a flag that is not supported by the installed version of ZFS, the zfs error will show up at runtime in the logs and zrepl status.
See the `upstream man page <https://openzfs.github.io/openzfs-docs/man/8/zfs-send.8.html>`_ (``man zfs-send``) for their semantics.
``encryption`` option
---------------------
.. list-table::
:widths: 20 10 70
:header-rows: 1
The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
More specifically, if ``encryption=true``, zrepl
* - ``send.``
- ``zfs send``
- Comment
* - ``encrypted``
-
- Specific to zrepl, :ref:`see below <job-send-options-encrypted>`.
* - ``bandwidth_limit``
-
- Specific to zrepl, :ref:`see below <job-send-recv-options--bandwidth-limit>`.
* - ``raw``
- ``-w``
- Use ``encrypted`` to only allow encrypted sends. Mixed sends are not supported.
* - ``send_properties``
- ``-p``
- **Be careful**, read the :ref:`note on property replication below <job-note-property-replication>`.
* - ``backup_properties``
- ``-b``
- **Be careful**, read the :ref:`note on property replication below <job-note-property-replication>`.
* - ``large_blocks``
- ``-L``
- **Potential data loss on OpenZFS < 2.0**, see :ref:`warning below <job-send-options-large-blocks>`.
* - ``compressed``
- ``-c``
-
* - ``embedded_data``
- ``-e``
-
* - ``saved``
- ``-S``
-
.. _job-send-options-encrypted:
``encrypted``
-------------
The ``encrypted`` option controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
More specifically, if ``encrypted=true``, zrepl
* checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and
* invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and
@@ -32,14 +75,208 @@ More specifically, if ``encryption=true``, zrepl
Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages.
If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
If ``encrypted=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
.. NOTE::
Use ``encrypted`` instead of ``raw`` to make your intent clear that zrepl must only replicate filesystems that are actually encrypted by OpenZFS native encryption.
It is meant as a safeguard to prevent unintended sends of unencrypted filesystems in raw mode.
.. _job-send-options-properties:
``properties``
--------------
Sends the dataset properties along with snapshots.
Please be careful with this option and read the :ref:`note on property replication below <job-note-property-replication>`.
.. _job-send-options-backup-properties:
``backup_properties``
---------------------
When properties are modified on a filesystem that was received from a send stream with ``send.properties=true``, ZFS archives the original received value internally.
This also applies to :ref:`inheriting or overriding properties during zfs receive <job-recv-options--inherit-and-override>`.
When sending those received filesystems another hop, the ``backup_properties`` flag instructs ZFS to send the original property values rather than the current locally set values.
This is useful for replicating properties across multiple levels of backup machines.
**Example:**
Suppose we want to flow snapshots from Machine A to B, then from B to C.
A will enable the :ref:`properties send option <job-send-options-properties>`.
B will want to override :ref:`critical properties such as mountpoint or canmount <job-note-property-replication>`.
But the job that replicates from B to C should be sending the original property values received from A.
Thus, B sets the ``backup_properties`` option.
Please be careful with this option and read the :ref:`note on property replication below <job-note-property-replication>`.
.. _job-send-options-large-blocks:
``large_blocks``
----------------
This flag should not be changed after initial replication.
Prior to `OpenZFS commit 7bcb7f08 <https://github.com/openzfs/zfs/pull/10383/files#diff-4c1e47568f46fb63546e984943b09a3e6b051e2242649523f7835bbdfe2a9110R337-R342>`_
it was possible to change this setting which resulted in **data loss on the receiver**.
The commit in question is included in OpenZFS 2.0 and works around the problem by prohibiting receives of incremental streams with a flipped setting.
.. WARNING::
This bug has **not been fixed in the OpenZFS 0.8 releases** which means that changing this flag after initial replication might cause **data loss** on the receiver.
.. _job-recv-options:
Recv Options
~~~~~~~~~~~~
:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section.
However, there are currently no variables to configure there.
:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section:
::
jobs:
- type: pull
recv:
properties:
inherit:
- "mountpoint"
override: {
"org.openzfs.systemd:ignore": "on"
}
bandwidth_limit: ...
placeholder:
encryption: unspecified | off | inherit
...
Jump to
:ref:`properties <job-recv-options--inherit-and-override>` ,
:ref:`bandwidth_limit <job-send-recv-options--bandwidth-limit>` , and
:ref:`placeholder <job-recv-options--placeholder>`.
.. _job-recv-options--inherit-and-override:
``properties``
--------------
``override`` maps directly to the `zfs recv -o flag <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_.
Property name-value pairs specified in this map will apply to all received filesystems, regardless of whether the send stream contains properties or not.
``inherit`` maps directly to the `zfs recv -x flag <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_.
Property names specified in this list will be inherited from the receiving side's parent filesystem (e.g. ``root_fs``).
With both options, the sending side's property value is still stored on the receiver, but the local override or inherit is the one that takes effect.
You can send the original properties from the first receiver to another receiver using :ref:`send.backup_properties<job-send-options-backup-properties>`.
.. _job-note-property-replication:
A Note on Property Replication
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If a send stream contains properties, as per ``send.properties`` or ``send.backup_properties``,
the default ZFS behavior is to use those properties on the receiving side, verbatim.
In many use cases for zrepl, this can have devastating consequences.
For example, when backing up a filesystem that has ``mountpoint=/`` to a storage server,
that storage server's root filesystem will be shadowed by the received file system on some platforms.
Also, many scripts and tools use ZFS user properties for configuration and do not check the property source (``local`` vs. ``received``).
If they are installed on the receiving side as well as the sending side, property replication could have unintended effects.
**zrepl currently does not provide any automatic safe-guards for property replication:**
* Make sure to read the entire man page on zfs recv (`man zfs recv <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_) before enabling this feature.
* Use ``recv.properties.override`` whenever possible, e.g. for ``mountpoint=none`` or ``canmount=off``.
* Use ``recv.properties.inherit`` if that makes more sense to you.
Below is an **non-exhaustive list of problematic properties**.
Please open a pull request if you find a property that is missing from this list.
(Both with regards to core ZFS tools and other software in the broader ecosystem.)
Mount behaviour
---------------
* ``mountpoint``
* ``canmount``
* ``overlay``
Note: inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
This is an `issue in OpenZFS <https://github.com/openzfs/zfs/issues/11416>`_ .
As a workaround, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.
Please comment at zrepl :issue:`430` if you encounter this issue and/or would like zrepl to automatically work around it.
Systemd
-------
With systemd, you should also consider the properties processed by the `zfs-mount-generator <https://manpages.debian.org/buster-backports/zfsutils-linux/zfs-mount-generator.8.en.html>`_ .
Most notably:
* ``org.openzfs.systemd:ignore``
* ``org.openzfs.systemd:wanted-by``
* ``org.openzfs.systemd:required-by``
Encryption
----------
If the sender filesystems are encrypted but the sender does :ref:`plain sends <zfs-background-knowledge-plain-vs-raw-sends>`
and property replication is enabled, the receiver must :ref:`inherit the following properties<job-recv-options--inherit-and-override>`:
* ``keylocation``
* ``keyformat``
* ``encryption``
.. _job-recv-options--placeholder:
Placeholders
~~~~~~~~~~~~
::
placeholder:
encryption: unspecified | off | inherit
During replication, zrepl :ref:`creates placeholder datasets <replication-placeholder-property>` on the receiving side if the sending side's ``filesystems`` filter creates gaps in the dataset hierarchy.
This is generally fully transparent to the user.
However, with OpenZFS Native Encryption, placeholders require zrepl user attention.
Specifically, the problem is that, when zrepl attempts to create the placeholder dataset on the receiver, and that placeholder's parent dataset is encrypted, ZFS wants to inherit encryption to the placeholder.
This is relevant to two use cases that zrepl supports:
1. **encrypted-send-to-untrusted-receiver** In this use case, the sender sends an :ref:`encrypted send stream <job-send-options-encrypted>` and the receiver doesn't have the key loaded.
2. **send-plain-encrypt-on-receive** The receive-side ``root_fs`` dataset is encrypted, and the senders are unencrypted.
The key of ``root_fs`` is loaded, and the goal is that the plain sends (e.g., from production) are encrypted on-the-fly during receive, with ``root_fs``'s key.
For **encrypted-send-to-untrusted-receiver**, the placeholder datasets need to be created with ``-o encryption=off``.
Without it, creation would fail with an error, indicating that the placeholder's parent dataset's key needs to be loaded.
But we don't trust the receiver, so we can't expect that to ever happen.
However, for **send-plain-encrypt-on-receive**, we cannot set ``-o encryption=off``.
The reason is that if we did, any of the (non-placeholder) child datasets below the placeholder would inherit ``encryption=off``, thereby silently breaking our encrypt-on-receive use case.
So, to cover this use case, we need to create placeholders without specifying ``-o encryption``.
This will make ``zfs create`` inherit the encryption mode from the parent dataset, and thereby transitively from ``root_fs``.
The zrepl config provides the `recv.placeholder.encryption` knob to control this behavior.
In ``undefined`` mode (default), placeholder creation bails out and asks the user to configure a behavior.
In ``off`` mode, the placeholder is created with ``encryption=off``, i.e., **encrypted-send-to-untrusted-rceiver** use case.
In ``inherit`` mode, the placeholder is created without specifying ``-o encryption`` at all, i.e., the **send-plain-encrypt-on-receive** use case.
Common Options
~~~~~~~~~~~~~~
.. _job-send-recv-options--bandwidth-limit:
Bandwidth Limit (send & recv)
-----------------------------
::
bandwidth_limit:
max: 23.5 MiB # -1 is the default and disabled rate limiting
bucket_capacity: # token bucket capacity in bytes; defaults to 128KiB
Both ``send`` and ``recv`` can be limited to a maximum bandwidth through ``bandwidth_limit``.
For most users, it should be sufficient to just set ``bandwidth_limit.max``.
The ``bandwidth_limit.bucket_capacity`` refers to the `token bucket size <https://github.com/juju/ratelimit>`_.
The bandwidth limit only applies to the payload data, i.e., the ZFS send stream.
It does not account for transport protocol overheads.
The scope is the job level, i.e., all :ref:`concurrent <replication-option-concurrency>` sends or incoming receives of a job share the bandwidth limit.
docs/configuration/snapshotting.rst
+85 -21
View File
@@ -5,53 +5,117 @@
Taking Snaphots
===============
The ``push``, ``source`` and ``snap`` jobs can automatically take periodic snapshots of the filesystems matched by the ``filesystems`` filter field.
The snapshot names are composed of a user-defined prefix followed by a UTC date formatted like ``20060102_150405_000``.
You can configure zrepl to take snapshots of the filesystems in the ``filesystems`` field specified in ``push``, ``source`` and ``snap`` jobs.
The following snapshotting types are supported:
.. list-table::
:widths: 20 70
:header-rows: 1
* - ``snapshotting.type``
- Comment
* - ``periodic``
- Ensure that snapshots are taken at a particular interval.
* - ``cron``
- Use cron spec to take snapshots at particular points in time.
* - ``manual``
- zrepl does not take any snapshots by itself.
The ``periodic`` and ``cron`` snapshotting types share some common options and behavior:
* **Naming:** The snapshot names are composed of a user-defined ``prefix`` followed by a UTC date formatted like ``20060102_150405_000``.
We use UTC because it will avoid name conflicts when switching time zones or between summer and winter time.
* **Hooks:** You can configure hooks to run before or after zrepl takes the snapshots. See :ref:`below <job-snapshotting-hooks>` for details.
* **Push replication:** After creating all snapshots, the snapshotter will wake up the replication part of the job, if it's a ``push`` job.
Note that snapshotting is decoupled from replication, i.e., if it is down or takes too long, snapshots will still be taken.
Note further that other jobs are not woken up by snapshotting.
When a job is started, the snapshotter attempts to get the snapshotting rhythms of the matched ``filesystems`` in sync because snapshotting all filesystems at the same time results in a more consistent backup.
To find that sync point, the most recent snapshot, made by the snapshotter, in any of the matched ``filesystems`` is used.
A filesystem that does not have snapshots by the snapshotter has lower priority than filesystem that do, and thus might not be snapshotted (and replicated) until it is snapshotted at the next sync point.
.. NOTE::
For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
There is **no concept of ownership** of the snapshots that are created by ``periodic`` or ``cron``.
Thus, there is no distinction between zrepl-created snapshots and user-created snapshots during replication or pruning.
Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
In particular, pruning will take all snapshots into consideration by default.
To constrain pruning to just zrepl-created snapshots:
1. Assign a unique `prefix` to the snapshotter and
2. Use the ``regex`` functionality of the various pruning ``keep`` rules to just consider snapshots with that prefix.
There is currently no way to constrain replication to just zrepl-created snapshots.
Follow and comment at :issue:`403` if you need this functionality.
.. NOTE::
The ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
``periodic`` Snapshotting
-------------------------
::
jobs:
- type: push
filesystems: {
"<": true,
"tmp": false
}
- ...
filesystems: { ... }
snapshotting:
type: periodic
prefix: zrepl_
interval: 10m
hooks: ...
...
pruning: ...
There is also a ``manual`` snapshotting type, which covers the following use cases:
The ``periodic`` snapshotter ensures that snapshots are taken in the specified ``interval``.
If you use zrepl for backup, this translates into your recovery point objective (RPO).
To meet your RPO, you still need to monitor that replication, which happens asynchronously to snapshotting, actually works.
* Existing infrastructure for automatic snapshots: you only want to use this zrepl job for replication.
* Handling snapshotting through a separate ``snap`` job.
It is desirable to get all ``filesystems`` snapshotted simultaneously because it results in a more consistent backup.
To accomplish this while still maintaining the ``interval``, the ``periodic`` snapshotter attempts to get the snapshotting rhythms in sync.
To find that sync point, the most recent snapshot, created by the snapshotter, in any of the matched ``filesystems`` is used.
A filesystem that does not have snapshots by the snapshotter has lower priority than filesystem that do, and thus might not be snapshotted (and replicated) until it is snapshotted at the next sync point.
The snapshotter uses the ``prefix`` to identify which snapshots it created.
Note that you will have to trigger replication manually using the ``zrepl signal wakeup JOB`` subcommand in that case.
.. _job-snapshotting--cron:
``cron`` Snapshotting
---------------------
::
jobs:
- type: snap
filesystems: { ... }
snapshotting:
type: cron
prefix: zrepl_
# (second, optional) minute hour day-of-month month day-of-week
# This example takes snapshots daily at 3:00.
cron: "0 3 * * *"
pruning: ...
In ``cron`` mode, the snapshotter takes snaphots at fixed points in time.
See https://en.wikipedia.org/wiki/Cron for details on the syntax.
zrepl uses the ``the github.com/robfig/cron/v3`` Go package for parsing.
An optional field for "seconds" is supported to take snapshots at sub-minute frequencies.
``manual`` Snapshotting
-----------------------
::
jobs:
- type: push
filesystems: {
"<": true,
"tmp": false
}
snapshotting:
type: manual
...
In ``manual`` mode, zrepl does not take snapshots by itself.
Manual snapshotting is most useful if you have existing infrastructure for snapshot management.
Or, if you want to decouple snapshot management from replication using a zrepl ``snap`` job.
See :ref:`this quickstart guide <quickstart-backup-to-external-disk>` for an example.
To trigger replication after taking snapshots, use the ``zrepl signal wakeup JOB`` command.
.. _job-snapshotting-hooks:
Pre- and Post-Snapshot Hooks
docs/global.rst.inc
+4 -3
View File
@@ -5,7 +5,7 @@
.. |GitHub license| image:: https://img.shields.io/github/license/zrepl/zrepl.svg
:target: https://github.com/zrepl/zrepl/blob/master/LICENSE
.. |Language: Go| image:: https://img.shields.io/badge/language-Go-6ad7e5.svg
.. |Language: Go| image:: https://img.shields.io/badge/lang-Go-6ad7e5.svg
:target: https://golang.org/
.. |User Docs| image:: https://img.shields.io/badge/docs-web-blue.svg
:target: https://zrepl.github.io
@@ -13,13 +13,14 @@
:target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
.. |Donate via Liberapay| image:: https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay
:target: https://liberapay.com/zrepl/donate
.. |Donate via Patreon| image:: https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079
.. |Donate via Patreon| image:: https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079
:target: https://www.patreon.com/zrepl
.. |Donate via GitHub Sponsors| image:: https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow
:target: https://github.com/sponsors/problame
.. |Twitter| image:: https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social
:target: https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl
.. |Matrix| image:: https://img.shields.io/badge/chat-matrix-blue.svg
:target: https://matrix.to/#/#zrepl:matrix.org
.. |serve-transport| replace:: :ref:`serve specification<transport>`
.. |connect-transport| replace:: :ref:`connect specification<transport>`
docs/index.rst
+10 -4
View File
@@ -5,7 +5,7 @@
.. include:: global.rst.inc
|GitHub license| |Language: Go| |Twitter| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
|GitHub license| |Language: Go| |Twitter| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal| |Matrix|
zrepl - ZFS replication
@@ -25,10 +25,10 @@ zrepl - ZFS replication
Progress: [=========================\----] 246.7 MiB / 264.7 MiB @ 11.5 MiB/s
zroot STEPPING (step 1/2, 624 B/1.2 KiB) next: @a => @b
zroot/ROOT DONE (step 2/2, 1.2 KiB/1.2 KiB)
zroot/ROOT/default STEPPING (step 1/2, 123.4 MiB/129.3 MiB) next: @a => @b
* zroot/ROOT/default STEPPING (step 1/2, 123.4 MiB/129.3 MiB) next: @a => @b
zroot/tmp STEPPING (step 1/2, 29.9 KiB/44.2 KiB) next: @a => @b
zroot/usr STEPPING (step 1/2, 624 B/1.2 KiB) next: @a => @b
zroot/usr/home STEPPING (step 1/2, 123.3 MiB/135.3 MiB) next: @a => @b
* zroot/usr/home STEPPING (step 1/2, 123.3 MiB/135.3 MiB) next: @a => @b
zroot/var STEPPING (step 1/2, 624 B/1.2 KiB) next: @a => @b
zroot/var/audit DONE (step 2/2, 1.2 KiB/1.2 KiB)
zroot/var/crash DONE (step 2/2, 1.2 KiB/1.2 KiB)
@@ -61,7 +61,12 @@ Main Features
* [x] Automatic ZFS holds during send & receive
* [x] Automatic bookmark \& hold management for guaranteed incremental send & recv
* [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption)
* [ ] Compressed send & receive
* [x] Properties send & receive
* [x] Compressed send & receive
* [x] Large blocks send & receive
* [x] Embedded data send & receive
* [x] Resume state send & receive
* [x] Bandwidth limiting
* **Automatic snapshot management**
@@ -132,5 +137,6 @@ Table of Contents
pr
changelog
GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>
Chat: Matrix <https://matrix.to/#/#zrepl:matrix.org>
supporters
docs/installation/apt-repos.rst
+20 -9
View File
@@ -9,18 +9,29 @@ The fingerprint of the signing key is ``E101 418F D3D6 FBCB 9D65 A62D 7086 99FC
It is available at `<https://zrepl.cschwarz.com/apt/apt-key.asc>`_ .
Please open an issue in on GitHub if you encounter any issues with the repository.
The following snippet configure the repository for your Debian or Ubuntu release:
::
apt update && apt install curl gnupg lsb-release; \
ARCH="$(dpkg --print-architecture)"; \
CODENAME="$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') $(lsb_release -c -s | tr '[:upper:]' '[:lower:]')"; \
echo "Using Distro and Codename: $CODENAME"; \
(curl https://zrepl.cschwarz.com/apt/apt-key.asc | apt-key add -) && \
(echo "deb [arch=$ARCH] https://zrepl.cschwarz.com/apt/$CODENAME main" > /etc/apt/sources.list.d/zrepl.list) && \
apt update
(
set -ex
zrepl_apt_key_url=https://zrepl.cschwarz.com/apt/apt-key.asc
zrepl_apt_key_dst=/usr/share/keyrings/zrepl.gpg
zrepl_apt_repo_file=/etc/apt/sources.list.d/zrepl.list
# Install dependencies for subsequent commands
sudo apt update && sudo apt install curl gnupg lsb-release
# Deploy the zrepl apt key.
curl -fsSL "$zrepl_apt_key_url" | tee | gpg --dearmor | sudo tee "$zrepl_apt_key_dst" > /dev/null
# Add the zrepl apt repo.
ARCH="$(dpkg --print-architecture)"
CODENAME="$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') $(lsb_release -c -s | tr '[:upper:]' '[:lower:]')"
echo "Using Distro and Codename: $CODENAME"
echo "deb [arch=$ARCH signed-by=$zrepl_apt_key_dst] https://zrepl.cschwarz.com/apt/$CODENAME main" | sudo tee /etc/apt/sources.list.d/zrepl.list
# Update apt repos.
sudo apt update
)
.. NOTE::
docs/installation/freebsd-jail-with-iocage.rst
+12
View File
@@ -93,6 +93,7 @@ Enable the zrepl daemon to start automatically at boot:
sysrc zrepl_enable="YES"
Now jump to :ref:`the summary <installation-freebsd-jail-summary>` below.
Plugin
######
@@ -134,7 +135,18 @@ Now ``zrepl`` can be started.
service zrepl start
Now jump to :ref:`the summary <installation-freebsd-jail-summary>` below.
.. _installation-freebsd-jail-summary:
Summary
-------
Congratulations, you have a working jail!
.. NOTE::
With FreeBSD 13's transition to OpenZFS 2.0, please ensure that your jail's FreeBSD version matches the one in the kernel module.
If you are getting cryptic errors such as
``cannot receive new filesystem stream: invalid backup stream``
the instructions posted `here <https://github.com/zrepl/zrepl/issues/500#issuecomment-966215205>`_ might help.
docs/publish.sh
+11 -8
View File
@@ -1,6 +1,5 @@
#!/bin/bash
set -eo pipefail
set -euo pipefail
NON_INTERACTIVE=false
DO_CLONE=false
@@ -13,7 +12,7 @@ while getopts "ca" arg; do
DO_CLONE=true
;;
*)
echo invalid option
echo "invalid option '-$arg'"
exit 1
;;
esac
@@ -27,11 +26,6 @@ checkout_repo_msg() {
echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
}
exit_msg() {
echo "error, exiting..."
}
trap exit_msg EXIT
if ! type sphinx-versioning >/dev/null; then
echo "install sphinx-versioning and come back"
exit 1
@@ -71,6 +65,9 @@ git reset --hard origin/master
echo "cleaning GitHub pages repo"
git rm -rf .
cat > .gitignore <<EOF
**/.doctrees
EOF
popd
@@ -96,7 +93,13 @@ COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_CO
pushd "$PUBLICDIR"
echo "adding and commiting all changes in GitHub pages repo"
git add .gitignore
git add -A
if [ "$(git status --porcelain)" != "" ]; then
git commit -m "$COMMIT_MSG"
else
echo "nothing to commit"
fi
echo "pushing to GitHub pages repo"
git push origin master
docs/quickstart.rst
+14
View File
@@ -33,6 +33,7 @@ Keep the :ref:`full config documentation <configuration_toc>` handy if a config
quickstart/continuous_server_backup
quickstart/backup_to_external_disk
quickstart/fan_out_replication
Use ``zrepl configcheck`` to validate your configuration.
No output indicates that everything is fine.
@@ -51,6 +52,19 @@ We hope that you have found a configuration that fits your use case.
Use ``zrepl configcheck`` once again to make sure the config is correct (output indicates that everything is fine).
Then restart the zrepl daemon on all systems involved in the replication, likely using ``service zrepl restart`` or ``systemctl restart zrepl``.
.. WARNING::
Please :ref:`read up carefully <prune>` on the pruning rules before applying the config.
In particular, note that most example configs apply to all snapshots, not just zrepl-created snapshots.
Use the following keep rule on sender and receiver to prevent this:
::
- type: regex
negate: true
regex: "^zrepl_.*" # <- the 'prefix' specified in snapshotting.prefix
Watch it Work
=============
docs/quickstart/backup_to_external_disk.rst
+1 -1
View File
@@ -13,7 +13,7 @@ Thus, we only keep one hour worth of high-resolution snapshots, then fade them o
At the end of each work day, we connect our external disk that serves as our workstation's local offline backup.
We want zrepl to inspect the filesystems and snapshots on the external pool, figure out which snapshots were created since the last time we connected the external disk, and use incremental replication to efficiently mirror our workstation to our backup disk.
Afterwards, we want to clean up old snapshots on the backup pool: we want to keep all snapshots younger than one hour, 24 for each hor of the first day, then 360 daily backups.
Afterwards, we want to clean up old snapshots on the backup pool: we want to keep all snapshots younger than one hour, 24 for each hour of the first day, then 360 daily backups.
A few additional requirements:
docs/quickstart/continuous_server_backup.rst
+5 -5
View File
@@ -9,13 +9,13 @@ This config example shows how we can backup our ZFS-based server to another mach
* Production server ``prod`` with filesystems to back up:
* ``zroot/var/db``
* ``zroot/usr/home`` and all its child filesystems
* **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves
* The entire pool ``zroot``
* except ``zroot/var/tmp`` and all child datasets of it
* and except ``zroot/usr/home/paranoid`` which belongs to a user doing backups themselves.
* Backup server ``backups`` with
* Backup server ``backups`` with a dataset sub-tree for use by zrepl:
* Filesystem ``storage/zrepl/sink/prod`` + children dedicated to backups of ``prod``
* In our example, that will be ``storage/zrepl/sink/prod``.
Our backup solution should fulfill the following requirements:
docs/quickstart/fan_out_replication.rst
+93
View File
@@ -0,0 +1,93 @@
.. include:: ../global.rst.inc
.. _quickstart-fan-out-replication:
Fan-out replication
===================
This quick-start example demonstrates how to implement a fan-out replication setup where datasets on a server (A) are replicated to multiple targets (B, C, etc.).
This example uses multiple ``source`` jobs on server A and ``pull`` jobs on the target servers.
.. WARNING::
Before implementing this setup, please see the caveats listed in the :ref:`fan-out replication configuration overview <fan-out-replication>`.
Overview
--------
On the source server (A), there should be:
* A ``snap`` job
* Creates the snapshots
* Handles the pruning of snapshots
* A ``source`` job for target B
* Accepts connections from server B and B only
* Further ``source`` jobs for each additional target (C, D, etc.)
* Listens on a unique port
* Only accepts connections from the specific target
On each target server, there should be:
* A ``pull`` job that connects to the corresponding ``source`` job on A
* ``prune_sender`` should keep all snapshots since A's ``snap`` job handles the pruning
* ``prune_receiver`` can be configured as appropriate on each target server
Generate TLS Certificates
-------------------------
Mutual TLS via the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` can be used to secure the connections between the servers. In this example, a self-signed certificate is created for each server without setting up a CA.
.. code-block:: bash
source=a.example.com
targets=(
b.example.com
c.example.com
# ...
)
for server in "${source}" "${targets[@]}"; do
openssl req -x509 -sha256 -nodes \
-newkey rsa:4096 \
-days 365 \
-keyout "${server}.key" \
-out "${server}.crt" \
-addext "subjectAltName = DNS:${server}" \
-subj "/CN=${server}"
done
# Distribute each host's keypair
for server in "${source}" "${targets[@]}"; do
ssh root@"${server}" mkdir /etc/zrepl
scp "${server}".{crt,key} root@"${server}":/etc/zrepl/
done
# Distribute target certificates to the source
scp "${targets[@]/%/.crt}" root@"${source}":/etc/zrepl/
# Distribute source certificate to the targets
for server in "${targets[@]}"; do
scp "${source}.crt" root@"${server}":/etc/zrepl/
done
Configure source server A
-------------------------
.. literalinclude:: ../../config/samples/quickstart_fan_out_replication_source.yml
Configure each target server
----------------------------
.. literalinclude:: ../../config/samples/quickstart_fan_out_replication_target.yml
Go Back To Quickstart Guide
---------------------------
:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
docs/requirements.txt
+1 -1
View File
@@ -25,6 +25,6 @@ sphinxcontrib-htmlhelp==1.0.2
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.2
sphinxcontrib-serializinghtml==1.1.3
-e git://github.com/rwblair/sphinxcontrib-versioning.git@7e3885a389a809e17ea55261316b7b0e98dbf98f#egg=sphinxcontrib-versioning
git+https://github.com/rwblair/sphinxcontrib-versioning.git@7e3885a389a809e17ea55261316b7b0e98dbf98f#egg=sphinxcontrib-versioning
sphinxcontrib-websupport==1.1.2
urllib3==1.25.3
docs/supporters.rst
+2
View File
@@ -32,6 +32,8 @@ We would like to thank the following people and organizations for supporting zre
<div class="fa fa-code" style="width: 1em;"></div>
* |supporter-gold| Prominic.NET, Inc.
* |supporter-std| Torsten Blum
* |supporter-gold| Cyberiada GmbH
* |supporter-std| `Gordon Schulz <https://github.com/azmodude>`_
* |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
docs/usage.rst
+71 -1
View File
@@ -26,7 +26,7 @@ CLI Overview
* - ``zrepl daemon``
- run the daemon, required for all zrepl functionality
* - ``zrepl status``
- show job activity, or with ``--raw`` for JSON output
- show job activity, or with ``--mode raw`` for JSON output
* - ``zrepl stdinserver``
- see :ref:`transport-ssh+stdinserver`
* - ``zrepl signal wakeup JOB``
@@ -78,3 +78,73 @@ Systemd Unit File
A systemd service definition template is available in :repomasterlink:`dist/systemd`.
Note that some of the options only work on recent versions of systemd.
Any help & improvements are very welcome, see :issue:`145`.
============
Ops Runbooks
============
.. toctree::
usage/runbooks/migrating_sending_side_to_new_zpool.rst
.. _usage-platform-tests:
==============
Platform Tests
==============
Along with the main ``zrepl`` binary, we release the ``platformtest`` binaries.
The zrepl platform tests are an integration test suite that is complementary to the pure Go unit tests.
Any test that needs to interact with ZFS is a platform test.
The platform need to run as root.
For each test, we create a fresh dummy zpool backed by a file-based vdev.
The file path, and a root mountpoint for the dummy zpool, must be specified on the command line:
::
mkdir -p /tmp/zreplplatformtest
./platformtest \
-poolname 'zreplplatformtest' \ # <- name must contain zreplplatformtest
-imagepath /tmp/zreplplatformtest.img \ # <- zrepl will create the file
-mountpoint /tmp/zreplplatformtest # <- must exist
.. WARNING::
``platformtest`` will unconditionally overwrite the file at `imagepath`
and unconditionally ``zpool destroy $poolname``.
So, don't use a production poolname, and consider running the test in a VM.
It'll be a lot faster as well because the underlying operations, ``zfs list`` in particular, will be faster.
While the platformtests are running, there will be a log of log output.
After all tests have run, it prints a summary with a list of tests, grouped by result type (success, failure, skipped):
::
PASSING TESTS:
github.com/zrepl/zrepl/platformtest/tests.BatchDestroy
github.com/zrepl/zrepl/platformtest/tests.CreateReplicationCursor
github.com/zrepl/zrepl/platformtest/tests.GetNonexistent
github.com/zrepl/zrepl/platformtest/tests.HoldsWork
...
github.com/zrepl/zrepl/platformtest/tests.SendStreamNonEOFReadErrorHandling
github.com/zrepl/zrepl/platformtest/tests.UndestroyableSnapshotParsing
SKIPPED TESTS:
github.com/zrepl/zrepl/platformtest/tests.SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false
FAILED TESTS: []
If there is a failure, or a skipped test that you believe should be passing, re-run the test suite, capture stderr & stdout to a text file, and create an issue on GitHub.
To run a specific test case, or a subset of tests matched by regex, use the ``-run REGEX`` command line flag.
To stop test execution at the first failing test, and prevent cleanup of the dummy zpool, use the ``-failure.stop-and-keep-pool`` flag.
To build the platformtests yourself, use ``make test-platform-bin``.
There's also the ``make test-platform`` target to run the platform tests with a default command line.
docs/usage/runbooks/migrating_sending_side_to_new_zpool.rst
+36
View File
@@ -0,0 +1,36 @@
Migrating Sending Side
~~~~~~~~~~~~~~~~~~~~~~
**Objective**:
Move sending-side zpool to new hardware.
Make the move fully transparent to the sending-side jobs.
After the move is done, all sending-side zrepl jobs should continue to work as if the move had not happened.
In particular, incremental replication should be able to pick up where it left before the move.
Suppose we want to migrate all data from one zpool ``oldpool`` to another zpool ``newpool``.
A possible reason might be that we want to change RAID levels, ``ashift``, or just migrate over to next-gen hardware.
If the pool names are different, zrepl's matching between sender and receiver dataset will break becase the receive-side dataset names contain ``oldpool``.
To avoid this, we will need the name of the new pool to match that of the old pool.
The following steps will accomplish this:
1. Stop zrepl.
2. Create the new pool: ``zpool create newpool ...``
3. Take a snapshot of the old pool so that you have something that you can ``zfs send``.
For example, run ``zfs snapshot -r oldpool@migration_oldpool_newpool``.
4. Send all of the oldpool's datasets to the new pool:
``zfs send -R oldpool@migration_oldpool_newpool | zfs recv -F newpool``
5. Export the old pool: ``zpool export oldpool``
6. Export the new pool: ``zpool export newpool``
7. (Optional) Change the name of the old pool to something that does not conflict with the new pool.
We are going to use the name ``oldoldpool`` in this example.
Use ``zpool import`` with no arguments to see the pool id.
Then ``zpool import <id> oldoldpool && zpool export oldoldpool``.
8. Import the new pool, while changing the name to match the old pool: ``zpool import newpool oldpool``
9. Start zrepl again and wake up the relevant jobs.
10. Use ``zrepl status`` or you monitoring to ensure that replication works.
The best test is an end-to-end test where you write some junk data on a sender dataset and wait until a snapshot with that data appears on the receiving side.
11. Once you are confident that replication is working, you may dispose of the old pool.
Note that, depending on pruning rules, it will not be possible to switch back to the old pool seamlessly, i.e., without a full re-replication.
endpoint/endpoint.go
+217 -97
View File
@@ -14,45 +14,67 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/util/bandwidthlimit"
"github.com/zrepl/zrepl/util/chainedio"
"github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/semaphore"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
type SenderConfig struct {
FSF zfs.DatasetFilter
Encrypt *zfs.NilBool
JobID JobID
Encrypt *nodefault.Bool
SendRaw bool
SendProperties bool
SendBackupProperties bool
SendLargeBlocks bool
SendCompressed bool
SendEmbeddedData bool
SendSaved bool
BandwidthLimit bandwidthlimit.Config
}
func (c *SenderConfig) Validate() error {
c.JobID.MustValidate()
if err := c.Encrypt.Validate(); err != nil {
if err := c.Encrypt.ValidateNoDefault(); err != nil {
return errors.Wrap(err, "`Encrypt` field invalid")
}
if _, err := StepHoldTag(c.JobID); err != nil {
return fmt.Errorf("JobID cannot be used for hold tag: %s", err)
}
if err := bandwidthlimit.ValidateConfig(c.BandwidthLimit); err != nil {
return errors.Wrap(err, "`BandwidthLimit` field invalid")
}
return nil
}
// Sender implements replication.ReplicationEndpoint for a sending side
type Sender struct {
pdu.UnsafeReplicationServer // prefer compilation errors over default 'method X not implemented' impl
FSFilter zfs.DatasetFilter
encrypt *zfs.NilBool
jobId JobID
config SenderConfig
bwLimit bandwidthlimit.Wrapper
}
func NewSender(conf SenderConfig) *Sender {
if err := conf.Validate(); err != nil {
panic("invalid config" + err.Error())
}
ratelimiter := bandwidthlimit.WrapperFromConfig(conf.BandwidthLimit)
return &Sender{
FSFilter: conf.FSF,
encrypt: conf.Encrypt,
jobId: conf.JobID,
config: conf,
bwLimit: ratelimiter,
}
}
@@ -83,15 +105,10 @@ func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq)
}
rfss := make([]*pdu.Filesystem, len(fss))
for i := range fss {
encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, fss[i].ToString())
if err != nil {
return nil, errors.Wrap(err, "cannot get filesystem encryption status")
}
rfss[i] = &pdu.Filesystem{
Path: fss[i].ToString(),
// ResumeToken does not make sense from Sender
IsPlaceholder: false, // sender FSs are never placeholders
IsEncrypted: encEnabled,
}
}
res := &pdu.ListFilesystemRes{Filesystems: rfss}
@@ -118,9 +135,6 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
}
var maxConcurrentZFSSend = envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10)
var maxConcurrentZFSSendSemaphore = semaphore.New(maxConcurrentZFSSend)
func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
if fsv == nil {
return nil
@@ -140,74 +154,44 @@ func sendArgsFromPDUAndValidateExistsAndGetVersion(ctx context.Context, fs strin
return version, nil
}
func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
func (s *Sender) sendMakeArgs(ctx context.Context, r *pdu.SendReq) (sendArgs zfs.ZFSSendArgsValidated, _ error) {
_, err := s.filterCheckFS(r.Filesystem)
if err != nil {
return nil, nil, err
}
switch r.Encrypted {
case pdu.Tri_DontCare:
// use s.encrypt setting
// ok, fallthrough outer
case pdu.Tri_False:
if s.encrypt.B {
return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
}
// fallthrough outer
case pdu.Tri_True:
if !s.encrypt.B {
return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
}
// fallthrough outer
default:
return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
return sendArgs, err
}
sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
FS: r.Filesystem,
From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
Encrypted: s.encrypt,
ZFSSendFlags: zfs.ZFSSendFlags{
ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
Encrypted: s.config.Encrypt,
Properties: s.config.SendProperties,
BackupProperties: s.config.SendBackupProperties,
Raw: s.config.SendRaw,
LargeBlocks: s.config.SendLargeBlocks,
Compressed: s.config.SendCompressed,
EmbeddedData: s.config.SendEmbeddedData,
Saved: s.config.SendSaved,
},
}
sendArgs, err := sendArgsUnvalidated.Validate(ctx)
sendArgs, err = sendArgsUnvalidated.Validate(ctx)
if err != nil {
return nil, nil, errors.Wrap(err, "validate send arguments")
return sendArgs, errors.Wrap(err, "validate send arguments")
}
return sendArgs, nil
}
getLogger(ctx).Debug("acquire concurrent send semaphore")
// TODO use try-acquire and fail with resource-exhaustion rpc status
// => would require handling on the client-side
// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
guard, err := maxConcurrentZFSSendSemaphore.Acquire(ctx)
func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
sendArgs, err := s.sendMakeArgs(ctx, r)
if err != nil {
return nil, nil, err
}
defer guard.Release()
si, err := zfs.ZFSSendDry(ctx, sendArgs)
if err != nil {
return nil, nil, errors.Wrap(err, "zfs send dry failed")
}
// From now on, assume that sendArgs has been validated by ZFSSendDry
// (because validation involves shelling out, it's actually a little expensive)
var expSize int64 = 0 // protocol says 0 means no estimate
if si.SizeEstimate != -1 { // but si returns -1 for no size estimate
expSize = si.SizeEstimate
}
res := &pdu.SendRes{
ExpectedSize: expSize,
UsedResumeToken: r.ResumeToken != "",
}
if r.DryRun {
return res, nil, nil
}
// create holds or bookmarks of `From` and `To` to guarantee one of the following:
// - that the replication step can always be resumed (`holds`),
@@ -294,15 +278,47 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
abstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, destroyTypes, keep, check)
}()
sendStream, err := zfs.ZFSSend(ctx, sendArgs)
var sendStream io.ReadCloser
sendStream, err = zfs.ZFSSend(ctx, sendArgs)
if err != nil {
// it's ok to not destroy the abstractions we just created here, a new send attempt will take care of it
return nil, nil, errors.Wrap(err, "zfs send failed")
}
// apply rate limit
sendStream = s.bwLimit.WrapReadCloser(sendStream)
res := &pdu.SendRes{
ExpectedSize: 0,
UsedResumeToken: r.ResumeToken != "",
}
return res, sendStream, nil
}
func (s *Sender) SendDry(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
sendArgs, err := s.sendMakeArgs(ctx, r)
if err != nil {
return nil, err
}
si, err := zfs.ZFSSendDry(ctx, sendArgs)
if err != nil {
return nil, errors.Wrap(err, "zfs send dry failed")
}
// From now on, assume that sendArgs has been validated by ZFSSendDry
// (because validation involves shelling out, it's actually a little expensive)
res := &pdu.SendRes{
ExpectedSize: si.SizeEstimate,
UsedResumeToken: r.ResumeToken != "",
}
return res, nil
}
func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
@@ -422,30 +438,89 @@ type FSMap interface { // FIXME unused
AsFilter() FSFilter
}
// NOTE: when adding members to this struct, remember
// to add them to `ReceiverConfig.copyIn()`
type ReceiverConfig struct {
JobID JobID
RootWithoutClientComponent *zfs.DatasetPath // TODO use
RootWithoutClientComponent *zfs.DatasetPath
AppendClientIdentity bool
InheritProperties []zfsprop.Property
OverrideProperties map[zfsprop.Property]string
BandwidthLimit bandwidthlimit.Config
PlaceholderEncryption PlaceholderCreationEncryptionProperty
}
//go:generate enumer -type=PlaceholderCreationEncryptionProperty -transform=kebab -trimprefix=PlaceholderCreationEncryptionProperty
type PlaceholderCreationEncryptionProperty int
// Note: the constant names, transformed through enumer, are part of the config format!
const (
PlaceholderCreationEncryptionPropertyUnspecified PlaceholderCreationEncryptionProperty = 1 << iota
PlaceholderCreationEncryptionPropertyInherit
PlaceholderCreationEncryptionPropertyOff
)
func (c *ReceiverConfig) copyIn() {
c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
pInherit := make([]zfsprop.Property, len(c.InheritProperties))
copy(pInherit, c.InheritProperties)
c.InheritProperties = pInherit
pOverride := make(map[zfsprop.Property]string, len(c.OverrideProperties))
for key, value := range c.OverrideProperties {
pOverride[key] = value
}
c.OverrideProperties = pOverride
}
func (c *ReceiverConfig) Validate() error {
c.JobID.MustValidate()
for _, prop := range c.InheritProperties {
err := prop.Validate()
if err != nil {
return errors.Wrapf(err, "inherit property %q", prop)
}
}
for prop := range c.OverrideProperties {
err := prop.Validate()
if err != nil {
return errors.Wrapf(err, "override property %q", prop)
}
}
if c.RootWithoutClientComponent.Length() <= 0 {
return errors.New("RootWithoutClientComponent must not be an empty dataset path")
}
if err := bandwidthlimit.ValidateConfig(c.BandwidthLimit); err != nil {
return errors.Wrap(err, "`BandwidthLimit` field invalid")
}
if !c.PlaceholderEncryption.IsAPlaceholderCreationEncryptionProperty() {
return errors.Errorf("`PlaceholderEncryption` field is invalid")
}
return nil
}
// Receiver implements replication.ReplicationEndpoint for a receiving side
type Receiver struct {
pdu.UnsafeReplicationServer // prefer compilation errors over default 'method X not implemented' impl
conf ReceiverConfig // validated
bwLimit bandwidthlimit.Wrapper
recvParentCreationMtx *chainlock.L
Test_OverrideClientIdentityFunc func() string // for use by platformtest
}
func NewReceiver(config ReceiverConfig) *Receiver {
@@ -456,6 +531,7 @@ func NewReceiver(config ReceiverConfig) *Receiver {
return &Receiver{
conf: config,
recvParentCreationMtx: chainlock.New(),
bwLimit: bandwidthlimit.WrapperFromConfig(config.BandwidthLimit),
}
}
@@ -482,9 +558,15 @@ func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
return s.conf.RootWithoutClientComponent.Copy()
}
clientIdentity, ok := ctx.Value(ClientIdentityKey).(string)
var clientIdentity string
if s.Test_OverrideClientIdentityFunc != nil {
clientIdentity = s.Test_OverrideClientIdentityFunc()
} else {
var ok bool
clientIdentity, ok = ctx.Value(ClientIdentityKey).(string) // no shadow
if !ok {
panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
panic("ClientIdentityKey context value must be set")
}
}
clientRoot, err := clientRoot(s.conf.RootWithoutClientComponent, clientIdentity)
@@ -525,7 +607,8 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
if rphs, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, s.conf.RootWithoutClientComponent); err != nil {
return nil, errors.Wrap(err, "cannot determine whether root_fs exists")
} else if !rphs.FSExists {
return nil, errors.New("root_fs does not exist")
getLogger(ctx).WithField("root_fs", s.conf.RootWithoutClientComponent).Error("root_fs does not exist")
return nil, errors.Errorf("root_fs does not exist")
}
root := s.clientRootFromCtx(ctx)
@@ -553,11 +636,6 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
l.WithError(err).Error("cannot get receive resume token")
return nil, err
}
encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, a.ToString())
if err != nil {
l.WithError(err).Error("cannot get encryption enabled status")
return nil, err
}
l.WithField("receive_resume_token", token).Debug("receive resume token")
a.TrimPrefix(root)
@@ -566,7 +644,6 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
Path: a.ToString(),
IsPlaceholder: ph.IsPlaceholder,
ResumeToken: token,
IsEncrypted: encEnabled,
}
fss = append(fss, fs)
}
@@ -629,7 +706,37 @@ func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io
return nil, nil, fmt.Errorf("receiver does not implement Send()")
}
var maxConcurrentZFSRecvSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_RECV", 10))
func (s *Receiver) SendDry(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
return nil, fmt.Errorf("receiver does not implement SendDry()")
}
func (s *Receiver) receive_GetPlaceholderCreationEncryptionValue(client_root, path *zfs.DatasetPath) (zfs.FilesystemPlaceholderCreateEncryptionValue, error) {
if !s.conf.PlaceholderEncryption.IsAPlaceholderCreationEncryptionProperty() {
panic(s.conf.PlaceholderEncryption)
}
if client_root.Equal(path) && s.conf.PlaceholderEncryption == PlaceholderCreationEncryptionPropertyUnspecified {
// If our Receiver is configured to append a client component to s.conf.RootWithoutClientComponent
// then that dataset is always going to be a placeholder.
// We don't want to burden users with the concept of placeholders if their `filesystems` filter on the sender
// doesn't introduce any gaps.
// Since the dataset hierarchy up to and including that client component dataset is still fully controlled by us,
// using `inherit` is going to make it work in all expected use cases.
return zfs.FilesystemPlaceholderCreateEncryptionInherit, nil
}
switch s.conf.PlaceholderEncryption {
case PlaceholderCreationEncryptionPropertyUnspecified:
return 0, fmt.Errorf("placeholder filesystem encryption handling is unspecified in receiver config")
case PlaceholderCreationEncryptionPropertyInherit:
return zfs.FilesystemPlaceholderCreateEncryptionInherit, nil
case PlaceholderCreationEncryptionPropertyOff:
return zfs.FilesystemPlaceholderCreateEncryptionOff, nil
default:
panic(s.conf.PlaceholderEncryption)
}
}
func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
@@ -671,15 +778,18 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
if v.Path.Equal(lp) {
return false
}
l := getLogger(ctx).
WithField("placeholder_fs", v.Path.ToString()).
WithField("receive_fs", lp.ToString())
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, v.Path)
getLogger(ctx).
WithField("fs", v.Path.ToString()).
WithField("placeholder_state", fmt.Sprintf("%#v", ph)).
l.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).
WithField("err", fmt.Sprintf("%s", err)).
WithField("errType", fmt.Sprintf("%T", err)).
Debug("placeholder state for filesystem")
Debug("get placeholder state for filesystem")
if err != nil {
visitErr = err
visitErr = errors.Wrapf(err, "cannot get placeholder state of %s", v.Path.ToString())
return false
}
@@ -690,21 +800,34 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
} else {
visitErr = fmt.Errorf("root_fs %q does not exist", s.conf.RootWithoutClientComponent.ToString())
}
getLogger(ctx).WithError(visitErr).Error("placeholders are only created automatically below root_fs")
l.WithError(visitErr).Error("placeholders are only created automatically below root_fs")
return false
}
l := getLogger(ctx).WithField("placeholder_fs", v.Path)
l.Debug("create placeholder filesystem")
err := zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path, v.Parent.Path)
// compute the value lazily so that users who don't rely on
// placeholders can use the default value PlaceholderCreationEncryptionPropertyUnspecified
placeholderEncryption, err := s.receive_GetPlaceholderCreationEncryptionValue(root, v.Path)
if err != nil {
l.WithError(err).Error("cannot create placeholder filesystem")
visitErr = err
l.WithError(err).Error("cannot create placeholder filesystem") // logger already contains path
visitErr = errors.Wrapf(err, "cannot create placeholder filesystem %s", v.Path.ToString())
return false
}
return true
l := l.WithField("encryption", placeholderEncryption)
l.Debug("creating placeholder filesystem")
err = zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path, v.Parent.Path, placeholderEncryption)
if err != nil {
l.WithError(err).Error("cannot create placeholder filesystem") // logger already contains path
visitErr = errors.Wrapf(err, "cannot create placeholder filesystem %s", v.Path.ToString())
return false
}
getLogger(ctx).WithField("filesystem", v.Path.ToString()).Debug("exists")
l.Info("created placeholder filesystem")
return true
} else {
l.Debug("filesystem exists")
return true // leave this fs as is
}
})
}()
getLogger(ctx).WithField("visitErr", visitErr).Debug("complete tree-walk")
@@ -722,6 +845,10 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
return nil, errors.Wrap(err, "cannot get placeholder state")
}
log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state")
recvOpts.InheritProperties = s.conf.InheritProperties
recvOpts.OverrideProperties = s.conf.OverrideProperties
if ph.FSExists && ph.IsPlaceholder {
recvOpts.RollbackAndForceRecv = true
clearPlaceholderProperty = true
@@ -746,15 +873,8 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
}
log.Debug("acquire concurrent recv semaphore")
// TODO use try-acquire and fail with resource-exhaustion rpc status
// => would require handling on the client-side
// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
guard, err := maxConcurrentZFSRecvSemaphore.Acquire(ctx)
if err != nil {
return nil, err
}
defer guard.Release()
// apply rate limit
receive = s.bwLimit.WrapReadCloser(receive)
var peek bytes.Buffer
var MaxPeek = envconst.Int64("ZREPL_ENDPOINT_RECV_PEEK_SIZE", 1<<20)
endpoint/endpoint_zfs_abstraction.go
+30 -23
View File
@@ -13,6 +13,7 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/util/semaphore"
"github.com/zrepl/zrepl/zfs"
)
@@ -257,7 +258,7 @@ type ListZFSHoldsAndBookmarksQuery struct {
type CreateTXGRangeBound struct {
CreateTXG uint64
Inclusive *zfs.NilBool // must not be nil
Inclusive *nodefault.Bool // must not be nil
}
// A non-empty range of CreateTXGs
@@ -300,7 +301,7 @@ func (q *ListZFSHoldsAndBookmarksQuery) Validate() error {
var createTXGRangeBoundAllowCreateTXG0 = envconst.Bool("ZREPL_ENDPOINT_LIST_ABSTRACTIONS_QUERY_CREATETXG_RANGE_BOUND_ALLOW_0", false)
func (i *CreateTXGRangeBound) Validate() error {
if err := i.Inclusive.Validate(); err != nil {
if err := i.Inclusive.ValidateNoDefault(); err != nil {
return errors.Wrap(err, "Inclusive")
}
if i.CreateTXG == 0 && !createTXGRangeBoundAllowCreateTXG0 {
@@ -419,7 +420,7 @@ func (r *CreateTXGRange) String() string {
if r.Since == nil {
fmt.Fprintf(&buf, "~")
} else {
if err := r.Since.Inclusive.Validate(); err != nil {
if err := r.Since.Inclusive.ValidateNoDefault(); err != nil {
fmt.Fprintf(&buf, "?")
} else if r.Since.Inclusive.B {
fmt.Fprintf(&buf, "[")
@@ -435,7 +436,7 @@ func (r *CreateTXGRange) String() string {
fmt.Fprintf(&buf, "~")
} else {
fmt.Fprintf(&buf, "%d", r.Until.CreateTXG)
if err := r.Until.Inclusive.Validate(); err != nil {
if err := r.Until.Inclusive.ValidateNoDefault(); err != nil {
fmt.Fprintf(&buf, "?")
} else if r.Until.Inclusive.B {
fmt.Fprintf(&buf, "]")
@@ -515,10 +516,12 @@ func (e ListAbstractionsErrors) Error() string {
}
func ListAbstractions(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (out []Abstraction, outErrs []ListAbstractionsError, err error) {
outChan, outErrsChan, err := ListAbstractionsStreamed(ctx, query)
outChan, outErrsChan, drainDone, err := ListAbstractionsStreamed(ctx, query)
if err != nil {
return nil, nil, err
}
defer drainDone()
var wg sync.WaitGroup
wg.Add(1)
go func() {
@@ -539,19 +542,20 @@ func ListAbstractions(ctx context.Context, query ListZFSHoldsAndBookmarksQuery)
}
// if err != nil, the returned channels are both nil
// if err == nil, both channels must be fully drained by the caller to avoid leaking goroutines
func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (<-chan Abstraction, <-chan ListAbstractionsError, error) {
// if err == nil, both channels must be fully drained by the caller to avoid leaking goroutines.
// After draining is done, the caller must call the returned drainDone func.
func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (_ <-chan Abstraction, _ <-chan ListAbstractionsError, drainDone func(), _ error) {
// impl note: structure the query processing in such a way that
// a minimum amount of zfs shell-outs needs to be done
if err := query.Validate(); err != nil {
return nil, nil, errors.Wrap(err, "validate query")
return nil, nil, nil, errors.Wrap(err, "validate query")
}
fss, err := query.FS.Filesystems(ctx)
if err != nil {
return nil, nil, errors.Wrap(err, "list filesystems")
return nil, nil, nil, errors.Wrap(err, "list filesystems")
}
outErrs := make(chan ListAbstractionsError)
@@ -573,28 +577,32 @@ func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmark
sem := semaphore.New(int64(query.Concurrency))
ctx, endTask := trace.WithTask(ctx, "list-abstractions-streamed-producer")
go func() {
defer endTask()
defer close(out)
defer close(outErrs)
_, add, wait := trace.WithTaskGroup(ctx, "list-abstractions-impl-fs")
defer wait()
for i := range fss {
for _, fs := range fss {
fs := fs // capture by copy
add(func(ctx context.Context) {
g, err := sem.Acquire(ctx)
if err != nil {
errCb(err, fss[i], err.Error())
errCb(err, fs, err.Error())
return
}
func() {
defer g.Release()
listAbstractionsImplFS(ctx, fss[i], &query, emitAbstraction, errCb)
listAbstractionsImplFS(ctx, fs, &query, emitAbstraction, errCb)
}()
})
}
}()
return out, outErrs, nil
drainDone = func() {
endTask()
}
return out, outErrs, drainDone, nil
}
func listAbstractionsImplFS(ctx context.Context, fs string, query *ListZFSHoldsAndBookmarksQuery, emitCandidate putListAbstraction, errCb putListAbstractionErr) {
@@ -811,7 +819,7 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
l := by[k]
if k.Type == AbstractionStepHold {
// all older than the most recent cursor are stale, others are always live
// all step holds older than the most recent cursor are stale, others are always live
// if we don't have a replication cursor yet, use untilBound = nil
// to consider all steps stale (...at first)
@@ -819,23 +827,22 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
{
sfnsc := stepFirstNotStaleCandidates[k.fsAndJobId]
// if there's a replication cursor, use it as a cutoff between live and stale
// if there's none, we are in initial replication and only need to keep
// the most recent step hold live, since that's what our initial replication strategy
// uses (both initially and on resume)
// (FIXME hardcoded replication strategy)
// If there's a replication cursor, use it as a cutoff between live and stale
// for both cursors and holds.
// If there's no cursor, we are in initial replication and only need to keep
// the most recent step hold live, which hold the .To of the initial send step.
if sfnsc.cursor != nil {
untilBound = &CreateTXGRangeBound{
CreateTXG: (*sfnsc.cursor).GetCreateTXG(),
// if we have a cursor, can throw away step hold on both From and To
Inclusive: &zfs.NilBool{B: true},
Inclusive: &nodefault.Bool{B: true},
}
} else if sfnsc.step != nil {
untilBound = &CreateTXGRangeBound{
CreateTXG: (*sfnsc.step).GetCreateTXG(),
// if we don't have a cursor, the step most recent step hold is our
// initial replication cursor and it's possibly still live (interrupted initial replication)
Inclusive: &zfs.NilBool{B: false},
Inclusive: &nodefault.Bool{B: false},
}
} else {
untilBound = nil // consider everything stale
@@ -856,7 +863,7 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
}
} else if k.Type == AbstractionReplicationCursorBookmarkV2 || k.Type == AbstractionLastReceivedHold {
// all but the most recent are stale by definition (we always _move_ them)
// all cursors but the most recent cursor are stale by definition (we always _move_ them)
// NOTE: must not use firstNotStale in this branch, not computed for these types
// sort descending (highest createtxg first), then cut off
endpoint/endpoint_zfs_abstraction_test.go
+18 -18
View File
@@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/zfs"
"github.com/zrepl/zrepl/util/nodefault"
)
func TestCreateTXGRange(t *testing.T) {
@@ -47,8 +47,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "wrong order obvious",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{20, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{23, &nodefault.Bool{B: true}},
Until: &CreateTXGRangeBound{20, &nodefault.Bool{B: true}},
},
expectString: "[23,20]",
},
@@ -56,8 +56,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "wrong order edge-case could also be empty",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{22, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{23, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{22, &nodefault.Bool{B: true}},
},
expectString: "(23,22]",
},
@@ -65,8 +65,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "empty",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
},
expectString: "(2,2)",
},
@@ -74,8 +74,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "inclusive-since-exclusive-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: true}},
Until: &CreateTXGRangeBound{5, &nodefault.Bool{B: false}},
},
expectString: "[2,5)",
expect: []testCaseExpectation{
@@ -92,8 +92,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "exclusive-since-inclusive-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{5, &nodefault.Bool{B: true}},
},
expectString: "(2,5]",
expect: []testCaseExpectation{
@@ -111,7 +111,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: true,
config: &CreateTXGRange{
Since: nil,
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: true}},
},
expectString: "~,0]",
},
@@ -119,7 +119,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "half-open-no-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: nil,
},
expectString: "(2,~",
@@ -138,7 +138,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: false,
config: &CreateTXGRange{
Since: nil,
Until: &CreateTXGRangeBound{4, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{4, &nodefault.Bool{B: true}},
},
expectString: "~,4]",
expect: []testCaseExpectation{
@@ -154,7 +154,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeSince",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{math.MaxUint64, &nodefault.Bool{B: true}},
Until: nil,
},
expectString: "[18446744073709551615,~",
@@ -169,7 +169,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeSinceNegative",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{math.MaxUint64, &nodefault.Bool{B: false}},
Until: nil,
},
expectString: "(18446744073709551615,~",
@@ -178,7 +178,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeUntil",
expectInvalid: false,
config: &CreateTXGRange{
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: true}},
},
configAllowZeroCreateTXG: true,
expectString: "~,0]",
@@ -193,7 +193,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: true,
configAllowZeroCreateTXG: true,
config: &CreateTXGRange{
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: false}},
},
expectString: "~,0)",
},
endpoint/placeholdercreationencryptionproperty_enumer.go
+62
View File
@@ -0,0 +1,62 @@
// Code generated by "enumer -type=PlaceholderCreationEncryptionProperty -transform=kebab -trimprefix=PlaceholderCreationEncryptionProperty"; DO NOT EDIT.
//
package endpoint
import (
"fmt"
)
const (
_PlaceholderCreationEncryptionPropertyName_0 = "unspecifiedinherit"
_PlaceholderCreationEncryptionPropertyName_1 = "off"
)
var (
_PlaceholderCreationEncryptionPropertyIndex_0 = [...]uint8{0, 11, 18}
_PlaceholderCreationEncryptionPropertyIndex_1 = [...]uint8{0, 3}
)
func (i PlaceholderCreationEncryptionProperty) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _PlaceholderCreationEncryptionPropertyName_0[_PlaceholderCreationEncryptionPropertyIndex_0[i]:_PlaceholderCreationEncryptionPropertyIndex_0[i+1]]
case i == 4:
return _PlaceholderCreationEncryptionPropertyName_1
default:
return fmt.Sprintf("PlaceholderCreationEncryptionProperty(%d)", i)
}
}
var _PlaceholderCreationEncryptionPropertyValues = []PlaceholderCreationEncryptionProperty{1, 2, 4}
var _PlaceholderCreationEncryptionPropertyNameToValueMap = map[string]PlaceholderCreationEncryptionProperty{
_PlaceholderCreationEncryptionPropertyName_0[0:11]: 1,
_PlaceholderCreationEncryptionPropertyName_0[11:18]: 2,
_PlaceholderCreationEncryptionPropertyName_1[0:3]: 4,
}
// PlaceholderCreationEncryptionPropertyString retrieves an enum value from the enum constants string name.
// Throws an error if the param is not part of the enum.
func PlaceholderCreationEncryptionPropertyString(s string) (PlaceholderCreationEncryptionProperty, error) {
if val, ok := _PlaceholderCreationEncryptionPropertyNameToValueMap[s]; ok {
return val, nil
}
return 0, fmt.Errorf("%s does not belong to PlaceholderCreationEncryptionProperty values", s)
}
// PlaceholderCreationEncryptionPropertyValues returns all values of the enum
func PlaceholderCreationEncryptionPropertyValues() []PlaceholderCreationEncryptionProperty {
return _PlaceholderCreationEncryptionPropertyValues
}
// IsAPlaceholderCreationEncryptionProperty returns "true" if the value is listed in the enum definition. "false" otherwise
func (i PlaceholderCreationEncryptionProperty) IsAPlaceholderCreationEncryptionProperty() bool {
for _, v := range _PlaceholderCreationEncryptionPropertyValues {
if i == v {
return true
}
}
return false
}
go.mod
+21 -12
View File
@@ -5,18 +5,22 @@ go 1.12
require (
github.com/fatih/color v1.7.0
github.com/gdamore/tcell v1.2.0
github.com/gdamore/tcell/v2 v2.2.0
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909
github.com/go-logfmt/logfmt v0.4.0
github.com/go-playground/validator v9.31.0+incompatible
github.com/go-playground/validator/v10 v10.4.1
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
github.com/golang/protobuf v1.3.2
github.com/google/uuid v1.1.1
github.com/golang/protobuf v1.4.3
github.com/google/uuid v1.1.2
github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
github.com/juju/ratelimit v1.0.1
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
github.com/kr/pretty v0.1.0
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.2.0
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.8
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
github.com/modern-go/reflect2 v1.0.1 // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
github.com/montanaflynn/stats v0.5.0
github.com/onsi/ginkgo v1.10.2 // indirect
github.com/onsi/gomega v1.7.0 // indirect
@@ -25,18 +29,23 @@ require (
github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5
github.com/prometheus/client_golang v1.2.1
github.com/prometheus/common v0.7.0
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
github.com/robfig/cron/v3 v3.0.1
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // indirect; go1.12 thinks it needs this
github.com/spf13/cobra v0.0.2
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.6.1
github.com/willf/bitset v1.1.10
github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect; go1.12 thinks it needs this
github.com/yudai/pp v2.0.1+incompatible // indirect
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
gitlab.com/tslocum/cview v1.5.3
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e
gonum.org/v1/gonum v0.7.0 // indirect
google.golang.org/grpc v1.17.0
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579 // indirect
google.golang.org/grpc v1.35.0
google.golang.org/protobuf v1.25.0
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
)
go.sum
+118 -225
View File
@@ -1,37 +1,30 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff h1:zk1wwii7uXmI0znwU+lqg+wFL9G5+vm5I+9rv2let60=
@@ -40,86 +33,54 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
github.com/gdamore/tcell/v2 v2.0.0-dev/go.mod h1:vSVL/GV5mCSlPC6thFP5kfOFdM9MGZcalipmpTxTgQA=
github.com/gdamore/tcell/v2 v2.1.1-0.20201225194624-29bb185874fd/go.mod h1:vSVL/GV5mCSlPC6thFP5kfOFdM9MGZcalipmpTxTgQA=
github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4=
github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909 h1:9NC8seTx6/zRmMTAdsHj/uOMi0EGHGQtjyLafBjk77Q=
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909/go.mod h1:lP+DW8LR6Rw3ru9Vo2/y/3iiLaLWmofYql/va+7zJOk=
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA=
github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4 h1:0suja/iKSDbEIYLbrS/8C7iArJiWpgCNcR+zwAHu7Ig=
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -128,20 +89,13 @@ github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8 h1:+dKzeuiDYbD/Cfi/s
github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY=
github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -149,56 +103,37 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lucasb-eyer/go-colorful v1.0.2 h1:mCMFu6PgSozg9tDNMMK3g18oJBX7oYGrC09mS6CXfO4=
github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -206,197 +141,155 @@ github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284 h1:GX8YFBfSL6GlKymQZOsQllPbxVamfEYpis7oaJLXYZA=
github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284/go.mod h1:XMR9fjP5/wMyFK0Ifk9ZHDFem4Rukm+ChvoYr2nxrgw=
github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1 h1:HH8yzlaZq/A8xdJaSj/eu32yFFySWAJMTmOQPAGwfYg=
github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1/go.mod h1:JRs3nyBjvOv/9YHC+Vlw9z1Jfzl5s5t0BNnTzmclj1c=
github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7 h1:Oik8tLrLhoMq4fduDRuNNOAQ+q0M0dXkjAYLUf4+mAc=
github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5 h1:1eSrO2WAeSXjMol8WRKW9LekL1252VIMaKQwWkmEdvs=
github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/theckman/goconstraint v1.11.0 h1:oBUwN5wpE4dwyPhRGraEgJsFTr+JtLWiDnaJZJeeXI0=
github.com/theckman/goconstraint v1.11.0/go.mod h1:zkCR/f2kOULTk/h1ujgyB9BlCNLaqlQ6GN2Zl4mg81g=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M=
github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f h1:3MuiGfgMHCSwKUcsuI7ODbi50j+evTB7SsoOBMNC5Fk=
github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QESvbW8Meibz7zCrxshP71U9dH5KOCXM=
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
gitlab.com/tslocum/cbind v0.1.4 h1:cbZXPPcieXspk8cShoT6efz7HAT8yMNQcofYWNizis4=
gitlab.com/tslocum/cbind v0.1.4/go.mod h1:RvwYE3auSjBNlCmWeGspzn+jdLUVQ8C2QGC+0nP9ChI=
gitlab.com/tslocum/cview v1.5.3 h1:6OTCtIUp1EkfGeLqQFRHtW8ynMJ66BhoBwuW8oZ84AQ=
gitlab.com/tslocum/cview v1.5.3/go.mod h1:k/eLWRIF3B26VLDgtRRPkjLUXmcCsy+YCSPEAtNQgIY=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeoexdbeMjttk6Oh1rD10=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e h1:Io7mpb+aUAGF0MKxbyQ7HQl1VgB+cL6ZJZUFaFNqVV4=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579 h1:Iwh0ba2kTgq2Q6mJiXhzrrjD7h11nEVnbMHFmp0/HsQ=
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
lazy.sh
+2 -7
View File
@@ -39,15 +39,10 @@ godep() {
export GOOS="$GOHOSTOS"
export GOARCH="$GOHOSTARCH"
# TODO GOARM=$GOHOSTARM?
go build -v -mod=readonly -o "$GOPATH/bin/stringer" golang.org/x/tools/cmd/stringer
go build -v -mod=readonly -o "$GOPATH/bin/protoc-gen-go" github.com/golang/protobuf/protoc-gen-go
go build -v -mod=readonly -o "$GOPATH/bin/enumer" github.com/alvaroloes/enumer
go build -v -mod=readonly -o "$GOPATH/bin/goimports" golang.org/x/tools/cmd/goimports
go build -v -mod=readonly -o "$GOPATH/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
go build -v -mod=readonly -o "$GOPATH/bin/gocovmerge" github.com/wadey/gocovmerge
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
set +x
popd
if ! type stringer || ! type protoc-gen-go || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
if ! type stringer || ! type protoc-gen-go || ! type protoc-gen-go-grpc || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
exit 1
fi
main.go
+2 -1
View File
@@ -4,12 +4,13 @@ package main
import (
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/client"
"github.com/zrepl/zrepl/client/status"
"github.com/zrepl/zrepl/daemon"
)
func init() {
cli.AddSubcommand(daemon.DaemonCmd)
cli.AddSubcommand(client.StatusCmd)
cli.AddSubcommand(status.Subcommand)
cli.AddSubcommand(client.SignalCmd)
cli.AddSubcommand(client.StdinserverCmd)
cli.AddSubcommand(client.ConfigcheckCmd)
packaging/deb/Dockerfile
+5 -1
View File
@@ -1,9 +1,13 @@
FROM debian:latest
# binutils are for cross-compilation to work in bullseye
RUN apt-get update && apt-get install -y \
build-essential \
devscripts \
dh-exec
dh-exec \
binutils-aarch64-linux-gnu \
binutils-arm-linux-gnueabihf \
binutils-i686-linux-gnu
RUN mkdir -p /build/src && chmod -R 0777 /build
platformtest/harness/harness.go
+57 -11
View File
@@ -1,6 +1,7 @@
package main
import (
"container/list"
"context"
"flag"
"fmt"
@@ -37,6 +38,7 @@ func main() {
flag.StringVar(&args.CreateArgs.Mountpoint, "mountpoint", "", "")
flag.BoolVar(&args.StopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
flag.StringVar(&args.Run, "run", "", "")
flag.DurationVar(&platformtest.ZpoolExportTimeout, "zfs.zpool-export-timeout", platformtest.ZpoolExportTimeout, "")
flag.Parse()
if err := HarnessRun(args); err != nil {
@@ -52,6 +54,40 @@ type HarnessArgs struct {
Run string
}
type invocation struct {
runFunc tests.Case
idstring string
result *testCaseResult
children map[string]*invocation
}
func newInvocation(runFunc tests.Case, id string) *invocation {
return &invocation{
runFunc: runFunc,
idstring: id,
children: make(map[string]*invocation),
}
}
func (i *invocation) String() string {
idsuffix := ""
if i.idstring != "" {
idsuffix = fmt.Sprintf(": %s", i.idstring)
}
return fmt.Sprintf("%s%s", i.runFunc.String(), idsuffix)
}
func (i *invocation) RegisterChild(c *invocation) error {
if c.idstring == "" {
return fmt.Errorf("child must have id string")
}
if oc := i.children[c.idstring]; oc != nil {
return fmt.Errorf("idstring %q is already taken by %s", c.idstring, oc)
}
i.children[c.idstring] = c
return nil
}
func HarnessRun(args HarnessArgs) error {
runRE := regexp.MustCompile(args.Run)
@@ -78,21 +114,19 @@ func HarnessRun(args HarnessArgs) error {
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(logger))
ex := platformtest.NewEx(logger)
type invocation struct {
runFunc tests.Case
result *testCaseResult
}
invocations := make([]*invocation, 0, len(tests.Cases))
testQueue := list.New()
for _, c := range tests.Cases {
if runRE.MatchString(c.String()) {
invocations = append(invocations, &invocation{runFunc: c})
testQueue.PushBack(newInvocation(c, ""))
}
}
for _, inv := range invocations {
completedTests := list.New()
bold.Printf("BEGIN TEST CASE %s\n", inv.runFunc.String())
for testQueue.Len() > 0 {
inv := testQueue.Remove(testQueue.Front()).(*invocation)
bold.Printf("BEGIN TEST CASE %s\n", inv)
pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.CreateArgs)
if err != nil {
@@ -102,6 +136,15 @@ func HarnessRun(args HarnessArgs) error {
ctx := &platformtest.Context{
Context: ctx,
RootDataset: filepath.Join(pool.Name(), "rootds"),
QueueSubtest: func(id string, stf func(*platformtest.Context)) {
stinv := newInvocation(stf, id)
err := inv.RegisterChild(stinv)
if err != nil {
panic(err)
}
bold.Printf(" QUEUING SUBTEST %q\n", id)
testQueue.PushFront(stinv)
},
}
res := runTestCase(ctx, ex, inv.runFunc)
@@ -119,6 +162,8 @@ func HarnessRun(args HarnessArgs) error {
panic(fmt.Sprintf("error destroying test pool: %s", err))
}
completedTests.PushBack(inv)
if res.failed {
boldRed.Printf("TEST FAILED\n")
} else if res.skipped {
@@ -135,7 +180,8 @@ func HarnessRun(args HarnessArgs) error {
var summary struct {
succ, fail, skip []*invocation
}
for _, inv := range invocations {
for completedTests.Len() > 0 {
inv := completedTests.Remove(completedTests.Front()).(*invocation)
var bucket *[]*invocation
if inv.result.failed {
bucket = &summary.fail
@@ -156,7 +202,7 @@ func HarnessRun(args HarnessArgs) error {
}
fmt.Printf("\n")
for _, inv := range bucket {
fmt.Printf(" %s\n", inv.runFunc.String())
fmt.Printf(" %s\n", inv)
}
}
printBucket("PASSING TESTS", boldGreen, summary.succ)
platformtest/platformtest.go
+7
View File
@@ -11,6 +11,13 @@ import (
type Context struct {
context.Context
RootDataset string
// Use this callback from a top-level test case to queue the
// execution of sub-tests after this test case is complete.
//
// Note that the testing harness executes the subtest
// _after_ the current top-level test. Hence, the subtest
// cannot use any ZFS state of the top-level test.
QueueSubtest func(id string, stf func(*Context))
}
var FailNowSentinel = fmt.Errorf("platformtest: FailNow called on context")
platformtest/platformtest_ops.go
+3 -3
View File
@@ -256,14 +256,14 @@ nextLine:
case string(DestroyRoot):
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
stmts = append(stmts, &DestroyRootOp{rootds})
continue nextLine
case string(CreateRoot):
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
stmts = append(stmts, &FSOp{Op: Add, Path: rootds})
continue nextLine
@@ -319,7 +319,7 @@ nextLine:
}
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
}
return stmts, nil
platformtest/platformtest_zpool.go
+21 -1
View File
@@ -5,12 +5,17 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/zfs"
)
var ZpoolExportTimeout time.Duration = 500 * time.Millisecond
type Zpool struct {
args ZpoolCreateArgs
}
@@ -93,9 +98,24 @@ func (p *Zpool) Name() string { return p.args.PoolName }
func (p *Zpool) Destroy(ctx context.Context, e Execer) error {
if err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", p.args.PoolName); err != nil {
exportDeadline := time.Now().Add(ZpoolExportTimeout)
for {
if time.Now().After(exportDeadline) {
return errors.Errorf("could not zpool export (got 'pool is busy'): %s", p.args.PoolName)
}
err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", p.args.PoolName)
if err == nil {
break
}
if strings.Contains(err.Error(), "pool is busy") {
runtime.Gosched()
continue
}
if err != nil {
return errors.Wrapf(err, "export pool %q", p.args.PoolName)
}
}
if err := os.Remove(p.args.ImagePath); err != nil {
return errors.Wrapf(err, "remove pool image")
platformtest/tests/generated_cases.go
+13
View File
@@ -21,17 +21,30 @@ var Cases = []Case{BatchDestroy,
ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
ReplicationInitialAll,
ReplicationInitialFail,
ReplicationInitialMostRecent,
ReplicationIsResumableFullSend__both_GuaranteeResumability,
ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication,
ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication,
ReplicationPlaceholderEncryption__EncryptOnReceiverUseCase__WorksIfConfiguredWithInherit,
ReplicationPlaceholderEncryption__UnspecifiedIsOkForClientIdentityPlaceholder,
ReplicationPlaceholderEncryption__UnspecifiedLeadsToFailureAtRuntimeWhenCreatingPlaceholders,
ReplicationPropertyReplicationWorks,
ReplicationReceiverErrorWhileStillSending,
ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication,
ReplicationStepCompletedLostBehavior__GuaranteeResumability,
ResumableRecvAndTokenHandling,
ResumeTokenParsing,
SendArgsValidationEE_EncryptionAndRaw,
SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false,
SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true,
SendArgsValidationResumeTokenDifferentFilesystemForbidden,
SendArgsValidationResumeTokenEncryptionMismatchForbidden,
SendStreamCloseAfterBlockedOnPipeWrite,
SendStreamCloseAfterEOFRead,
SendStreamMultipleCloseAfterEOF,
SendStreamMultipleCloseBeforeEOF,
SendStreamNonEOFReadErrorHandling,
UndestroyableSnapshotParsing,
}
platformtest/tests/helpers.go
+6
View File
@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/limitio"
"github.com/zrepl/zrepl/zfs"
@@ -174,3 +175,8 @@ func datasetToStringSortedTrimPrefix(prefix *zfs.DatasetPath, paths []*zfs.Datas
sort.Strings(pstrs)
return pstrs
}
func mustAddToSFilter(ctx *platformtest.Context, f *filters.DatasetMapFilter, fs string) {
err := f.Add(fs, "ok")
require.NoError(ctx, err)
}
platformtest/tests/recvForceIntoEncryptedErr.go
+4 -1
View File
@@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -32,10 +33,12 @@ func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
sendArgs, err := zfs.ZFSSendArgsUnvalidated{
FS: sfs,
Encrypted: &zfs.NilBool{B: false},
From: nil,
To: &sfsSnap1,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: "",
},
}.Validate(ctx)
require.NoError(ctx, err)
platformtest/tests/recvRollback.go
+5 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -28,15 +29,18 @@ func ReceiveForceRollbackWorksUnencrypted(ctx *platformtest.Context) {
sendArgs, err := zfs.ZFSSendArgsUnvalidated{
FS: sfs,
Encrypted: &zfs.NilBool{B: false},
From: nil,
To: &sfsSnap1,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: "",
},
}.Validate(ctx)
require.NoError(ctx, err)
sendStream, err := zfs.ZFSSend(ctx, sendArgs)
require.NoError(ctx, err)
defer sendStream.Close()
recvOpts := zfs.RecvOptions{
RollbackAndForceRecv: true,
platformtest/tests/replication.go
+512 -33
View File
@@ -7,6 +7,8 @@ import (
"os"
"path"
"sort"
"strings"
"time"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
@@ -15,11 +17,15 @@ import (
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/replication/driver"
"github.com/zrepl/zrepl/replication/logic"
"github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/replication/report"
"github.com/zrepl/zrepl/util/bandwidthlimit"
"github.com/zrepl/zrepl/util/limitio"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
// mimics the replication invocations of an active-side job
@@ -35,7 +41,11 @@ type replicationInvocation struct {
rfsRoot string
interceptSender func(e *endpoint.Sender) logic.Sender
interceptReceiver func(e *endpoint.Receiver) logic.Receiver
guarantee pdu.ReplicationConfigProtection
guarantee *pdu.ReplicationConfigProtection
senderConfigHook func(*endpoint.SenderConfig)
receiverConfigHook func(*endpoint.ReceiverConfig)
plannerPolicyHook func(*logic.PlannerPolicy)
skipSendArgsValidation bool
}
func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
@@ -55,25 +65,58 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
err := i.sfilter.Add(i.sfs, "ok")
require.NoError(ctx, err)
}
sender := i.interceptSender(endpoint.NewSender(endpoint.SenderConfig{
senderConfig := endpoint.SenderConfig{
FSF: i.sfilter.AsFilter(),
Encrypt: &zfs.NilBool{B: false},
Encrypt: &nodefault.Bool{B: false},
JobID: i.sjid,
}))
receiver := i.interceptReceiver(endpoint.NewReceiver(endpoint.ReceiverConfig{
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
}
if i.senderConfigHook != nil {
i.senderConfigHook(&senderConfig)
}
receiverConfig := endpoint.ReceiverConfig{
JobID: i.rjid,
AppendClientIdentity: false,
RootWithoutClientComponent: mustDatasetPath(i.rfsRoot),
}))
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
PlaceholderEncryption: endpoint.PlaceholderCreationEncryptionPropertyUnspecified,
}
if i.receiverConfigHook != nil {
i.receiverConfigHook(&receiverConfig)
}
require.Equal(ctx, senderConfig.JobID, i.sjid)
require.Equal(ctx, receiverConfig.JobID, i.rjid)
sender := i.interceptSender(endpoint.NewSender(senderConfig))
receiver := i.interceptReceiver(endpoint.NewReceiver(receiverConfig))
plannerPolicy := logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(false),
ReplicationConfig: pdu.ReplicationConfig{
Protection: &i.guarantee,
ReplicationConfig: &pdu.ReplicationConfig{
Protection: i.guarantee,
},
ConflictResolution: &logic.ConflictResolution{
InitialReplication: logic.InitialReplicationAutoResolutionMostRecent,
},
SizeEstimationConcurrency: 1,
}
if i.plannerPolicyHook != nil {
i.plannerPolicyHook(&plannerPolicy)
}
var doCtx context.Context = ctx
if i.skipSendArgsValidation {
doCtx = zfs.ZFSSendArgsSkipValidation(ctx)
}
report, wait := replication.Do(
ctx,
doCtx,
driver.Config{
MaxAttempts: 1,
StepQueueConcurrency: 1,
ReconnectHardFailTimeout: 1 * time.Second,
},
logic.NewPlanner(nil, nil, sender, receiver, plannerPolicy),
)
wait(true)
@@ -106,7 +149,7 @@ func ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed(ctx *platformte
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
@@ -170,7 +213,7 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
@@ -343,7 +386,7 @@ func (s *PartialSender) Send(ctx context.Context, r *pdu.SendReq) (r1 *pdu.SendR
func ReplicationIsResumableFullSend__both_GuaranteeResumability(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
},
@@ -358,7 +401,7 @@ func ReplicationIsResumableFullSend__both_GuaranteeResumability(ctx *platformtes
func ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
},
@@ -373,7 +416,7 @@ func ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_G
func ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
},
@@ -386,7 +429,7 @@ func ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_inc
}
type replicationIsResumableFullSendSetup struct {
protection pdu.ReplicationConfigProtection
protection *pdu.ReplicationConfigProtection
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated bool
expectAllThreeSnapshotsToThreeBePresentAfterLoop bool
expectNoSnapshotsOnReceiverAfterLoop bool
@@ -501,7 +544,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
report := rep.Do(ctx)
@@ -528,7 +571,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability), // !
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability), // !
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
},
@@ -574,7 +617,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication), // !
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication), // !
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
},
@@ -684,7 +727,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
}
rfs := rep.ReceiveSideFilesystem()
report := rep.Do(ctx)
@@ -702,7 +745,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &FailSendCompletedSender{e}
},
@@ -750,7 +793,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
}
report := rep.Do(ctx)
ctx.Logf("expecting failure:\n%s", pretty.Sprint(report))
@@ -779,13 +822,22 @@ type NeverEndingSender struct {
*endpoint.Sender
}
func (s *NeverEndingSender) SendDry(ctx context.Context, req *pdu.SendReq) (r *pdu.SendRes, err error) {
r, _, err = s.sendImpl(ctx, req, true)
return r, err
}
func (s *NeverEndingSender) Send(ctx context.Context, req *pdu.SendReq) (r *pdu.SendRes, stream io.ReadCloser, _ error) {
return s.sendImpl(ctx, req, false)
}
func (s *NeverEndingSender) sendImpl(ctx context.Context, req *pdu.SendReq, dry bool) (r *pdu.SendRes, stream io.ReadCloser, _ error) {
stream = nil
r = &pdu.SendRes{
UsedResumeToken: false,
ExpectedSize: 1 << 30,
}
if req.DryRun {
if dry {
return r, stream, nil
}
dz, err := os.Open("/dev/zero")
@@ -818,7 +870,7 @@ func ReplicationReceiverErrorWhileStillSending(ctx *platformtest.Context) {
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
},
@@ -864,13 +916,9 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
fsAA := ctx.RootDataset + "/sender/aa"
sfilter := filters.NewDatasetMapFilter(3, true)
mustAddToSFilter := func(fs string) {
err := sfilter.Add(fs, "ok")
require.NoError(ctx, err)
}
mustAddToSFilter(fsA)
mustAddToSFilter(fsAChild)
mustAddToSFilter(fsAA)
mustAddToSFilter(ctx, sfilter, fsA)
mustAddToSFilter(ctx, sfilter, fsAChild)
mustAddToSFilter(ctx, sfilter, fsAA)
rfsRoot := ctx.RootDataset + "/receiver"
mockRecvErr := fmt.Errorf("yifae4ohPhaquaes0hohghiep9oufie4roo7quoWooluaj2ee8")
@@ -880,7 +928,7 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
},
@@ -900,7 +948,7 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
}
require.Contains(ctx, fsByName, fsA)
require.Contains(ctx, fsByName, fsAA)
require.Contains(ctx, fsByName, fsAChild)
require.Contains(ctx, fsByName, fsAA)
checkFS := func(fs string, expectErrMsg string) {
@@ -915,3 +963,434 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
checkFS(fsAChild, "parent(s) failed during initial replication")
checkFS(fsAA, mockRecvErr.Error()) // fsAA is not treated as a child of fsA
}
func ReplicationPropertyReplicationWorks(ctx *platformtest.Context) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "sender"
+ "sender/a"
+ "sender/a@1"
+ "sender/a/child"
+ "sender/a/child@1"
+ "receiver"
R zfs create -p "${ROOTDS}/receiver/${ROOTDS}/sender"
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
fsA := ctx.RootDataset + "/sender/a"
fsAChild := ctx.RootDataset + "/sender/a/child"
sfilter := filters.NewDatasetMapFilter(2, true)
mustAddToSFilter(ctx, sfilter, fsA)
mustAddToSFilter(ctx, sfilter, fsAChild)
rfsRoot := ctx.RootDataset + "/receiver"
type testPropExpectation struct {
Exists bool
Source zfs.PropertySource
ExpectSpecialValue string
}
type testProp struct {
Name string
SetOnSender map[string]bool
ExpectReceiver map[string]testPropExpectation
}
testProps := []testProp{
{
Name: "zrepl:ignored",
SetOnSender: map[string]bool{fsA: true, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {
Exists: false,
},
fsAChild: {
Exists: false,
},
},
},
{
Name: "zrepl:replicate",
SetOnSender: map[string]bool{fsA: true, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {Exists: true, Source: zfs.SourceReceived},
fsAChild: {Exists: true, Source: zfs.SourceReceived},
},
},
{
Name: "zrepl:overridden",
SetOnSender: map[string]bool{fsA: false, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"},
fsAChild: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"},
},
},
}
for _, prop := range testProps {
for fs := range prop.SetOnSender {
err := zfs.ZFSSet(ctx, mustDatasetPath(fs), map[string]string{prop.Name: prop.Name})
require.NoError(ctx, err)
}
}
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
receiverConfigHook: func(c *endpoint.ReceiverConfig) {
c.InheritProperties = []zfsprop.Property{"zrepl:ignored"}
c.OverrideProperties = map[zfsprop.Property]string{
"zrepl:overridden": "overridden value",
}
},
senderConfigHook: func(c *endpoint.SenderConfig) {
c.SendProperties = true // TODO: do another tier with SendBackupProperties
},
}
r := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(r))
require.Len(ctx, r.Attempts, 1)
attempt := r.Attempts[0]
require.Nil(ctx, attempt.PlanError)
require.Len(ctx, attempt.Filesystems, 2)
fsByName := make(map[string]*report.FilesystemReport, len(attempt.Filesystems))
for _, fs := range attempt.Filesystems {
fsByName[fs.Info.Name] = fs
}
require.Contains(ctx, fsByName, fsA)
require.Contains(ctx, fsByName, fsAChild)
require.Len(ctx, fsByName, 2)
requireFSReportSucceeded := func(fs string) *zfs.DatasetPath {
rep := fsByName[fs]
require.Len(ctx, rep.Steps, 1)
require.Nil(ctx, rep.PlanError)
if rep.StepError != nil && strings.Contains(rep.StepError.Error(), "invalid option 'x'") {
ctx.SkipNow() // XXX feature detection
}
require.Nil(ctx, rep.StepError)
require.Len(ctx, rep.Steps, 1)
require.Equal(ctx, 1, rep.CurrentStep)
return mustDatasetPath(path.Join(rfsRoot, fs))
}
rfsA := requireFSReportSucceeded(fsA)
rfsAChild := requireFSReportSucceeded(fsAChild)
rfsmap := map[string]*zfs.DatasetPath{
fsA: rfsA,
fsAChild: rfsAChild,
}
for fs, rfs := range rfsmap {
for _, tp := range testProps {
r_, err := zfs.ZFSGet(ctx, rfs, []string{tp.Name})
require.NoError(ctx, err)
r := r_.GetDetails(tp.Name)
expect := tp.ExpectReceiver[fs]
if !expect.Exists {
require.Equal(ctx, zfs.SourceNone, r.Source)
} else {
require.Equal(ctx, expect.Source, r.Source)
if expect.ExpectSpecialValue != "" {
require.Equal(ctx, expect.ExpectSpecialValue, r.Value)
} else {
require.Equal(ctx, tp.Name, r.Value)
}
}
}
}
}
func ReplicationPlaceholderEncryption__UnspecifiedLeadsToFailureAtRuntimeWhenCreatingPlaceholders(ctx *platformtest.Context) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "sender"
+ "sender/a"
+ "sender/a/child"
+ "receiver"
R zfs snapshot -r ${ROOTDS}/sender@initial
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
childfs := ctx.RootDataset + "/sender/a/child"
sfilter := filters.NewDatasetMapFilter(3, true)
mustAddToSFilter(ctx, sfilter, childfs)
rfsRoot := ctx.RootDataset + "/receiver"
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
receiverConfigHook: func(rc *endpoint.ReceiverConfig) {
rc.PlaceholderEncryption = endpoint.PlaceholderCreationEncryptionPropertyUnspecified
},
}
r := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(r))
require.Len(ctx, r.Attempts, 1)
attempt := r.Attempts[0]
require.Nil(ctx, attempt.PlanError)
require.Len(ctx, attempt.Filesystems, 1)
afs := attempt.Filesystems[0]
require.Equal(ctx, childfs, afs.Info.Name)
require.Equal(ctx, 1, len(afs.Steps))
require.Equal(ctx, 0, afs.CurrentStep)
require.Equal(ctx, report.FilesystemSteppingErrored, afs.State)
childfsFirstComponent := strings.Split(childfs, "/")[0]
require.Contains(ctx, afs.StepError.Err, "cannot create placeholder filesystem "+rfsRoot+"/"+childfsFirstComponent+": placeholder filesystem encryption handling is unspecified in receiver config")
}
type ClientIdentityReceiver struct {
clientIdentity string
*endpoint.Receiver
}
func (r *ClientIdentityReceiver) Receive(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, r.clientIdentity)
return r.Receiver.Receive(ctx, req, stream)
}
func ReplicationPlaceholderEncryption__UnspecifiedIsOkForClientIdentityPlaceholder(ctx *platformtest.Context) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "receiver"
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
sfilter := filters.NewDatasetMapFilter(1, true)
// hacky...
comps := strings.Split(ctx.RootDataset, "/")
require.GreaterOrEqual(ctx, len(comps), 2)
pool := comps[0]
require.Contains(ctx, pool, "zreplplatformtest", "don't want to cause accidents")
poolchild := pool + "/" + comps[1]
err := zfs.ZFSSnapshot(ctx, mustDatasetPath(pool), "testsnap", false)
require.NoError(ctx, err)
err = zfs.ZFSSnapshot(ctx, mustDatasetPath(poolchild), "testsnap", false)
require.NoError(ctx, err)
mustAddToSFilter(ctx, sfilter, pool)
mustAddToSFilter(ctx, sfilter, poolchild)
clientIdentity := "testclientid"
rfsRoot := ctx.RootDataset + "/receiver"
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
receiverConfigHook: func(rc *endpoint.ReceiverConfig) {
rc.PlaceholderEncryption = endpoint.PlaceholderCreationEncryptionPropertyUnspecified
rc.AppendClientIdentity = true
},
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
r.Test_OverrideClientIdentityFunc = func() string { return clientIdentity }
return r
},
}
r := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(r))
require.Len(ctx, r.Attempts, 1)
attempt := r.Attempts[0]
require.Nil(ctx, attempt.PlanError)
require.Len(ctx, attempt.Filesystems, 2)
filesystemsByName := make(map[string]*report.FilesystemReport)
for _, fs := range attempt.Filesystems {
filesystemsByName[fs.Info.Name] = fs
}
require.Len(ctx, filesystemsByName, len(attempt.Filesystems))
afs, ok := filesystemsByName[pool]
require.True(ctx, ok)
require.Nil(ctx, afs.PlanError)
require.Nil(ctx, afs.StepError)
require.Equal(ctx, report.FilesystemDone, afs.State)
afs, ok = filesystemsByName[poolchild]
require.True(ctx, ok)
require.Nil(ctx, afs.PlanError)
require.Nil(ctx, afs.StepError)
require.Equal(ctx, report.FilesystemDone, afs.State)
mustGetFilesystemVersion(ctx, rfsRoot+"/"+clientIdentity+"/"+pool+"@testsnap")
mustGetFilesystemVersion(ctx, rfsRoot+"/"+clientIdentity+"/"+poolchild+"@testsnap")
}
func replicationPlaceholderEncryption__EncryptOnReceiverUseCase__impl(ctx *platformtest.Context, placeholderEncryption endpoint.PlaceholderCreationEncryptionProperty) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "sender"
+ "sender/a"
+ "sender/a/child"
+ "receiver" encrypted
R zfs snapshot -r ${ROOTDS}/sender@initial
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
childfs := ctx.RootDataset + "/sender/a/child"
sfilter := filters.NewDatasetMapFilter(3, true)
mustAddToSFilter(ctx, sfilter, childfs)
rfsRoot := ctx.RootDataset + "/receiver"
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
receiverConfigHook: func(rc *endpoint.ReceiverConfig) {
rc.PlaceholderEncryption = placeholderEncryption
rc.AppendClientIdentity = false
},
}
r := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(r))
require.Len(ctx, r.Attempts, 1)
attempt := r.Attempts[0]
require.Equal(ctx, report.AttemptDone, attempt.State)
require.Len(ctx, attempt.Filesystems, 1)
afs := attempt.Filesystems[0]
require.Equal(ctx, childfs, afs.Info.Name)
require.Equal(ctx, 1, len(afs.Steps))
rfs := mustDatasetPath(rfsRoot + "/" + childfs)
mustGetFilesystemVersion(ctx, rfs.ToString()+"@initial")
}
func ReplicationPlaceholderEncryption__EncryptOnReceiverUseCase__WorksIfConfiguredWithInherit(ctx *platformtest.Context) {
placeholderEncryption := endpoint.PlaceholderCreationEncryptionPropertyInherit
replicationPlaceholderEncryption__EncryptOnReceiverUseCase__impl(ctx, placeholderEncryption)
childfs := ctx.RootDataset + "/sender/a/child"
rfsRoot := ctx.RootDataset + "/receiver"
rfs := mustDatasetPath(rfsRoot + "/" + childfs)
// The leaf child dataset should be inhering from rfsRoot.
// If we had replicated with PlaceholderCreationEncryptionPropertyOff
// then it would be unencrypted and inherit from the placeholder.
props, err := zfs.ZFSGet(ctx, rfs, []string{"encryptionroot"})
require.NoError(ctx, err)
require.Equal(ctx, rfsRoot, props.Get("encryptionroot"))
}
func replicationInitialImpl(ctx *platformtest.Context, iras logic.InitialReplicationAutoResolution, expectExactRfsSnaps []string) *report.Report {
// reverse order for snap names to expose sorting assumptions
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "sender"
+ "sender@3"
+ "sender@2"
+ "sender@1"
+ "receiver"
R zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
sfs := ctx.RootDataset + "/sender"
rfsRoot := ctx.RootDataset + "/receiver"
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
plannerPolicyHook: func(pp *logic.PlannerPolicy) {
pp.ConflictResolution.InitialReplication = iras
},
}
rfs := rep.ReceiveSideFilesystem()
// first replication
report := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(report))
versions, err := zfs.ZFSListFilesystemVersions(ctx, mustDatasetPath(rfs), zfs.ListFilesystemVersionsOptions{Types: zfs.Snapshots})
if _, ok := err.(*zfs.DatasetDoesNotExist); ok {
versions = nil
} else {
require.NoError(ctx, err)
}
bySnapName := make(map[string]int)
for _, v := range versions {
bySnapName[v.GetName()] += 1
}
for _, v := range expectExactRfsSnaps {
bySnapName[v] -= 1
}
for _, v := range bySnapName {
if v != 0 {
ctx.Logf("unexpected snaps:\n%#v", bySnapName)
ctx.FailNow()
}
}
return report
}
func ReplicationInitialAll(ctx *platformtest.Context) {
replicationInitialImpl(ctx, logic.InitialReplicationAutoResolutionAll, []string{"3", "2", "1"})
}
func ReplicationInitialMostRecent(ctx *platformtest.Context) {
replicationInitialImpl(ctx, logic.InitialReplicationAutoResolutionMostRecent, []string{"1"})
}
func ReplicationInitialFail(ctx *platformtest.Context) {
report := replicationInitialImpl(ctx, logic.InitialReplicationAutoResolutionFail, []string{})
require.Len(ctx, report.Attempts, 1)
require.Nil(ctx, report.Attempts[0].PlanError)
require.Len(ctx, report.Attempts[0].Filesystems, 1)
require.NotNil(ctx, report.Attempts[0].Filesystems[0].PlanError)
require.Contains(ctx, report.Attempts[0].Filesystems[0].PlanError.Err, "automatic conflict resolution for initial replication is disabled in config")
}

Some files were not shown because too many files have changed in this diff Show More