adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: adminer/zrepl_patched:problame/zync
Branches Tags
adminer/zrepl_patched:master adminer/zrepl_patched:dependabot/uv/docs/idna-3.15 adminer/zrepl_patched:dependabot/uv/docs/urllib3-2.7.0 adminer/zrepl_patched:dependabot/uv/docs/pygments-2.20.0 adminer/zrepl_patched:dependabot/uv/docs/requests-2.33.0 adminer/zrepl_patched:dependabot/go_modules/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:dependabot/go_modules/build/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:releases/0.7.0-2 adminer/zrepl_patched:dependabot/go_modules/filippo.io/edwards25519-1.1.1 adminer/zrepl_patched:docs-user-delegation adminer/zrepl_patched:platformtest-simplify adminer/zrepl_patched:problame/go-1.24-tools adminer/zrepl_patched:problame/metric-for-running-jobs adminer/zrepl_patched:problame/triggers adminer/zrepl_patched:releases/0.6.1-2 adminer/zrepl_patched:problame/static-binaries adminer/zrepl_patched:problame/release-circleci-binaries adminer/zrepl_patched:problame/prep-release-0.6.1 adminer/zrepl_patched:problame/circleci-build adminer/zrepl_patched:iss-611-fix adminer/zrepl_patched:go1.19-update adminer/zrepl_patched:iss-486-duration-units adminer/zrepl_patched:iss-611-timezone-snapper adminer/zrepl_patched:problame/develop adminer/zrepl_patched:524-keep_not_replicated-for-pull-jobs adminer/zrepl_patched:517-tls-bad-cert adminer/zrepl_patched:495-fix-stuck-waiting-for-zfs-send-on-freebsd adminer/zrepl_patched:fix-encrypt-on-recv adminer/zrepl_patched:trigger-wait-reset-rearchitecture adminer/zrepl_patched:parallel-replication adminer/zrepl_patched:problame/golang-freebsd-investigation adminer/zrepl_patched:problame/zync adminer/zrepl_patched:problame/fix-replication-order-deadlock-on-second-attempt adminer/zrepl_patched:property_send_reformat_squash adminer/zrepl_patched:problame/platformtest-ci adminer/zrepl_patched:problame/322-fix-ssh-client-leaks-backport-0.2.1 adminer/zrepl_patched:problame/322-fix-ssh-client-leaks adminer/zrepl_patched:problame/pruner_rewrite_step_holds adminer/zrepl_patched:problame/307-tracing adminer/zrepl_patched:problame/pr-fix-replication-cursor-v1-crash-in-list-stale adminer/zrepl_patched:problame/fix-initial-replication-for-encrypted-send adminer/zrepl_patched:problame/ci-fixes-20200418 adminer/zrepl_patched:problame/308-zsh-completions-fixup adminer/zrepl_patched:problame/308-zsh-completions adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v3 adminer/zrepl_patched:problame/fix-zfscmd-nil-deref adminer/zrepl_patched:problame/cleanups-20200408 adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v2 adminer/zrepl_patched:problame/277-fix-receive-f adminer/zrepl_patched:problame/280-racy-holds-list adminer/zrepl_patched:problame/274-fix-incorrect-early-release-concurrent-send-sempahore adminer/zrepl_patched:problame/262-fix-status-eof-error adminer/zrepl_patched:problame/186-debug-dump-command adminer/zrepl_patched:problame/wip/tiered-snapshot-management adminer/zrepl_patched:problame/tracing adminer/zrepl_patched:problame/wip/keep-rule-most-recent-common-ancestor adminer/zrepl_patched:problame/experiment-runtime-controllable-concurrency adminer/zrepl_patched:parallel_zfs_ops adminer/zrepl_patched:resumable_send_recv
adminer/zrepl_patched:v0.7.0 adminer/zrepl_patched:v0.6.1 adminer/zrepl_patched:v0.6.0 adminer/zrepl_patched:v0.5.0 adminer/zrepl_patched:v0.5.0-rc1 adminer/zrepl_patched:v0.4.0 adminer/zrepl_patched:v0.4.0-rc2 adminer/zrepl_patched:v0.4.0-rc1 adminer/zrepl_patched:v0.3.1 adminer/zrepl_patched:v0.3.0 adminer/zrepl_patched:v0.3.0-rc2 adminer/zrepl_patched:v0.3.0-rc1 adminer/zrepl_patched:v0.2.1 adminer/zrepl_patched:v0.2.0 adminer/zrepl_patched:v0.2.0-rc1 adminer/zrepl_patched:v0.1.1 adminer/zrepl_patched:v0.1.0 adminer/zrepl_patched:v0.1.0-rc4 adminer/zrepl_patched:0.1.0-rc3 adminer/zrepl_patched:0.1.0-rc2 adminer/zrepl_patched:0.1.0-rc1 adminer/zrepl_patched:0.0.3 adminer/zrepl_patched:0.0.2 adminer/zrepl_patched:0.0.1
...
compare: adminer/zrepl_patched:v0.4.0-rc1
Branches Tags
adminer/zrepl_patched:master adminer/zrepl_patched:dependabot/uv/docs/idna-3.15 adminer/zrepl_patched:dependabot/uv/docs/urllib3-2.7.0 adminer/zrepl_patched:dependabot/uv/docs/pygments-2.20.0 adminer/zrepl_patched:dependabot/uv/docs/requests-2.33.0 adminer/zrepl_patched:dependabot/go_modules/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:dependabot/go_modules/build/google.golang.org/grpc-1.79.3 adminer/zrepl_patched:releases/0.7.0-2 adminer/zrepl_patched:dependabot/go_modules/filippo.io/edwards25519-1.1.1 adminer/zrepl_patched:docs-user-delegation adminer/zrepl_patched:platformtest-simplify adminer/zrepl_patched:problame/go-1.24-tools adminer/zrepl_patched:problame/metric-for-running-jobs adminer/zrepl_patched:problame/triggers adminer/zrepl_patched:releases/0.6.1-2 adminer/zrepl_patched:problame/static-binaries adminer/zrepl_patched:problame/release-circleci-binaries adminer/zrepl_patched:problame/prep-release-0.6.1 adminer/zrepl_patched:problame/circleci-build adminer/zrepl_patched:iss-611-fix adminer/zrepl_patched:go1.19-update adminer/zrepl_patched:iss-486-duration-units adminer/zrepl_patched:iss-611-timezone-snapper adminer/zrepl_patched:problame/develop adminer/zrepl_patched:524-keep_not_replicated-for-pull-jobs adminer/zrepl_patched:517-tls-bad-cert adminer/zrepl_patched:495-fix-stuck-waiting-for-zfs-send-on-freebsd adminer/zrepl_patched:fix-encrypt-on-recv adminer/zrepl_patched:trigger-wait-reset-rearchitecture adminer/zrepl_patched:parallel-replication adminer/zrepl_patched:problame/golang-freebsd-investigation adminer/zrepl_patched:problame/zync adminer/zrepl_patched:problame/fix-replication-order-deadlock-on-second-attempt adminer/zrepl_patched:property_send_reformat_squash adminer/zrepl_patched:problame/platformtest-ci adminer/zrepl_patched:problame/322-fix-ssh-client-leaks-backport-0.2.1 adminer/zrepl_patched:problame/322-fix-ssh-client-leaks adminer/zrepl_patched:problame/pruner_rewrite_step_holds adminer/zrepl_patched:problame/307-tracing adminer/zrepl_patched:problame/pr-fix-replication-cursor-v1-crash-in-list-stale adminer/zrepl_patched:problame/fix-initial-replication-for-encrypted-send adminer/zrepl_patched:problame/ci-fixes-20200418 adminer/zrepl_patched:problame/308-zsh-completions-fixup adminer/zrepl_patched:problame/308-zsh-completions adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v3 adminer/zrepl_patched:problame/fix-zfscmd-nil-deref adminer/zrepl_patched:problame/cleanups-20200408 adminer/zrepl_patched:problame/holds-release-and-hold-leak-fix-v2 adminer/zrepl_patched:problame/277-fix-receive-f adminer/zrepl_patched:problame/280-racy-holds-list adminer/zrepl_patched:problame/274-fix-incorrect-early-release-concurrent-send-sempahore adminer/zrepl_patched:problame/262-fix-status-eof-error adminer/zrepl_patched:problame/186-debug-dump-command adminer/zrepl_patched:problame/wip/tiered-snapshot-management adminer/zrepl_patched:problame/tracing adminer/zrepl_patched:problame/wip/keep-rule-most-recent-common-ancestor adminer/zrepl_patched:problame/experiment-runtime-controllable-concurrency adminer/zrepl_patched:parallel_zfs_ops adminer/zrepl_patched:resumable_send_recv
adminer/zrepl_patched:v0.7.0 adminer/zrepl_patched:v0.6.1 adminer/zrepl_patched:v0.6.0 adminer/zrepl_patched:v0.5.0 adminer/zrepl_patched:v0.5.0-rc1 adminer/zrepl_patched:v0.4.0 adminer/zrepl_patched:v0.4.0-rc2 adminer/zrepl_patched:v0.4.0-rc1 adminer/zrepl_patched:v0.3.1 adminer/zrepl_patched:v0.3.0 adminer/zrepl_patched:v0.3.0-rc2 adminer/zrepl_patched:v0.3.0-rc1 adminer/zrepl_patched:v0.2.1 adminer/zrepl_patched:v0.2.0 adminer/zrepl_patched:v0.2.0-rc1 adminer/zrepl_patched:v0.1.1 adminer/zrepl_patched:v0.1.0 adminer/zrepl_patched:v0.1.0-rc4 adminer/zrepl_patched:0.1.0-rc3 adminer/zrepl_patched:0.1.0-rc2 adminer/zrepl_patched:0.1.0-rc1 adminer/zrepl_patched:0.0.3 adminer/zrepl_patched:0.0.2 adminer/zrepl_patched:0.0.1

24 Commits
problame/zync ... v0.4.0-rc1

Author SHA1 Message Date
Christian Schwarz 5c6d69a69c zfs: PropertySource: set type to uint32 so that enumer-generated code is platform-independent 
make zrepl-bin test-platform-bin vet lint GOOS=freebsd   GOARCH=386
make[2]: Entering directory '/src'
GO111MODULE=on go build -mod=readonly  -ldflags "-X github.com/zrepl/zrepl/version.zreplVersion=v0.3.1-20-g07f2bff" -o "artifacts/zrepl-freebsd-386"
zfs/propertysource_enumer.go:41:9: constant 18446744073709551615 overflows PropertySource
zfs/propertysource_enumer.go:48:66: constant 18446744073709551615 overflows PropertySource
zfs/propertysource_enumer.go:57:23: constant 18446744073709551615 overflows PropertySource

fixes #429
 2021-03-14 22:32:45 +01:00
Christian Schwarz 299808aaaf docs: 0.4 changelog: mention the 0.3.1 fix to prune:grid 
fixes #400
refs 3a4e841c73
 2021-03-14 21:13:26 +01:00
Christian Schwarz 7071adc774 docs: 0.4 changelog 2021-03-14 20:56:02 +01:00
InsanePrawn 8d678eed19 docs: add note about zfs recv -x mountpoint with ZVOLs 
refs #430
 2021-03-14 20:26:39 +01:00
Calistoc ab7abd9686 status: show the current replication attempt's runtime 2021-03-14 18:24:28 +01:00
Christian Schwarz a58ce74ed0 implement new 'zrepl status' 
Primary goals:

- Scrollable output ( fixes #245 )
- Sending job signals from status view
- Filtering of output by filesystem

Implementation:

- original TUI framework: github.com/rivo/tview
- but: tview is quasi-unmaintained, didn't support some features
- => use fork https://gitlab.com/tslocum/cview
- however, don't buy into either too much to avoid lock-in

- instead: **port over the existing status UI drawing code
  and adjust it to produce strings instead of directly
  drawing into the termbox buffer**

Co-authored-by: Calistoc <calistoc@protonmail.com>
Co-authored-by: InsanePrawn <insane.prawny@gmail.com>

fixes #245
fixes #220
 2021-03-14 18:24:25 +01:00
Calistoc 2c8c2cfa14 add signal 'snapshot', rename existing signal 'wakeup' to 'replication' 2021-03-14 18:16:23 +01:00
Christian Schwarz 0ceea1b792 replication: simplify parallel replication variables & expose them in config 
closes #140
 2021-03-14 17:30:10 +01:00
Christian Schwarz 07f2bfff6a build: bump release build Go version + quickcheck default to Go 1.16 2021-02-20 17:29:17 +01:00
Christian Schwarz b9ee14f6d7 endpoint.ReceiverConfig: remove obsolete TODO comment 2021-02-20 17:20:58 +01:00
InsanePrawn 393fc10a69 [#285] support setting zfs send / recv flags in the config (send: -wLcepbS, recv: -ox) 
Co-authored-by: Christian Schwarz <me@cschwarz.com>
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>

closes #285
closes #276
closes #24
 2021-02-20 17:20:45 +01:00
Christian Schwarz 1c937e58f7 zfs.NilBool: document its purpose and move it to its own package 'nodefault' 2021-02-20 17:04:57 +01:00
Christian Schwarz 70bbdfe760 zfs: ResumeToken: parse embedok, largeblockok, savedok if available 
Developed for #285 but ultimately not used for it.
 2021-02-20 17:04:57 +01:00
Christian Schwarz efe7b17d21 Update to protobuf v1.25 and grpc 1.35; bump CI to go1.12 
From:
github.com/golang/protobuf v1.3.2
google.golang.org/grpc v1.17.0

To:
github.com/golang/protobuf v1.4.3
google.golang.org/grpc v1.35.0
google.golang.org/protobuf v1.25.0

About the two protobuf packages:
https://developers.google.com/protocol-buffers/docs/reference/go/faq
> Version v1.4.0 and higher of github.com/golang/protobuf wrap the new
implementation and permit programs to adopt the new API incrementally. For
example, the well-known types defined in github.com/golang/protobuf/ptypes are
simply aliases of those defined in the newer module. Thus,
google.golang.org/protobuf/types/known/emptypb and
github.com/golang/protobuf/ptypes/empty may be used interchangeably.

Notable Code Changes in zrepl:
- generate protobufs now contain a mutex so we can't copy them by value
  anymore
- grpc.WithDialer is deprecated => use grpc.WithContextDialer instead

Go1.12 is now actually required by some of the dependencies.
 2021-01-25 00:39:01 +01:00
Christian Schwarz 166e80bb96 lazy.sh: use 'go install' + import paths from build/tools.go 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 6ac537210b daemon: avoid math/rand race by using global source 
Unless we're using the global source for math/rand, (*rand.Rand).Read
should not be called concurrently. We seed the rng in daemon.Run to
avoid ambiguity or hiding global side effects inside packages.

closes #414
 2021-01-25 00:16:01 +01:00
Christian Schwarz 596a39c0f5 bump golangci-lint to 1.35.2 and fix resulting lint errors 
GO111MODULE=on golangci-lint run ./...
endpoint/endpoint.go:487:9: S1039: unnecessary use of fmt.Sprintf (gosimple)
                panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
                      ^
platformtest/platformtest_ops.go:259:41: S1039: unnecessary use of fmt.Sprintf (gosimple)
                                return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
                                                                    ^
platformtest/platformtest_ops.go:266:41: S1039: unnecessary use of fmt.Sprintf (gosimple)
                                return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
                                                                    ^
util/optionaldeadline/optionaldeadline_test.go:97:50: SA1029: should not use built-in type string as key for value; define your own type to avoid collisions (staticcheck)
        pctx := context.WithValue(context.Background(), "key", "value")
                                                        ^
rpc/rpc_debug.go:8:5: var `debugEnabled` is unused (unused)
rpc/dataconn/dataconn_debug.go:8:5: var `debugEnabled` is unused (unused)
rpc/dataconn/frameconn/frameconn.go:42:9: S1039: unnecessary use of fmt.Sprintf (gosimple)
                panic(fmt.Sprintf("frame header is 8 bytes long"))
                      ^
platformtest/platformtest_ops.go:322:40: S1039: unnecessary use of fmt.Sprintf (gosimple)
                        return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson d118bcc717 rpc: fix data race in timeoutconn 
- `timeoutconn` handles state, yet calls to Read/Write make a copy of
  that state (non-pointer receiver) so any outbound calls will not have
  the state updated

- Even without the copy issue, the renew methods can in edge cases set a
  new deadline _after_ DisableTimeouts have been called, consider the
  following racy behavior:
    1. `renewReadDeadline` is called, checks `renewDeadlinesDisabled`
       (not disabled)
    2. `DisableTimeouts` is called, sets `renewDeadlinesDisabled`
    3. `DisableTimeouts` invokes `c.SetDeadline`
    4. `renewReadDeadline` invokes `c.SetReadDeadline`

To fix the above, the `Conn` receiver was made to be a pointer
everywhere and access to renewDeadlinesDisabled is now guarded
by an RWMutex instead of using atomics.

closes #415
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 48be4032a2 daemon: fix data race in snapjob pruner report 
closes #416
 2021-01-25 00:16:01 +01:00
Mathias Fredriksson 2d545c87cd rpc: fix read mutex unlock in dataconn stream 
closes #413
 2021-01-25 00:16:01 +01:00
Christian Schwarz 0b48ba54f8 replication/driver: simplify second-attempt step correlation code & fix statekeeping 
Before this change, the step correlation code returned early in several cases:
- did not set f.planning.done in the cases where it was a no-op
- did not set f.planning.err in the cases where correlation did not
  succeed

Reported-by: InsanePrawn <insane.prawny@gmail.com>
 2021-01-25 00:16:01 +01:00
Christian Schwarz c3d87289bb [#388] util/semaphore: fix TestSemaphore test 
fixes #388
 2021-01-24 22:28:47 +01:00
Christian Schwarz 0d96627ffb [#388] trace: make WithTaskGroup actually concurrent 2021-01-24 22:28:47 +01:00
Christian Schwarz 61acc7494a [#388] endpoint: fix incorrect use of trace.WithTaskGroup in ListAbstractionsStreamed 
Capturing behavior was broken.
 2021-01-24 13:58:23 +01:00
97 changed files with 6388 additions and 2783 deletions
Expand all files Collapse all files
.circleci/config.yml
+7 -7
View File
@@ -150,7 +150,7 @@ parameters:
release_docker_baseimage_tag:
type: string
default: "1.15"
default: "1.16"
workflows:
version: 2
@@ -160,19 +160,19 @@ workflows:
jobs:
- quickcheck-docs
- quickcheck-go: &quickcheck-go-smoketest
name: quickcheck-go-amd64-linux-1.15
goversion: &latest-go-release "1.15"
name: quickcheck-go-amd64-linux-1.16
goversion: &latest-go-release "1.16"
goos: linux
goarch: amd64
- test-go-on-latest-go-release:
goversion: *latest-go-release
- quickcheck-go:
requires:
- quickcheck-go-amd64-linux-1.15 #quickcheck-go-smoketest.name
- quickcheck-go-amd64-linux-1.16 #quickcheck-go-smoketest.name
matrix: &quickcheck-go-matrix
alias: quickcheck-go-matrix
parameters:
goversion: [*latest-go-release, "1.11"]
goversion: [*latest-go-release, "1.12"]
goos: ["linux", "freebsd"]
goarch: ["amd64", "arm64"]
exclude:
@@ -180,8 +180,8 @@ workflows:
- goversion: *latest-go-release
goos: linux
goarch: amd64
# not supported by Go 1.11
- goversion: "1.11"
# not supported by Go 1.12
- goversion: "1.12"
goos: freebsd
goarch: arm64
Makefile
+3 -2
View File
@@ -28,7 +28,7 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
GOLANGCI_LINT := golangci-lint
GOCOVMERGE := gocovmerge
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.15
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.16
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
ifneq ($(GOARM),)
@@ -286,7 +286,8 @@ cover-full:
.PHONY: generate formatcheck format
generate: generate-platform-test-list
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
build/go.mod
+5 -41
View File
@@ -3,50 +3,14 @@ module github.com/zrepl/zrepl/build
go 1.12
require (
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
github.com/alvaroloes/enumer v1.1.1
github.com/fatih/color v1.7.0 // indirect
github.com/go-critic/go-critic v0.3.4 // indirect
github.com/gogo/protobuf v1.2.1 // indirect
github.com/golang/mock v1.2.0 // indirect
github.com/golang/protobuf v1.2.0
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
github.com/golangci/golangci-lint v1.17.1
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
github.com/golangci/golangci-lint v1.35.2
github.com/golangci/misspell v0.3.4 // indirect
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
github.com/google/go-cmp v0.3.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/pkg/errors v0.8.1 // indirect
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.3.2 // indirect
github.com/stretchr/testify v1.4.0 // indirect
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad // indirect
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
)
// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
replace (
github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
google.golang.org/protobuf v1.25.0
)
build/go.sum
+530 -114
View File
@@ -1,66 +1,133 @@
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw=
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk=
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/ashanbrown/forbidigo v1.0.0 h1:QdNXBduDUopc3GW+YVYZn8jzmIMklQiCfdN2N5+dQeE=
github.com/ashanbrown/forbidigo v1.0.0/go.mod h1:PH+zMRWE15yW69fYfe7Kn8nYR6yYyafc3ntEGh2BBAg=
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a h1:/U9tbJzDRof4fOR51vwzWdIBsIH6R2yU0KG1MBRM2Js=
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8=
github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg=
github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218=
github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-critic/go-critic v0.5.3 h1:xQEweNxzBNpSqI3wotXZAixRarETng3PTG4pkcrLCOA=
github.com/go-critic/go-critic v0.5.3/go.mod h1:2Lrs1m4jtOnnG/EdezbSpAoL0F2pRW+9HWJUZ+QaktY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=
github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
@@ -69,26 +136,16 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpc
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.35.2 h1:hD1999/sq3tCPXhhI4UpunxpAAdH9pK7kDIObqoGuWA=
github.com/golangci/golangci-lint v1.35.2/go.mod h1:Sg5fFp5oLLI1B8gXfUVUSePju8XF0uWefMkuZuGIHUo=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
@@ -101,182 +158,541 @@ github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gookit/color v1.3.1/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw=
github.com/gostaticanalysis/analysisutil v0.4.1 h1:/7clKqrVfiVwiBQLM0Uke4KvXnO6JcCTS7HwF2D6wG8=
github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0=
github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI=
github.com/gostaticanalysis/comment v1.4.1 h1:xHopR5L2lRz6OsjH4R2HG5wRhW9ySl3FsHIvi5pcXwc=
github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 h1:7nkB9fLPMwtn/R6qfPcHileL/x9ydlhw8XyDrLI1ZXg=
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kulti/thelper v0.2.1 h1:H4rSHiB3ALx//SXr+k9OPqKoOw2cAZpIQwVNH1RL5T4=
github.com/kulti/thelper v0.2.1/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
github.com/kunwardeep/paralleltest v1.0.2 h1:/jJRv0TiqPoEy/Y8dQxCFJhD56uS/pnvtatgTZBHokU=
github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30=
github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ=
github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mbilski/exhaustivestruct v1.1.0 h1:4ykwscnAFeHJruT+EY3M3vdeP8uXMh0VV2E61iR7XD8=
github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4=
github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw=
github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4=
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278 h1:5gcJ7tORNCNB2QjOJF+MYjzS9aiWpxhP3gntf7RVrOQ=
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryancurrah/gomodguard v1.2.0 h1:YWfhGOrXwLGiqcC/u5EqG6YeS8nh+1fw0HEc85CVZro=
github.com/ryancurrah/gomodguard v1.2.0/go.mod h1:rNqbC4TOIdUDcVMSIpNNAzTbzXAZa6W5lnUepvuMMgQ=
github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/securego/gosec/v2 v2.5.0 h1:kjfXLeKdk98gBe2+eYRFMpC4+mxmQQtbidpiiOQ69Qc=
github.com/securego/gosec/v2 v2.5.0/go.mod h1:L/CDXVntIff5ypVHIkqPXbtRpJiNCh6c6Amn68jXDjo=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI=
github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA=
github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
github.com/tetafro/godot v1.3.2 h1:HzWC3XjadkyeuBZxkfAFNY20UVvle0YD51I6zf6RKlU=
github.com/tetafro/godot v1.3.2/go.mod h1:ah7jjYmOMnIjS9ku2krapvGQrFNtTLo9Z/qB3dGU1eU=
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d h1:3EZyvNUMsGD1QA8cu0STNn1L7I77rvhf2IhOcHYQhSw=
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65 h1:Y0bLA422kvb32uZI4fy/Plop/Tbld0l9pSzl+j1FWok=
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65/go.mod h1:T22e7iRN4LsFPZGyRLRXeF+DWVXFuV9thsyO7NjbbTI=
github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs=
github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201030010431-2feb2bb1ff51/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130 h1:8qSBr5nyKsEgkP918Pu5FFDZpTtLIjXSo6mrtdVOFfk=
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc=
honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw=
mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
build/tools.go
+3 -1
View File
@@ -2,11 +2,13 @@
package main
// the lines are parsed by lazy.sh, do not edit
import (
_ "github.com/alvaroloes/enumer"
_ "github.com/golang/protobuf/protoc-gen-go"
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
_ "github.com/wadey/gocovmerge"
_ "golang.org/x/tools/cmd/goimports"
_ "golang.org/x/tools/cmd/stringer"
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
_ "google.golang.org/protobuf/cmd/protoc-gen-go"
)
client/signal.go
+3 -3
View File
@@ -11,8 +11,8 @@ import (
)
var SignalCmd = &cli.Subcommand{
Use: "signal [wakeup|reset] JOB",
Short: "wake up a job from wait state or abort its current invocation",
Use: "signal [replication|reset|snapshot] JOB",
Short: "run a job replication, abort its current invocation, run a snapshot job",
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runSignalCmd(subcommand.Config(), args)
},
@@ -20,7 +20,7 @@ var SignalCmd = &cli.Subcommand{
func runSignalCmd(config *config.Config, args []string) error {
if len(args) != 2 {
return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
return errors.Errorf("Expected 2 arguments: [replication|reset|snapshot] JOB")
}
httpc, err := controlHttpClient(config.Global.Control.SockPath)
client/status.go
-816
View File
@@ -1,816 +0,0 @@
package client
import (
"context"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strings"
"sync"
"time"
// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
// as of tcell#252, the number of default distributed terminals is relatively limited
// additional terminal definitions can be included via side-effect import
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
"github.com/gdamore/tcell/termbox"
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/replication/report"
)
type byteProgressMeasurement struct {
time time.Time
val int64
}
type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional
changeCount int
lastChange time.Time
bpsAvg float64
}
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
if p.last == nil {
p.last = &byteProgressMeasurement{
time: time.Now(),
val: currentVal,
}
return 0, 0
}
if p.last.val != currentVal {
p.changeCount++
p.lastChange = time.Now()
}
if time.Since(p.lastChange) > 3*time.Second {
p.last = nil
return 0, 0
}
deltaV := currentVal - p.last.val
deltaT := time.Since(p.last.time)
rate := float64(deltaV) / deltaT.Seconds()
factor := 0.3
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
p.last.time = time.Now()
p.last.val = currentVal
return int64(p.bpsAvg), p.changeCount
}
type tui struct {
x, y int
indent int
lock sync.Mutex //For report and error
report map[string]*job.Status
err error
jobFilter string
replicationProgress map[string]*bytesProgressHistory // by job name
}
func newTui() tui {
return tui{
replicationProgress: make(map[string]*bytesProgressHistory),
}
}
const INDENT_MULTIPLIER = 4
func (t *tui) moveLine(dl int, col int) {
t.y += dl
t.x = t.indent*INDENT_MULTIPLIER + col
}
func (t *tui) write(text string) {
for _, c := range text {
if c == '\n' {
t.newline()
continue
}
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
t.x += 1
}
}
func (t *tui) printf(text string, a ...interface{}) {
t.write(fmt.Sprintf(text, a...))
}
func wrap(s string, width int) string {
var b strings.Builder
for len(s) > 0 {
rem := width
if rem > len(s) {
rem = len(s)
}
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
rem = idx + 1
}
untilNewline := strings.TrimRight(s[:rem], "\n\r")
s = s[rem:]
if len(untilNewline) == 0 {
continue
}
b.WriteString(untilNewline)
b.WriteString("\n")
}
return strings.TrimRight(b.String(), "\n\r")
}
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
whole := fmt.Sprintf(format, a...)
width, _ := termbox.Size()
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
t.printf(format, a...)
} else {
t.addIndent(1)
t.newline()
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
t.addIndent(-1)
}
}
func (t *tui) newline() {
t.moveLine(1, 0)
}
func (t *tui) setIndent(indent int) {
t.indent = indent
t.moveLine(0, 0)
}
func (t *tui) addIndent(indent int) {
t.indent += indent
t.moveLine(0, 0)
}
var statusFlags struct {
Raw bool
Job string
}
var StatusCmd = &cli.Subcommand{
Use: "status",
Short: "show job activity or dump as JSON for monitoring",
SetupFlags: func(f *pflag.FlagSet) {
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
},
Run: runStatus,
}
func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
if err != nil {
return err
}
if statusFlags.Raw {
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Fprintf(os.Stderr, "Received error response:\n")
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
if err != nil {
return err
}
return errors.Errorf("exit")
}
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
return err
}
return nil
}
t := newTui()
t.lock.Lock()
t.err = errors.New("Got no report yet")
t.lock.Unlock()
t.jobFilter = statusFlags.Job
err = termbox.Init()
if err != nil {
return err
}
defer termbox.Close()
update := func() {
var m daemon.Status
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
struct{}{},
&m,
)
t.lock.Lock()
t.err = err2
t.report = m.Jobs
t.lock.Unlock()
t.draw()
}
update()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
go func() {
for range ticker.C {
update()
}
}()
termbox.HideCursor()
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
loop:
for {
switch ev := termbox.PollEvent(); ev.Type {
case termbox.EventKey:
switch ev.Key {
case termbox.KeyEsc:
break loop
case termbox.KeyCtrlC:
break loop
}
case termbox.EventResize:
t.draw()
}
}
return nil
}
func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
p, ok := t.replicationProgress[jobName]
if !ok {
p = &bytesProgressHistory{}
t.replicationProgress[jobName] = p
}
return p
}
func (t *tui) draw() {
t.lock.Lock()
defer t.lock.Unlock()
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
t.x = 0
t.y = 0
t.indent = 0
if t.err != nil {
t.write(t.err.Error())
} else {
//Iterate over map in alphabetical order
keys := make([]string, 0, len(t.report))
for k := range t.report {
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
continue
}
if t.jobFilter != "" && k != t.jobFilter {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
if len(keys) == 0 {
t.setIndent(0)
t.printf("no jobs to display")
t.newline()
termbox.Flush()
return
}
for _, k := range keys {
v := t.report[k]
t.setIndent(0)
t.printf("Job: %s", k)
t.setIndent(1)
t.newline()
t.printf("Type: %s", v.Type)
t.setIndent(1)
t.newline()
if v.Type == job.TypePush || v.Type == job.TypePull {
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
if !ok || activeStatus == nil {
t.printf("ActiveSideStatus is null")
t.newline()
continue
}
t.printf("Replication:")
t.newline()
t.addIndent(1)
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
t.addIndent(-1)
t.printf("Pruning Sender:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(activeStatus.PruningSender)
t.addIndent(-1)
t.printf("Pruning Receiver:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(activeStatus.PruningReceiver)
t.addIndent(-1)
if v.Type == job.TypePush {
t.printf("Snapshotting:")
t.newline()
t.addIndent(1)
t.renderSnapperReport(activeStatus.Snapshotting)
t.addIndent(-1)
}
} else if v.Type == job.TypeSnap {
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
if !ok || snapStatus == nil {
t.printf("SnapJobStatus is null")
t.newline()
continue
}
t.printf("Pruning snapshots:")
t.newline()
t.addIndent(1)
t.renderPrunerReport(snapStatus.Pruning)
t.addIndent(-1)
t.printf("Snapshotting:")
t.newline()
t.addIndent(1)
t.renderSnapperReport(snapStatus.Snapshotting)
t.addIndent(-1)
} else if v.Type == job.TypeSource {
st := v.JobSpecific.(*job.PassiveStatus)
t.printf("Snapshotting:\n")
t.addIndent(1)
t.renderSnapperReport(st.Snapper)
t.addIndent(-1)
} else {
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
t.newline()
asYaml, err := yaml.Marshal(v.JobSpecific)
if err != nil {
t.printf("Error marshaling status to YAML: %s", err)
t.newline()
continue
}
t.write(string(asYaml))
t.newline()
continue
}
}
}
termbox.Flush()
}
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
if rep == nil {
t.printf("...\n")
return
}
if rep.WaitReconnectError != nil {
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
t.newline()
}
if !rep.WaitReconnectSince.IsZero() {
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
var until string
if rep.WaitReconnectUntil.IsZero() {
until = "waiting indefinitely"
} else {
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
}
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
rep.WaitReconnectSince, until)
} else {
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
}
t.newline()
}
// TODO visualize more than the latest attempt by folding all attempts into one
if len(rep.Attempts) == 0 {
t.printf("no attempts made yet")
return
} else {
t.printf("Attempt #%d", len(rep.Attempts))
if len(rep.Attempts) > 1 {
t.printf(". Previous attempts failed with the following statuses:")
t.newline()
t.addIndent(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
t.newline()
}
t.addIndent(-1)
} else {
t.newline()
}
}
latest := rep.Attempts[len(rep.Attempts)-1]
sort.Slice(latest.Filesystems, func(i, j int) bool {
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
})
t.printf("Status: %s", latest.State)
t.newline()
if latest.State == report.AttemptPlanningError {
t.printf("Problem: ")
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
t.newline()
} else if latest.State == report.AttemptFanOutError {
t.printf("Problem: one or more of the filesystems encountered errors")
t.newline()
}
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
// Draw global progress bar
// Progress: [---------------]
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
rate, changeCount := history.Update(replicated)
eta := time.Duration(0)
if rate > 0 {
eta = time.Duration((expected-replicated)/rate) * time.Second
}
t.write("Progress: ")
t.drawBar(50, replicated, expected, changeCount)
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
if eta != 0 {
t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
}
t.newline()
if containsInvalidSizeEstimates {
t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
t.newline()
}
if len(latest.Filesystems) == 0 {
t.write("NOTE: no filesystems were considered for replication!")
t.newline()
}
var maxFSLen int
for _, fs := range latest.Filesystems {
if len(fs.Info.Name) > maxFSLen {
maxFSLen = len(fs.Info.Name)
}
}
for _, fs := range latest.Filesystems {
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
}
}
}
func (t *tui) renderPrunerReport(r *pruner.Report) {
if r == nil {
t.printf("...\n")
return
}
state, err := pruner.StateString(r.State)
if err != nil {
t.printf("Status: %q (parse error: %q)\n", r.State, err)
return
}
t.printf("Status: %s", state)
t.newline()
if r.Error != "" {
t.printf("Error: %s\n", r.Error)
}
type commonFS struct {
*pruner.FSReport
completed bool
}
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false})
}
for i := range r.Completed {
all = append(all, commonFS{&r.Completed[i], true})
}
switch state {
case pruner.Plan:
fallthrough
case pruner.PlanErr:
return
}
if len(all) == 0 {
t.printf("nothing to do\n")
return
}
var totalDestroyCount, completedDestroyCount int
var maxFSname int
for _, fs := range all {
totalDestroyCount += len(fs.DestroyList)
if fs.completed {
completedDestroyCount += len(fs.DestroyList)
}
if maxFSname < len(fs.Filesystem) {
maxFSname = len(fs.Filesystem)
}
}
// global progress bar
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
t.write("Progress: ")
t.write("[")
t.write(times("=", progress))
t.write(">")
t.write(times("-", 80-progress))
t.write("]")
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.newline()
sort.SliceStable(all, func(i, j int) bool {
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
})
// Draw a table-like representation of 'all'
for _, fs := range all {
t.write(rightPad(fs.Filesystem, maxFSname, " "))
t.write(" ")
if !fs.SkipReason.NotSkipped() {
t.printf("skipped: %s\n", fs.SkipReason)
continue
}
if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") {
t.printf("ERROR:")
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else {
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
}
t.newline()
continue
}
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed {
t.printf("Completed %s\n", pruneRuleActionStr)
continue
}
t.write("Pending ") // whitespace is padding 10
if len(fs.DestroyList) == 1 {
t.write(fs.DestroyList[0].Name)
} else {
t.write(pruneRuleActionStr)
}
t.newline()
}
}
func (t *tui) renderSnapperReport(r *snapper.Report) {
if r == nil {
t.printf("<snapshot type does not have a report>\n")
return
}
t.printf("Status: %s", r.State)
t.newline()
if r.Error != "" {
t.printf("Error: %s\n", r.Error)
}
if !r.SleepUntil.IsZero() {
t.printf("Sleep until: %s\n", r.SleepUntil)
}
sort.Slice(r.Progress, func(i, j int) bool {
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
})
t.addIndent(1)
defer t.addIndent(-1)
dur := func(d time.Duration) string {
return d.Round(100 * time.Millisecond).String()
}
type row struct {
path, state, duration, remainder, hookReport string
}
var widths struct {
path, state, duration int
}
rows := make([]*row, len(r.Progress))
for i, fs := range r.Progress {
r := &row{
path: fs.Path,
state: fs.State.String(),
}
if fs.HooksHadError {
r.hookReport = fs.Hooks // FIXME render here, not in daemon
}
switch fs.State {
case snapper.SnapPending:
r.duration = "..."
r.remainder = ""
case snapper.SnapStarted:
r.duration = dur(time.Since(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
case snapper.SnapDone:
fallthrough
case snapper.SnapError:
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
}
rows[i] = r
if len(r.path) > widths.path {
widths.path = len(r.path)
}
if len(r.state) > widths.state {
widths.state = len(r.state)
}
if len(r.duration) > widths.duration {
widths.duration = len(r.duration)
}
}
for _, r := range rows {
path := rightPad(r.path, widths.path, " ")
state := rightPad(r.state, widths.state, " ")
duration := rightPad(r.duration, widths.duration, " ")
t.printf("%s %s %s", path, state, duration)
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
if r.hookReport != "" {
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
}
t.newline()
}
}
func times(str string, n int) (out string) {
for i := 0; i < n; i++ {
out += str
}
return
}
func rightPad(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
var arrowPositions = `>\|/`
// changeCount = 0 indicates stall / no progress
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
var completedLength int
if totalBytes > 0 {
completedLength = int(int64(length) * bytes / totalBytes)
if completedLength > length {
completedLength = length
}
} else if totalBytes == bytes {
completedLength = length
}
t.write("[")
t.write(times("=", completedLength))
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
t.write(times("-", length-completedLength))
t.write("]")
}
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
sizeEstimationImpreciseNotice := ""
if containsInvalidSizeEstimates {
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
}
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
}
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
strings.ToUpper(string(rep.State)),
rep.CurrentStep, len(rep.Steps),
ByteCountBinary(replicated), ByteCountBinary(expected),
sizeEstimationImpreciseNotice,
)
activeIndicator := " "
if active {
activeIndicator = "*"
}
t.printf("%s %s %s ",
activeIndicator,
rightPad(rep.Info.Name, maxFS, " "),
status)
next := ""
if err := rep.Error(); err != nil {
next = err.Err
} else if rep.State != report.FilesystemDone {
if nextStep := rep.NextStep(); nextStep != nil {
if nextStep.IsIncremental() {
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
} else {
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
}
attribs := []string{}
if nextStep.Info.Resumed {
attribs = append(attribs, "resumed")
}
attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
} else {
next = "" // individual FSes may still be in planning state
}
}
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
t.newline()
}
func ByteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
func humanizeDuration(duration time.Duration) string {
days := int64(duration.Hours() / 24)
hours := int64(math.Mod(duration.Hours(), 24))
minutes := int64(math.Mod(duration.Minutes(), 60))
seconds := int64(math.Mod(duration.Seconds(), 60))
var parts []string
force := false
chunks := []int64{days, hours, minutes, seconds}
for i, chunk := range chunks {
if force || chunk > 0 {
padding := 0
if force {
padding = 2
}
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
force = true
}
}
return strings.Join(parts, " ")
}
client/status/client/client.go
+105
View File
@@ -0,0 +1,105 @@
package client
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon"
)
type Client struct {
h http.Client
}
func New(network, addr string) (*Client, error) {
httpc, err := controlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
if err != nil {
return nil, err
}
return &Client{httpc}, nil
}
func (c *Client) Status() (s daemon.Status, _ error) {
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
struct{}{},
&s,
)
return s, err
}
func (c *Client) StatusRaw() ([]byte, error) {
var r json.RawMessage
err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
if err != nil {
return nil, err
}
return r, nil
}
func (c *Client) signal(job, sig string) error {
return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
struct {
Name string
Op string
}{
Name: job,
Op: sig,
},
struct{}{},
)
}
func (c *Client) SignalReplication(job string) error {
return c.signal(job, "replication")
}
func (c *Client) SignalSnapshot(job string) error {
return c.signal(job, "snapshot")
}
func (c *Client) SignalReset(job string) error {
return c.signal(job, "reset")
}
func controlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client http.Client, err error) {
return http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return dialfunc(ctx)
},
},
}, nil
}
func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
var buf bytes.Buffer
encodeErr := json.NewEncoder(&buf).Encode(req)
if encodeErr != nil {
return encodeErr
}
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
var msg bytes.Buffer
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
return errors.Errorf("%s", msg.String())
}
decodeError := json.NewDecoder(resp.Body).Decode(&res)
if decodeError != nil {
return decodeError
}
return nil
}
client/status/status.go
+94
View File
@@ -0,0 +1,94 @@
package status
import (
"context"
"os"
"time"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/client/status/client"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/util/choices"
)
type Client interface {
Status() (daemon.Status, error)
StatusRaw() ([]byte, error)
SignalReplication(job string) error
SignalSnapshot(job string) error
SignalReset(job string) error
}
type statusFlags struct {
Mode choices.Choices
Job string
Delay time.Duration
}
var statusv2Flags statusFlags
type statusv2Mode int
const (
StatusV2ModeInteractive statusv2Mode = 1 + iota
StatusV2ModeDump
StatusV2ModeRaw
StatusV2ModeLegacy
)
var Subcommand = &cli.Subcommand{
Use: "status",
Short: "retrieve & display daemon status information",
SetupFlags: func(f *pflag.FlagSet) {
statusv2Flags.Mode.Init(
"interactive", StatusV2ModeInteractive,
"dump", StatusV2ModeDump,
"raw", StatusV2ModeRaw,
"legacy", StatusV2ModeLegacy,
)
statusv2Flags.Mode.SetTypeString("mode")
statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
},
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return runStatusV2Command(ctx, subcommand.Config(), args)
},
}
func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
c, err := client.New("unix", config.Global.Control.SockPath)
if err != nil {
return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
}
mode := statusv2Flags.Mode.Value().(statusv2Mode)
if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump {
usemode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
if err != nil {
panic(err)
}
return errors.Errorf("error: stdout is not a tty, please use --mode %s", usemode)
}
switch mode {
case StatusV2ModeInteractive:
return interactive(c, statusv2Flags)
case StatusV2ModeDump:
return dump(c, statusv2Flags.Job)
case StatusV2ModeRaw:
return raw(c)
case StatusV2ModeLegacy:
return legacy(c, statusv2Flags)
default:
panic("unreachable")
}
}
client/status/status_dump.go
+70
View File
@@ -0,0 +1,70 @@
package status
import (
"fmt"
"os"
"strings"
"github.com/gdamore/tcell"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func dump(c Client, job string) error {
s, err := c.Status()
if err != nil {
return err
}
if job != "" {
if _, ok := s.Jobs[job]; !ok {
return errors.Errorf("job %q not found", job)
}
}
width := (1 << 31) - 1
wrap := false
hline := strings.Repeat("-", 80)
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
hline = strings.Repeat("-", width)
}
m := viewmodel.New()
params := viewmodel.Params{
Report: s.Jobs,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
m.Update(params)
for _, j := range m.Jobs() {
if job != "" && j.Name() != job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Println(m.SelectedJob().FullDescription())
if job != "" {
return nil
} else {
fmt.Println(hline)
}
}
return nil
}
client/status/status_interactive.go
+347
View File
@@ -0,0 +1,347 @@
package status
import (
"fmt"
"regexp"
"strings"
"sync"
"time"
"github.com/gdamore/tcell/v2"
tview "gitlab.com/tslocum/cview"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func interactive(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
jobDetailSplit := tview.NewFlex()
jobMenu := tview.NewTreeView()
jobMenuRoot := tview.NewTreeNode("jobs")
jobMenuRoot.SetSelectable(true)
jobMenu.SetRoot(jobMenuRoot)
jobMenu.SetCurrentNode(jobMenuRoot)
jobMenu.SetSelectedTextColor(tcell.ColorGreen)
jobTextDetail := tview.NewTextView()
jobTextDetail.SetWrap(false)
jobMenu.SetBorder(true)
jobTextDetail.SetBorder(true)
toolbarSplit := tview.NewFlex()
toolbarSplit.SetDirection(tview.FlexRow)
inputBarContainer := tview.NewFlex()
fsFilterInput := tview.NewInputField()
fsFilterInput.SetBorder(false)
fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
inputBarLabel := tview.NewTextView()
inputBarLabel.SetText("[::b]FILTER ")
inputBarLabel.SetDynamicColors(true)
inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
bottombar := tview.NewFlex()
bottombar.SetDirection(tview.FlexColumn)
bottombarDateView := tview.NewTextView()
bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
bottomBarStatus := tview.NewTextView()
bottomBarStatus.SetDynamicColors(true)
bottomBarStatus.SetTextAlign(tview.AlignRight)
bottombar.AddItem(bottomBarStatus, 0, 10, false)
toolbarSplit.AddItem(bottombar, 1, 0, false)
tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
var tabbable []tview.Primitive
tabbableActiveIndex := 0
tabbableRedraw := func() {
if len(tabbable) == 0 {
app.SetFocus(nil)
return
}
if tabbableActiveIndex >= len(tabbable) {
app.SetFocus(tabbable[0])
return
}
app.SetFocus(tabbable[tabbableActiveIndex])
}
tabbableCycle := func() {
if len(tabbable) == 0 {
return
}
tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
app.SetFocus(tabbable[tabbableActiveIndex])
tabbableRedraw()
}
jobMenuVisisble := false
reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
if jobMenuVisisble == setJobMenuVisible {
return
}
jobMenuVisisble = setJobMenuVisible
if setJobMenuVisible {
jobDetailSplit.RemoveItem(jobTextDetail)
jobDetailSplit.AddItem(jobMenu, 0, 1, true)
jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
tabbable = tabbableWithJobMenu
} else {
jobDetailSplit.RemoveItem(jobMenu)
tabbable = tabbableWithoutJobMenu
}
tabbableRedraw()
}
showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
preModalFocus := app.GetFocus()
m.SetDoneFunc(func(idx int, label string) {
if modalDoneFunc != nil {
modalDoneFunc(idx, label)
}
app.SetRoot(toolbarSplit, true)
app.SetFocus(preModalFocus)
app.Draw()
})
app.SetRoot(m, true)
app.Draw()
}
app.SetRoot(toolbarSplit, true)
// initial focus
tabbableActiveIndex = len(tabbable)
tabbableCycle()
reconfigureJobDetailSplit(true)
m := viewmodel.New()
params := &viewmodel.Params{
Report: nil,
SelectedJob: nil,
FSFilter: func(_ string) bool { return true },
DetailViewWidth: 100,
DetailViewWrap: false,
ShortKeybindingOverview: "[::b]Q[::-] quit [::b]<TAB>[::-] switch panes [::b]Shift+M[::-] toggle navbar [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
}
paramsMtx := &sync.Mutex{}
var redraw func()
viewmodelupdate := func(cb func(*viewmodel.Params)) {
paramsMtx.Lock()
defer paramsMtx.Unlock()
cb(params)
m.Update(*params)
}
redraw = func() {
jobs := m.Jobs()
if flag.Job != "" {
job_found := false
for _, job := range jobs {
if strings.Compare(flag.Job, job.Name()) == 0 {
jobs = []*viewmodel.Job{job}
job_found = true
break
}
}
if !job_found {
jobs = nil
}
}
redrawJobsList := false
var selectedJobN *tview.TreeNode
if len(jobMenuRoot.GetChildren()) == len(jobs) {
for i, jobN := range jobMenuRoot.GetChildren() {
if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
redrawJobsList = true
break
}
if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
selectedJobN = jobN
}
}
} else {
redrawJobsList = true
}
if redrawJobsList {
selectedJobN = nil
children := make([]*tview.TreeNode, len(jobs))
for i := range jobs {
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
jobN.SetReference(jobs[i])
jobN.SetSelectable(true)
children[i] = jobN
jobN.SetSelectedFunc(func() {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
})
})
if jobs[i] == m.SelectedJob() {
selectedJobN = jobN
}
}
jobMenuRoot.SetChildren(children)
}
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
jobMenu.SetCurrentNode(selectedJobN)
} else if selectedJobN == nil {
// select something, otherwise selection breaks (likely bug in tview)
jobMenu.SetCurrentNode(jobMenuRoot)
}
if selJ := m.SelectedJob(); selJ != nil {
jobTextDetail.SetText(selJ.FullDescription())
} else {
jobTextDetail.SetText("please select a job")
}
bottombardatestring := m.DateString()
bottombarDateView.SetText(bottombardatestring)
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
bottomBarStatus.SetText(m.BottomBarStatus())
app.Draw()
}
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
viewmodelupdate(func(p *viewmodel.Params) {
p.Report = st.Jobs
p.ReportFetchError = err
})
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
viewmodelupdate(func(p *viewmodel.Params) {
p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
})
redraw()
jobTextDetail.ScrollToBeginning()
})
jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
app.SetFocus(jobTextDetail)
})
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
viewmodelupdate(func(p *viewmodel.Params) {
_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
})
return false
})
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
if e.Key() == tcell.KeyTab {
tabbableCycle()
return nil
}
if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == '/' {
if app.GetFocus() != fsFilterInput {
app.SetFocus(fsFilterInput)
}
return e
}
if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
reconfigureJobDetailSplit(!jobMenuVisisble)
return nil
}
if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
app.Stop()
}
if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
if !ok {
return nil
}
signals := []string{"replication", "snapshot", "reset"}
clientFuncs := []func(job string) error{c.SignalReplication, c.SignalSnapshot, c.SignalReset}
sigMod := tview.NewModal()
sigMod.SetBackgroundColor(tcell.ColorDefault)
sigMod.SetBorder(true)
sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
sigMod.AddButtons(signals)
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
showModal(sigMod, func(idx int, _ string) {
go func() {
if idx == -1 {
return
}
err := clientFuncs[idx](job.Name())
if err != nil {
app.QueueUpdate(func() {
me := tview.NewModal()
me.SetText(fmt.Sprintf("signal error: %s", err))
me.AddButtons([]string{"Close"})
showModal(me, nil)
})
}
}()
})
}
return e
})
fsFilterInput.SetChangedFunc(func(searchterm string) {
viewmodelupdate(func(p *viewmodel.Params) {
p.FSFilter = func(fs string) bool {
r, err := regexp.Compile(searchterm)
if err != nil {
return true
}
return r.MatchString(fs)
}
})
redraw()
jobTextDetail.ScrollToBeginning()
})
fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEnter {
app.SetFocus(jobTextDetail)
return nil
}
return event
})
jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
// toggle wrapping
viewmodelupdate(func(p *viewmodel.Params) {
p.DetailViewWrap = !p.DetailViewWrap
})
redraw()
return nil
}
return event
})
return app.Run()
}
client/status/status_legacy.go
+128
View File
@@ -0,0 +1,128 @@
package status
import (
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gdamore/tcell/v2"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
tview "gitlab.com/tslocum/cview"
"github.com/zrepl/zrepl/client/status/viewmodel"
)
func legacy(c Client, flag statusFlags) error {
// Set this so we don't overwrite the default terminal colors
// See https://github.com/rivo/tview/blob/master/styles.go
tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
tview.Styles.PrimaryTextColor = tcell.ColorDefault
tview.Styles.BorderColor = tcell.ColorDefault
app := tview.NewApplication()
textView := tview.NewTextView()
textView.SetWrap(true)
textView.SetScrollable(true) // so that it allows us to set scroll position
textView.SetScrollBarVisibility(tview.ScrollBarNever)
app.SetRoot(textView, true)
width := (1 << 31) - 1
wrap := false
if isatty.IsTerminal(os.Stdout.Fd()) {
wrap = true
screen, err := tcell.NewScreen()
if err != nil {
return errors.Wrap(err, "get terminal dimensions")
}
if err := screen.Init(); err != nil {
return errors.Wrap(err, "init screen")
}
width, _ = screen.Size()
screen.Fini()
}
paramsMtx := &sync.Mutex{}
params := viewmodel.Params{
Report: nil,
ReportFetchError: nil,
SelectedJob: nil,
FSFilter: func(s string) bool { return true },
DetailViewWidth: width,
DetailViewWrap: wrap,
ShortKeybindingOverview: "",
}
redraw := func() {
textView.Clear()
paramsMtx.Lock()
defer paramsMtx.Unlock()
if params.ReportFetchError != nil {
fmt.Fprintln(textView, params.ReportFetchError.Error())
} else if params.Report != nil {
m := viewmodel.New()
m.Update(params)
for _, j := range m.Jobs() {
if flag.Job != "" && j.Name() != flag.Job {
continue
}
params.SelectedJob = j
m.Update(params)
fmt.Fprintln(textView, m.SelectedJob().FullDescription())
if flag.Job != "" {
break
} else {
hline := strings.Repeat("-", params.DetailViewWidth)
fmt.Fprintln(textView, hline)
}
}
} else {
fmt.Fprintln(textView, "waiting for request results")
}
textView.ScrollToBeginning()
}
app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
// sync resizes to `params`
paramsMtx.Lock()
_, _, newWidth, _ := textView.GetInnerRect()
if newWidth != params.DetailViewWidth {
params.DetailViewWidth = newWidth
app.QueueUpdateDraw(redraw)
}
paramsMtx.Unlock()
textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
return false
})
go func() {
defer func() {
if err := recover(); err != nil {
app.Suspend(func() {
panic(err)
})
}
}()
for {
st, err := c.Status()
paramsMtx.Lock()
params.Report = st.Jobs
params.ReportFetchError = err
paramsMtx.Unlock()
app.QueueUpdateDraw(redraw)
time.Sleep(flag.Delay)
}
}()
return app.Run()
}
client/status/status_raw.go
+18
View File
@@ -0,0 +1,18 @@
package status
import (
"bytes"
"io"
"os"
)
func raw(c Client) error {
b, err := c.StatusRaw()
if err != nil {
return err
}
if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
return err
}
return nil
}
client/status/viewmodel/bytecountbinary.go
+18
View File
@@ -0,0 +1,18 @@
package viewmodel
import (
"fmt"
)
func ByteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
client/status/viewmodel/bytesprogresshistory.go
+48
View File
@@ -0,0 +1,48 @@
package viewmodel
import "time"
type byteProgressMeasurement struct {
time time.Time
val int64
}
type bytesProgressHistory struct {
last *byteProgressMeasurement // pointer as poor man's optional
changeCount int
lastChange time.Time
bpsAvg float64
}
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
if p.last == nil {
p.last = &byteProgressMeasurement{
time: time.Now(),
val: currentVal,
}
return 0, 0
}
if p.last.val != currentVal {
p.changeCount++
p.lastChange = time.Now()
}
if time.Since(p.lastChange) > 3*time.Second {
p.last = nil
return 0, 0
}
deltaV := currentVal - p.last.val
deltaT := time.Since(p.last.time)
rate := float64(deltaV) / deltaT.Seconds()
factor := 0.3
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
p.last.time = time.Now()
p.last.val = currentVal
return int64(p.bpsAvg), p.changeCount
}
client/status/viewmodel/render.go
+582
View File
@@ -0,0 +1,582 @@
package viewmodel
import (
"fmt"
"math"
"sort"
"strings"
"time"
"github.com/go-playground/validator/v10"
yaml "github.com/zrepl/yaml-config"
"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
"github.com/zrepl/zrepl/daemon"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/replication/report"
)
type M struct {
jobs map[string]*Job
jobsList []*Job
selectedJob *Job
dateString string
bottomBarStatus string
}
type Job struct {
// long-lived
name string
byteProgress *bytesProgressHistory
lastStatus *job.Status
fulldescription string
}
func New() *M {
return &M{
jobs: make(map[string]*Job),
jobsList: make([]*Job, 0),
selectedJob: nil,
}
}
type FilterFunc func(string) bool
type Params struct {
Report map[string]*job.Status
ReportFetchError error
SelectedJob *Job
FSFilter FilterFunc `validate:"required"`
DetailViewWidth int `validate:"gte=1"`
DetailViewWrap bool
ShortKeybindingOverview string
}
var validate = validator.New()
func (m *M) Update(p Params) {
if err := validate.Struct(p); err != nil {
panic(err)
}
if p.ReportFetchError != nil {
m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
} else {
m.bottomBarStatus = p.ShortKeybindingOverview
for jobname, st := range p.Report {
// TODO handle job renames & deletions
j, ok := m.jobs[jobname]
if !ok {
j = &Job{
name: jobname,
byteProgress: &bytesProgressHistory{},
}
m.jobs[jobname] = j
m.jobsList = append(m.jobsList, j)
}
j.lastStatus = st
}
}
// filter out internal jobs
var jobsList []*Job
for _, j := range m.jobsList {
if daemon.IsInternalJobName(j.name) {
continue
}
jobsList = append(jobsList, j)
}
m.jobsList = jobsList
// determinism!
sort.Slice(m.jobsList, func(i, j int) bool {
return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
})
// try to not lose the selected job
m.selectedJob = nil
for _, j := range m.jobsList {
j.updateFullDescription(p)
if j == p.SelectedJob {
m.selectedJob = j
}
}
m.dateString = time.Now().Format(time.RFC3339)
}
func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
func (m *M) Jobs() []*Job { return m.jobsList }
// may be nil
func (m *M) SelectedJob() *Job { return m.selectedJob }
func (m *M) DateString() string { return m.dateString }
func (j *Job) updateFullDescription(p Params) {
width := p.DetailViewWidth
if !p.DetailViewWrap {
width = 10000000 // FIXME
}
b := stringbuilder.New(stringbuilder.Config{
IndentMultiplier: 3,
Width: width,
})
drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
j.fulldescription = b.String()
}
func (j *Job) JobTreeTitle() string {
return j.name
}
func (j *Job) FullDescription() string {
return j.fulldescription
}
func (j *Job) Name() string {
return j.name
}
func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
t.Printf("Job: %s\n", name)
t.Printf("Type: %s\n\n", v.Type)
if v.Type == job.TypePush || v.Type == job.TypePull {
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
if !ok || activeStatus == nil {
t.Printf("ActiveSideStatus is null")
t.Newline()
return
}
t.Printf("Replication:")
t.AddIndentAndNewline(1)
renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Sender:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Pruning Receiver:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
t.AddIndentAndNewline(-1)
if v.Type == job.TypePush {
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
}
} else if v.Type == job.TypeSnap {
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
if !ok || snapStatus == nil {
t.Printf("SnapJobStatus is null")
t.Newline()
return
}
t.Printf("Pruning snapshots:")
t.AddIndentAndNewline(1)
renderPrunerReport(t, snapStatus.Pruning, fsfilter)
t.AddIndentAndNewline(-1)
t.Printf("Snapshotting:")
t.AddIndentAndNewline(1)
renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
t.AddIndentAndNewline(-1)
} else if v.Type == job.TypeSource {
st := v.JobSpecific.(*job.PassiveStatus)
t.Printf("Snapshotting:\n")
t.AddIndent(1)
renderSnapperReport(t, st.Snapper, fsfilter)
t.AddIndentAndNewline(-1)
} else {
t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
t.Newline()
asYaml, err := yaml.Marshal(v.JobSpecific)
if err != nil {
t.Printf("Error marshaling status to YAML: %s", err)
t.Newline()
return
}
t.Write(string(asYaml))
t.Newline()
}
}
func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, active bool, maxFS int) {
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
sizeEstimationImpreciseNotice := ""
if containsInvalidSizeEstimates {
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
}
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
}
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
strings.ToUpper(string(rep.State)),
rep.CurrentStep, len(rep.Steps),
ByteCountBinary(replicated), ByteCountBinary(expected),
sizeEstimationImpreciseNotice,
)
activeIndicator := " "
if active {
activeIndicator = "*"
}
t.AddIndent(1)
t.Printf("%s %s %s ",
activeIndicator,
stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
status)
next := ""
if err := rep.Error(); err != nil {
next = err.Err
} else if rep.State != report.FilesystemDone {
if nextStep := rep.NextStep(); nextStep != nil {
if nextStep.IsIncremental() {
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
} else {
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
}
attribs := []string{}
if nextStep.Info.Resumed {
attribs = append(attribs, "resumed")
}
attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
} else {
next = "" // individual FSes may still be in planning state
}
}
t.Printf("%s", next)
t.AddIndent(-1)
t.Newline()
}
func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
if rep == nil {
t.Printf("...\n")
return
}
if rep.WaitReconnectError != nil {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
t.Newline()
}
if !rep.WaitReconnectSince.IsZero() {
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
var until string
if rep.WaitReconnectUntil.IsZero() {
until = "waiting indefinitely"
} else {
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
}
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
rep.WaitReconnectSince, until)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
}
t.Newline()
}
// TODO visualize more than the latest attempt by folding all attempts into one
if len(rep.Attempts) == 0 {
t.Printf("no attempts made yet")
return
} else {
t.Printf("Attempt #%d", len(rep.Attempts))
if len(rep.Attempts) > 1 {
t.Printf(". Previous attempts failed with the following statuses:")
t.AddIndentAndNewline(1)
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
}
t.AddIndentAndNewline(-1)
} else {
t.Newline()
}
}
latest := rep.Attempts[len(rep.Attempts)-1]
sort.Slice(latest.Filesystems, func(i, j int) bool {
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
})
// apply filter
filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
for _, fs := range latest.Filesystems {
if !fsfilter(fs.Info.Name) {
continue
}
filtered = append(filtered, fs)
}
latest.Filesystems = filtered
t.Printf("Status: %s", latest.State)
t.Newline()
if !latest.FinishAt.IsZero() {
t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
} else {
t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
}
if latest.State == report.AttemptPlanningError {
t.Printf("Problem: ")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
t.Newline()
} else if latest.State == report.AttemptFanOutError {
t.Printf("Problem: one or more of the filesystems encountered errors")
t.Newline()
}
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
// Draw global progress bar
// Progress: [---------------]
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
rate, changeCount := history.Update(replicated)
t.Write("Progress: ")
t.DrawBar(50, replicated, expected, changeCount)
t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
t.Newline()
if containsInvalidSizeEstimates {
t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
t.Newline()
}
if len(latest.Filesystems) == 0 {
t.Write("NOTE: no filesystems were considered for replication!")
t.Newline()
}
var maxFSLen int
for _, fs := range latest.Filesystems {
if len(fs.Info.Name) > maxFSLen {
maxFSLen = len(fs.Info.Name)
}
}
for _, fs := range latest.Filesystems {
printFilesystemStatus(t, fs, false, maxFSLen) // FIXME bring 'active' flag back
}
}
}
func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("...\n")
return
}
state, err := pruner.StateString(r.State)
if err != nil {
t.Printf("Status: %q (parse error: %q)\n", r.State, err)
return
}
t.Printf("Status: %s", state)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
type commonFS struct {
*pruner.FSReport
completed bool
}
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
for i := range r.Pending {
all = append(all, commonFS{&r.Pending[i], false})
}
for i := range r.Completed {
all = append(all, commonFS{&r.Completed[i], true})
}
// filter all
filtered := make([]commonFS, 0, len(all))
for _, fs := range all {
if fsfilter(fs.FSReport.Filesystem) {
filtered = append(filtered, fs)
}
}
all = filtered
switch state {
case pruner.Plan:
fallthrough
case pruner.PlanErr:
return
}
if len(all) == 0 {
t.Printf("nothing to do\n")
return
}
var totalDestroyCount, completedDestroyCount int
var maxFSname int
for _, fs := range all {
totalDestroyCount += len(fs.DestroyList)
if fs.completed {
completedDestroyCount += len(fs.DestroyList)
}
if maxFSname < len(fs.Filesystem) {
maxFSname = len(fs.Filesystem)
}
}
// global progress bar
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
t.Write("Progress: ")
t.Write("[")
t.Write(stringbuilder.Times("=", progress))
t.Write(">")
t.Write(stringbuilder.Times("-", 80-progress))
t.Write("]")
t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
t.Newline()
sort.SliceStable(all, func(i, j int) bool {
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
})
// Draw a table-like representation of 'all'
for _, fs := range all {
t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
t.Write(" ")
if !fs.SkipReason.NotSkipped() {
t.Printf("skipped: %s\n", fs.SkipReason)
continue
}
if fs.LastError != "" {
if strings.ContainsAny(fs.LastError, "\r\n") {
t.Printf("ERROR:")
t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
} else {
t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
}
t.Newline()
continue
}
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
len(fs.DestroyList), len(fs.SnapshotList))
if fs.completed {
t.Printf("Completed %s\n", pruneRuleActionStr)
continue
}
t.Write("Pending ") // whitespace is padding 10
if len(fs.DestroyList) == 1 {
t.Write(fs.DestroyList[0].Name)
} else {
t.Write(pruneRuleActionStr)
}
t.Newline()
}
}
func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
if r == nil {
t.Printf("<snapshot type does not have a report>\n")
return
}
t.Printf("Status: %s", r.State)
t.Newline()
if r.Error != "" {
t.Printf("Error: %s\n", r.Error)
}
if !r.SleepUntil.IsZero() {
t.Printf("Sleep until: %s\n", r.SleepUntil)
}
sort.Slice(r.Progress, func(i, j int) bool {
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
})
dur := func(d time.Duration) string {
return d.Round(100 * time.Millisecond).String()
}
type row struct {
path, state, duration, remainder, hookReport string
}
var widths struct {
path, state, duration int
}
rows := make([]*row, 0, len(r.Progress))
for _, fs := range r.Progress {
if !fsfilter(fs.Path) {
continue
}
r := &row{
path: fs.Path,
state: fs.State.String(),
}
if fs.HooksHadError {
r.hookReport = fs.Hooks // FIXME render here, not in daemon
}
switch fs.State {
case snapper.SnapPending:
r.duration = "..."
r.remainder = ""
case snapper.SnapStarted:
r.duration = dur(time.Since(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
case snapper.SnapDone:
fallthrough
case snapper.SnapError:
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
}
rows = append(rows, r)
if len(r.path) > widths.path {
widths.path = len(r.path)
}
if len(r.state) > widths.state {
widths.state = len(r.state)
}
if len(r.duration) > widths.duration {
widths.duration = len(r.duration)
}
}
for _, r := range rows {
path := stringbuilder.RightPad(r.path, widths.path, " ")
state := stringbuilder.RightPad(r.state, widths.state, " ")
duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
t.Printf("%s %s %s", path, state, duration)
t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
if r.hookReport != "" {
t.PrintfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
}
t.Newline()
}
}
client/status/viewmodel/stringbuilder/stringbuilder.go
+119
View File
@@ -0,0 +1,119 @@
package stringbuilder
import (
"fmt"
"strings"
"github.com/go-playground/validator/v10"
)
type B struct {
// const
indentMultiplier int
// mut
sb *strings.Builder
indent int
width int
x, y int
}
type Config struct {
IndentMultiplier int `validate:"gte=1"`
Width int `validate:"gte=1"`
}
var validate = validator.New()
func New(config Config) *B {
if err := validate.Struct(config); err != nil {
panic(err)
}
return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
}
func (b *B) String() string { return b.sb.String() }
func (w *B) Newline() {
w.Write("\n")
}
func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
if strings.ContainsAny(whole, "\n\r") {
w.AddIndent(1)
defer w.AddIndent(-1)
}
w.Write(whole)
}
func (w *B) Printf(format string, args ...interface{}) {
whole := fmt.Sprintf(format, args...)
w.Write(whole)
}
func (t *B) AddIndent(delta int) {
t.indent += delta * t.indentMultiplier
}
func (t *B) AddIndentAndNewline(delta int) {
t.indent += delta * t.indentMultiplier
t.Write("\n")
}
func (w *B) Write(s string) {
for _, c := range s {
if c == '\n' {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
w.y++
continue
}
if w.x >= w.width {
fmt.Fprint(w.sb, "\n")
w.x = 0
fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
w.x = w.indent
}
fmt.Fprintf(w.sb, "%c", c)
w.x++
}
}
func Times(str string, n int) (out string) {
for i := 0; i < n; i++ {
out += str
}
return
}
func RightPad(str string, length int, pad string) string {
if len(str) > length {
return str[:length]
}
return str + strings.Repeat(pad, length-len(str))
}
// changeCount = 0 indicates stall / no progress
func (w *B) DrawBar(length int, bytes, totalBytes int64, changeCount int) {
const arrowPositions = `>\|/`
var completedLength int
if totalBytes > 0 {
completedLength = int(int64(length) * bytes / totalBytes)
if completedLength > length {
completedLength = length
}
} else if totalBytes == bytes {
completedLength = length
}
w.Write("[")
w.Write(Times("=", completedLength))
w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
w.Write(Times("-", length-completedLength))
w.Write("]")
}
client/zfsabstractions_list.go
+1 -1
View File
@@ -52,9 +52,9 @@ func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
var line chainlock.L
var wg sync.WaitGroup
defer wg.Wait()
wg.Add(1)
// print results
wg.Add(1)
go func() {
defer wg.Done()
enc := json.NewEncoder(os.Stdout)
config/config.go
+28 -3
View File
@@ -12,6 +12,8 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
type Config struct {
@@ -77,19 +79,28 @@ type SnapJob struct {
}
type SendOptions struct {
Encrypted bool `yaml:"encrypted,optional,default=false"`
Encrypted bool `yaml:"encrypted,optional,default=false"`
Raw bool `yaml:"raw,optional,default=false"`
SendProperties bool `yaml:"send_properties,optional,default=false"`
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
Compressed bool `yaml:"compressed,optional,default=false"`
EmbeddedData bool `yaml:"embbeded_data,optional,default=false"`
Saved bool `yaml:"saved,optional,default=false"`
}
type RecvOptions struct {
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
// Encrypted bool `yaml:"may_encrypted"`
// Future:
// Reencrypt bool `yaml:"reencrypt"`
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
}
type Replication struct {
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
}
type ReplicationOptionsProtection struct {
@@ -97,6 +108,20 @@ type ReplicationOptionsProtection struct {
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
}
type ReplicationOptionsConcurrency struct {
Steps int `yaml:"steps,optional,default=1"`
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
}
func (l *RecvOptions) SetDefault() {
*l = RecvOptions{Properties: &PropertyRecvOptions{}}
}
type PropertyRecvOptions struct {
Inherit []zfsprop.Property `yaml:"inherit,optional"`
Override map[zfsprop.Property]string `yaml:"override,optional"`
}
type PushJob struct {
ActiveJob `yaml:",inline"`
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
config/config_recv_test.go
+134
View File
@@ -0,0 +1,134 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
func TestRecvOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: pull
connect:
type: local
listener_name: foo
client_identity: bar
root_fs: "zreplplatformtest"
%s
interval: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
recv_properties_empty := `
recv:
properties:
`
recv_inherit_empty := `
recv:
properties:
inherit:
`
recv_inherit := `
recv:
properties:
inherit:
- testprop
`
recv_override_empty := `
recv:
properties:
override:
`
recv_override := `
recv:
properties:
override:
testprop2: "test123"
`
recv_override_and_inherit := `
recv:
properties:
inherit:
- testprop
override:
testprop2: "test123"
`
recv_empty := `
recv: {}
`
recv_not_specified := `
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
t.Run("recv_inherit_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit_empty))
assert.NotNil(t, c)
})
t.Run("recv_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
})
t.Run("recv_override_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_empty))
assert.NotNil(t, c)
})
t.Run("recv_override", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override))
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
require.Len(t, override, 1)
require.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_override_and_inherit", func(t *testing.T) {
c := testValidConfig(t, fill(recv_override_and_inherit))
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
assert.NotEmpty(t, inherit)
assert.Contains(t, inherit, zfsprop.Property("testprop"))
assert.NotEmpty(t, override)
assert.Equal(t, "test123", override["testprop2"])
})
t.Run("recv_properties_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_properties_empty))
assert.NotNil(t, c)
})
t.Run("recv_empty", func(t *testing.T) {
c := testValidConfig(t, fill(recv_empty))
assert.NotNil(t, c)
})
t.Run("send_not_specified", func(t *testing.T) {
c := testValidConfig(t, fill(recv_not_specified))
assert.NotNil(t, c)
})
}
config/config_send_test.go
+80 -8
View File
@@ -38,7 +38,41 @@ jobs:
encrypted: true
`
encrypted_unspecified := `
raw_true := `
send:
raw: true
`
raw_false := `
send:
raw: false
`
raw_and_encrypted := `
send:
encrypted: true
raw: true
`
properties_and_encrypted := `
send:
encrypted: true
send_properties: true
`
properties_true := `
send:
send_properties: true
`
properties_false := `
send:
send_properties: false
`
send_empty := `
send: {}
`
@@ -46,28 +80,66 @@ jobs:
`
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
var c *Config
t.Run("encrypted_false", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_false))
c := testValidConfig(t, fill(encrypted_false))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("encrypted_true", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_true))
c := testValidConfig(t, fill(encrypted_true))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, encrypted)
})
t.Run("encrypted_unspecified", func(t *testing.T) {
c = testValidConfig(t, fill(encrypted_unspecified))
t.Run("send_empty", func(t *testing.T) {
c := testValidConfig(t, fill(send_empty))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, false, encrypted)
})
t.Run("properties_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(properties_and_encrypted))
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, encrypted)
assert.Equal(t, true, properties)
})
t.Run("properties_false", func(t *testing.T) {
c := testValidConfig(t, fill(properties_false))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, false, properties)
})
t.Run("properties_true", func(t *testing.T) {
c := testValidConfig(t, fill(properties_true))
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
assert.Equal(t, true, properties)
})
t.Run("raw_true", func(t *testing.T) {
c := testValidConfig(t, fill(raw_true))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, true, raw)
})
t.Run("raw_false", func(t *testing.T) {
c := testValidConfig(t, fill(raw_false))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
assert.Equal(t, false, raw)
})
t.Run("raw_and_encrypted", func(t *testing.T) {
c := testValidConfig(t, fill(raw_and_encrypted))
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
assert.Equal(t, true, raw)
assert.Equal(t, true, encrypted)
})
t.Run("send_not_specified", func(t *testing.T) {
c, err := testConfig(t, fill(send_not_specified))
assert.NoError(t, err)
c := testValidConfig(t, fill(send_not_specified))
assert.NotNil(t, c)
})
config/samples/quickstart_backup_to_external_disk.yml
+1 -1
View File
@@ -40,7 +40,7 @@ jobs:
# This job pushes to the local sink defined in job `backuppool_sink`.
# We trigger replication manually from the command line / udev rules using
# `zrepl signal wakeup push_to_drive`
# `zrepl signal replication push_to_drive`
- type: push
name: push_to_drive
connect:
daemon/control.go
+4 -2
View File
@@ -143,10 +143,12 @@ func (j *controlJob) Run(ctx context.Context) {
var err error
switch req.Op {
case "wakeup":
err = j.jobs.wakeup(req.Name)
case "replication":
err = j.jobs.doreplication(req.Name)
case "reset":
err = j.jobs.reset(req.Name)
case "snapshot":
err = j.jobs.dosnapshot(req.Name)
default:
err = fmt.Errorf("operation %q is invalid", req.Op)
}
daemon/daemon.go
+35 -12
View File
@@ -3,6 +3,7 @@ package daemon
import (
"context"
"fmt"
"math/rand"
"os"
"os/signal"
"strings"
@@ -19,8 +20,9 @@ import (
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/job/doreplication"
"github.com/zrepl/zrepl/daemon/job/dosnapshot"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/version"
@@ -38,6 +40,12 @@ func Run(ctx context.Context, conf *config.Config) error {
cancel()
}()
// The math/rand package is used presently for generating trace IDs, we
// seed it with the current time and pid so that the IDs are mostly
// unique.
rand.Seed(time.Now().UnixNano())
rand.Seed(int64(os.Getpid()))
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
if err != nil {
return errors.Wrap(err, "cannot build logging from config")
@@ -124,17 +132,19 @@ type jobs struct {
wg sync.WaitGroup
// m protects all fields below it
m sync.RWMutex
wakeups map[string]wakeup.Func // by Job.Name
resets map[string]reset.Func // by Job.Name
jobs map[string]job.Job
m sync.RWMutex
doreplications map[string]doreplication.Func // by Job.Name
resets map[string]reset.Func // by Job.Name
dosnapshots map[string]dosnapshot.Func // by Job.Name
jobs map[string]job.Job
}
func newJobs() *jobs {
return &jobs{
wakeups: make(map[string]wakeup.Func),
resets: make(map[string]reset.Func),
jobs: make(map[string]job.Job),
doreplications: make(map[string]doreplication.Func),
resets: make(map[string]reset.Func),
dosnapshots: make(map[string]dosnapshot.Func),
jobs: make(map[string]job.Job),
}
}
@@ -183,11 +193,11 @@ func (s *jobs) status() map[string]*job.Status {
return ret
}
func (s *jobs) wakeup(job string) error {
func (s *jobs) doreplication(job string) error {
s.m.RLock()
defer s.m.RUnlock()
wu, ok := s.wakeups[job]
wu, ok := s.doreplications[job]
if !ok {
return errors.Errorf("Job %s does not exist", job)
}
@@ -205,6 +215,17 @@ func (s *jobs) reset(job string) error {
return wu()
}
func (s *jobs) dosnapshot(job string) error {
s.m.RLock()
defer s.m.RUnlock()
wu, ok := s.dosnapshots[job]
if !ok {
return errors.Errorf("Job %s does not exist", job)
}
return wu()
}
const (
jobNamePrometheus = "_prometheus"
jobNameControl = "_control"
@@ -235,10 +256,12 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
s.jobs[jobName] = j
ctx = zfscmd.WithJobID(ctx, j.Name())
ctx, wakeup := wakeup.Context(ctx)
ctx, doreplication := doreplication.Context(ctx)
ctx, resetFunc := reset.Context(ctx)
s.wakeups[jobName] = wakeup
ctx, dosnapshotFunc := dosnapshot.Context(ctx)
s.doreplications[jobName] = doreplication
s.resets[jobName] = resetFunc
s.dosnapshots[jobName] = dosnapshotFunc
s.wg.Add(1)
go func() {
daemon/job/active.go
+41 -15
View File
@@ -11,10 +11,11 @@ import (
"github.com/prometheus/common/log"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job/doreplication"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -33,6 +34,8 @@ type ActiveSide struct {
name endpoint.JobID
connecter transport.Connecter
replicationDriverConfig driver.Config
prunerFactory *pruner.PrunerFactory
promRepStateSecs *prometheus.HistogramVec // labels: state
@@ -86,7 +89,7 @@ type activeMode interface {
SenderReceiver() (logic.Sender, logic.Receiver)
Type() Type
PlannerPolicy() logic.PlannerPolicy
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
RunPeriodic(ctx context.Context, replicationCommon chan<- struct{})
SnapperReport() *snapper.Report
ResetConnectBackoff()
}
@@ -128,8 +131,8 @@ func (m *modePush) Type() Type { return TypePush }
func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
m.snapper.Run(ctx, wakeUpCommon)
func (m *modePush) RunPeriodic(ctx context.Context, replicationCommon chan<- struct{}) {
m.snapper.Run(ctx, replicationCommon)
}
func (m *modePush) SnapperReport() *snapper.Report {
@@ -159,8 +162,12 @@ func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.Job
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
ReplicationConfig: *replicationConfig,
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
ReplicationConfig: replicationConfig,
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
}
if err := m.plannerPolicy.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build planner policy")
}
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
@@ -207,10 +214,10 @@ func (*modePull) Type() Type { return TypePull }
func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
func (m *modePull) RunPeriodic(ctx context.Context, replicationCommon chan<- struct{}) {
if m.interval.Manual {
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
// "waiting for wakeups" is printed in common ActiveSide.do
// "waiting for wakeup replications" is printed in common ActiveSide.do
return
}
t := time.NewTicker(m.interval.Interval)
@@ -219,12 +226,12 @@ func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
select {
case <-t.C:
select {
case wakeUpCommon <- struct{}{}:
case replicationCommon <- struct{}{}:
default:
GetLogger(ctx).
WithField("pull_interval", m.interval).
Warn("pull job took longer than pull interval")
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
replicationCommon <- struct{}{} // block anyways, to queue up the wakeup replication
}
case <-ctx.Done():
return
@@ -254,8 +261,12 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.DontCare,
ReplicationConfig: *replicationConfig,
EncryptedSend: logic.DontCare,
ReplicationConfig: replicationConfig,
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
}
if err := m.plannerPolicy.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build planner policy")
}
m.receiverConfig, err = buildReceiverConfig(in, jobID)
@@ -266,6 +277,16 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
return m, nil
}
func replicationDriverConfigFromConfig(in *config.Replication) (c driver.Config, err error) {
c = driver.Config{
StepQueueConcurrency: in.Concurrency.Steps,
MaxAttempts: envconst.Int("ZREPL_REPLICATION_MAX_ATTEMPTS", 3),
ReconnectHardFailTimeout: envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute),
}
err = c.Validate()
return c, err
}
func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
j = &ActiveSide{}
@@ -326,6 +347,11 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
return nil, err
}
j.replicationDriverConfig, err = replicationDriverConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.Wrap(err, "cannot build replication driver config")
}
return j, nil
}
@@ -409,13 +435,13 @@ func (j *ActiveSide) Run(ctx context.Context) {
invocationCount := 0
outer:
for {
log.Info("wait for wakeups")
log.Info("wait for replications")
select {
case <-ctx.Done():
log.WithError(ctx.Err()).Info("context")
break outer
case <-wakeup.Wait(ctx):
case <-doreplication.Wait(ctx):
j.mode.ResetConnectBackoff()
case <-periodicDone:
}
@@ -459,7 +485,7 @@ func (j *ActiveSide) do(ctx context.Context) {
*tasks = activeSideTasks{}
tasks.replicationCancel = func() { repCancel(); endSpan() }
tasks.replicationReport, repWait = replication.Do(
ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
)
tasks.state = ActiveSideReplicating
})
daemon/job/build_jobs_sendrecvoptions.go
+17 -4
View File
@@ -6,6 +6,7 @@ import (
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -20,11 +21,19 @@ func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.Sen
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
sendOpts := in.GetSendOptions()
return &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.GetSendOptions().Encrypted},
JobID: jobID,
FSF: fsf,
JobID: jobID,
Encrypt: &nodefault.Bool{B: sendOpts.Encrypted},
SendRaw: sendOpts.Raw,
SendProperties: sendOpts.SendProperties,
SendBackupProperties: sendOpts.BackupProperties,
SendLargeBlocks: sendOpts.LargeBlocks,
SendCompressed: sendOpts.Compressed,
SendEmbeddedData: sendOpts.EmbeddedData,
SendSaved: sendOpts.Saved,
}, nil
}
@@ -43,10 +52,14 @@ func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoi
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
}
recvOpts := in.GetRecvOptions()
rc = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootFs,
AppendClientIdentity: in.GetAppendClientIdentity(),
InheritProperties: recvOpts.Properties.Inherit,
OverrideProperties: recvOpts.Properties.Override,
}
if err := rc.Validate(); err != nil {
return rc, errors.Wrap(err, "cannot build receiver config")
daemon/job/build_jobs_test.go
+110
View File
@@ -144,3 +144,113 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
}
}
func TestReplicationOptions(t *testing.T) {
tmpl := `
jobs:
- name: foo
type: push
connect:
type: local
listener_name: foo
client_identity: bar
filesystems: {"<": true}
%s
snapshotting:
type: manual
pruning:
keep_sender:
- type: last_n
count: 10
keep_receiver:
- type: last_n
count: 10
`
type Test struct {
name string
input string
expectOk func(t *testing.T, a *ActiveSide, m *modePush)
expectError bool
}
tests := []Test{
{
name: "defaults",
input: `
replication: {}
`,
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {},
},
{
name: "steps_zero",
input: `
replication:
concurrency:
steps: 0
`,
expectError: true,
},
{
name: "size_estimates_zero",
input: `
replication:
concurrency:
size_estimates: 0
`,
expectError: true,
},
{
name: "custom_values",
input: `
replication:
concurrency:
steps: 23
size_estimates: 42
`,
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {
assert.Equal(t, 23, a.replicationDriverConfig.StepQueueConcurrency)
assert.Equal(t, 42, m.plannerPolicy.SizeEstimationConcurrency)
},
},
{
name: "negative_values_forbidden",
input: `
replication:
concurrency:
steps: -23
size_estimates: -42
`,
expectError: true,
},
}
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
for _, ts := range tests {
t.Run(ts.name, func(t *testing.T) {
assert.True(t, (ts.expectError) != (ts.expectOk != nil))
cstr := fill(ts.input)
t.Logf("testing config:\n%s", cstr)
c, err := config.ParseConfigBytes([]byte(cstr))
require.NoError(t, err)
jobs, err := JobsFromConfig(c)
if ts.expectOk != nil {
require.NoError(t, err)
require.NotNil(t, c)
require.NoError(t, err)
require.Len(t, jobs, 1)
a := jobs[0].(*ActiveSide)
m := a.mode.(*modePush)
ts.expectOk(t, a, m)
} else if ts.expectError {
require.Error(t, err)
} else {
t.Fatalf("test must define expectOk or expectError")
}
})
}
}
daemon/job/doreplication/doreplication.go
+35
View File
@@ -0,0 +1,35 @@
package doreplication
import (
"context"
"errors"
)
type contextKey int
const contextKeyReplication contextKey = iota
func Wait(ctx context.Context) <-chan struct{} {
wc, ok := ctx.Value(contextKeyReplication).(chan struct{})
if !ok {
wc = make(chan struct{})
}
return wc
}
type Func func() error
var AlreadyReplicating = errors.New("already replicating")
func Context(ctx context.Context) (context.Context, Func) {
wc := make(chan struct{})
wuf := func() error {
select {
case wc <- struct{}{}:
return nil
default:
return AlreadyReplicating
}
}
return context.WithValue(ctx, contextKeyReplication, wc), wuf
}
daemon/job/wakeup/wakeup.go → daemon/job/dosnapshot/dosnapshot.go
+6 -6
View File
@@ -1,4 +1,4 @@
package wakeup
package dosnapshot
import (
"context"
@@ -7,10 +7,10 @@ import (
type contextKey int
const contextKeyWakeup contextKey = iota
const contextKeyDosnapshot contextKey = iota
func Wait(ctx context.Context) <-chan struct{} {
wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
wc, ok := ctx.Value(contextKeyDosnapshot).(chan struct{})
if !ok {
wc = make(chan struct{})
}
@@ -19,7 +19,7 @@ func Wait(ctx context.Context) <-chan struct{} {
type Func func() error
var AlreadyWokenUp = errors.New("already woken up")
var AlreadyDosnapshot = errors.New("already snapshotting")
func Context(ctx context.Context) (context.Context, Func) {
wc := make(chan struct{})
@@ -28,8 +28,8 @@ func Context(ctx context.Context) (context.Context, Func) {
case wc <- struct{}{}:
return nil
default:
return AlreadyWokenUp
return AlreadyDosnapshot
}
}
return context.WithValue(ctx, contextKeyWakeup, wc), wuf
return context.WithValue(ctx, contextKeyDosnapshot, wc), wuf
}
daemon/job/snapjob.go
+12 -5
View File
@@ -4,15 +4,17 @@ import (
"context"
"fmt"
"sort"
"sync"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/job/doreplication"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -29,7 +31,8 @@ type SnapJob struct {
promPruneSecs *prometheus.HistogramVec // labels: prune_side
pruner *pruner.Pruner
prunerMtx sync.Mutex
pruner *pruner.Pruner
}
func (j *SnapJob) Name() string { return j.name.String() }
@@ -77,9 +80,11 @@ type SnapJobStatus struct {
func (j *SnapJob) Status() *Status {
s := &SnapJobStatus{}
t := j.Type()
j.prunerMtx.Lock()
if j.pruner != nil {
s.Pruning = j.pruner.Report()
}
j.prunerMtx.Unlock()
s.Snapshotting = j.snapper.Report()
return &Status{Type: t, JobSpecific: s}
}
@@ -107,13 +112,13 @@ func (j *SnapJob) Run(ctx context.Context) {
invocationCount := 0
outer:
for {
log.Info("wait for wakeups")
log.Info("wait for replications")
select {
case <-ctx.Done():
log.WithError(ctx.Err()).Info("context")
break outer
case <-wakeup.Wait(ctx):
case <-doreplication.Wait(ctx):
case <-periodicDone:
}
invocationCount++
@@ -175,9 +180,11 @@ func (j *SnapJob) doPrune(ctx context.Context) {
JobID: j.name,
FSF: j.fsfilter,
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
Encrypt: &zfs.NilBool{B: true},
Encrypt: &nodefault.Bool{B: true},
})
j.prunerMtx.Lock()
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
j.prunerMtx.Unlock()
log.Info("start pruning")
j.pruner.Prune()
log.Info("finished pruning")
daemon/logging/trace/trace_convenience.go
+8 -4
View File
@@ -43,15 +43,19 @@ func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Con
}
// create a span during which several child tasks are spawned using the `add` function
//
// IMPORTANT FOR USERS: Caller must ensure that the capturing behavior is correct, the Go linter doesn't catch this.
func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
var wg sync.WaitGroup
ctx, endSpan := WithSpan(ctx, taskGroup)
add = func(f func(context.Context)) {
wg.Add(1)
defer wg.Done()
ctx, endTask := WithTask(ctx, taskGroup)
defer endTask()
f(ctx)
go func() {
defer wg.Done()
ctx, endTask := WithTask(ctx, taskGroup)
defer endTask()
f(ctx)
}()
}
waitEnd = func() {
wg.Wait()
daemon/logging/trace/trace_convenience_test.go
+46
View File
@@ -1,7 +1,10 @@
package trace
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
@@ -14,3 +17,46 @@ func TestGetCallerOrPanic(t *testing.T) {
// zrepl prefix is stripped
assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
}
func TestWithTaskGroupRunTasksConcurrently(t *testing.T) {
// spawn a task group where each task waits for the other to start
// => without concurrency, they would hang
rootCtx, endRoot := WithTaskFromStack(context.Background())
defer endRoot()
_, add, waitEnd := WithTaskGroup(rootCtx, "test-task-group")
schedulerTimeout := 2 * time.Second
timeout := time.After(schedulerTimeout)
var hadTimeout uint32
started0, started1 := make(chan struct{}), make(chan struct{})
for i := 0; i < 2; i++ {
i := i // capture by copy
add(func(ctx context.Context) {
switch i {
case 0:
close(started0)
select {
case <-started1:
case <-timeout:
atomic.AddUint32(&hadTimeout, 1)
}
case 1:
close(started1)
select {
case <-started0:
case <-timeout:
atomic.AddUint32(&hadTimeout, 1)
}
default:
panic("unreachable")
}
})
}
waitEnd()
assert.Zero(t, hadTimeout, "either bad impl or scheduler timeout (which is %v)", schedulerTimeout)
}
daemon/logging/trace/trace_genID.go
+1 -10
View File
@@ -3,20 +3,11 @@ package trace
import (
"encoding/base64"
"math/rand"
"os"
"strings"
"time"
"github.com/zrepl/zrepl/util/envconst"
)
var genIdPRNG = rand.New(rand.NewSource(1))
func init() {
genIdPRNG.Seed(time.Now().UnixNano())
genIdPRNG.Seed(int64(os.Getpid()))
}
var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
func init() {
@@ -30,7 +21,7 @@ func genID() string {
enc := base64.NewEncoder(base64.RawStdEncoding, &out)
buf := make([]byte, genIdNumBytes)
for i := 0; i < len(buf); {
n, err := genIdPRNG.Read(buf[i:])
n, err := rand.Read(buf[i:])
if err != nil {
panic(err)
}
daemon/snapper/snapper.go
+9
View File
@@ -9,6 +9,7 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon/job/dosnapshot"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
@@ -210,6 +211,10 @@ func syncUp(a args, u updater) state {
return u(func(s *Snapper) {
s.state = Planning
}).sf()
case <-dosnapshot.Wait(a.ctx):
return u(func(s *Snapper) {
s.state = Planning
}).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}
@@ -378,6 +383,10 @@ func wait(a args, u updater) state {
return u(func(snapper *Snapper) {
snapper.state = Planning
}).sf()
case <-dosnapshot.Wait(a.ctx):
return u(func(snapper *Snapper) {
snapper.state = Planning
}).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}
daemon/snapper/snapper_all.go
+2 -2
View File
@@ -18,9 +18,9 @@ type PeriodicOrManual struct {
s *Snapper
}
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
func (s *PeriodicOrManual) Run(ctx context.Context, replicationCommon chan<- struct{}) {
if s.s != nil {
s.s.Run(ctx, wakeUpCommon)
s.s.Run(ctx, replicationCommon)
}
}
docs/changelog.rst
+34 -7
View File
@@ -6,6 +6,7 @@
.. |docs| replace:: [DOCS]
.. |feature| replace:: [FEATURE]
.. |mig| replace:: **[MIGRATION]**
.. |maint| replace:: [MAINT]
.. _changelog:
@@ -26,6 +27,39 @@ We use the following annotations for classifying changes:
* |feature| Change that introduces new functionality.
* |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
* |docs| Change to the documentation.
* |maint| Maintenance changes.
0.4.0
-----
* |break| Change syntax to trigger a job replication, rename ``zrepl signal wakeup JOB`` to ``zrepl signal replication JOB``
* |feature| support setting zfs send / recv flags in the config (send: ``-wLcepbS`` , recv: ``-ox`` ).
Config docs :ref:`here <job-send-options>` and :ref:`here <job-recv-options>` .
* |feature| parallel replication is now configurable (disabled by default, :ref:`config docs here <replication-option-concurrency>` ).
* |feature| New ``zrepl status`` UI:
* Interactive job selection.
* Interactively ``zrepl signal`` jobs.
* Filter filesystems in the job view by name.
* An approximation of the old UI is still included as `--mode legacy` but will be removed in a future release of zrepl.
* |bugfix| Actually use concurrency when listing zrepl abstractions & doing size estimation.
These operations were accidentally made sequential in zrepl 0.3.
* |bugfix| Job hang-up during second replication attempt.
* |bugfix| Data races conditions in the dataconn rpc stack.
* |maint| Update to protobuf v1.25 and grpc 1.35.
For users who skipped the 0.3.1 update: please make sure your pruning grid config is correct.
The following bugfix in 0.3.1 :issue:`caused problems for some users <400>`:
* |bugfix| pruning: ``grid``: add all snapshots that do not match the regex to the rule's destroy list.
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
0.3.1
-----
@@ -45,13 +79,6 @@ Mostly a bugfix release for :ref:`zrepl 0.3 <release-0.3>`.
* CI infrastructure rework
* Continuous deployment of that new `stable` branch to zrepl.github.io.
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
.. _release-0.3:
0.3
docs/configuration/jobs.rst
+1 -1
View File
@@ -78,7 +78,7 @@ Job Type ``pull``
``$root_fs/$source_path``
* - ``interval``
- | Interval at which to pull from the source job (e.g. ``10m``).
| ``manual`` disables periodic pulling, replication then only happens on :ref:`wakeup <cli-signal-wakeup>`.
| ``manual`` disables periodic pulling, replication then only happens on :ref:`replication <cli-signal-replication>`.
* - ``pruning``
- |pruning-spec|
docs/configuration/overview.rst
+16
View File
@@ -148,6 +148,22 @@ Incremental sends require that ``@from`` be present on the receiving side when r
Incremental sends can also use a ZFS bookmark as *from* on the sending side (``zfs send -i #bm_from fs@to``), where ``#bm_from`` was created using ``zfs bookmark fs@from fs#bm_from``.
The receiving side must always have the actual snapshot ``@from``, regardless of whether the sending side uses ``@from`` or a bookmark of it.
.. _zfs-background-knowledge-plain-vs-raw-sends:
**Plain and raw sends**
By default, ``zfs send`` sends the most generic, backwards-compatible data stream format (so-called 'plain send').
If the sent uses newer features, e.g. compression or encryption, ``zfs send`` has to un-do these operations on the fly to produce the plain send stream.
If the receiver uses newer features (e.g. compression or encryption inherited from the parent FS), it applies the necessary transformations again on the fly during ``zfs recv``.
Flags such as ``-e``, ``-c`` and ``-L`` tell ZFS to produce a send stream that is closer to how the data is stored on disk.
Sending with those flags removes computational overhead from sender and receiver.
However, the receiver will not apply certain transformations, e.g., it will not compress with the receive-side ``compression`` algorithm.
The ``-w`` (``--raw``) flag produces a send stream that is as *raw* as possible.
For unencrypted datasets, its current effect is the same as ``-Lce``.
Encrypted datasets can only be sent plain (unencrypted) or raw (encrypted) using the ``-w`` flag.
**Resumable Send & Recv**
The ``-s`` flag for ``zfs recv`` tells zfs to save the partially received send stream in case it is interrupted.
To resume the replication, the receiving side filesystem's ``receive_resume_token`` must be passed to a new ``zfs send -t <value> | zfs recv`` command.
docs/configuration/replication.rst
+27
View File
@@ -14,6 +14,10 @@ Replication Options
protection:
initial: guarantee_resumability # guarantee_{resumability,incremental,nothing}
incremental: guarantee_resumability # guarantee_{resumability,incremental,nothing}
concurrency:
size_estimates: 4
steps: 1
...
.. _replication-option-protection:
@@ -45,3 +49,26 @@ which is useful if replication happens so rarely (or fails so frequently) that t
When changing this flag, obsoleted zrepl-managed bookmarks and holds will be destroyed on the next replication step that is attempted for each filesystem.
.. _replication-option-concurrency:
``concurrency`` option
----------------------
The ``concurrency`` options control the maximum amount of concurrency during replication.
The default values allow some concurrency during size estimation but no parallelism for the actual replication.
* ``concurrency.steps`` (default = 1) controls the maximum number of concurrently executed :ref:`replication steps <overview-how-replication-works>`.
The planning step for each file system is counted as a single step.
* ``concurrency.size_estimates`` (default = 4) controls the maximum number of concurrent step size estimations done by the job.
Note that initial replication cannot start replicating child filesystems before the parent filesystem's initial replication step has completed.
Some notes on tuning these values:
* Disk: Size estimation is less I/O intensive than step execution because it does not need to access the data blocks.
* CPU: Size estimation is usually a dense CPU burst whereas step execution CPU utilization is stretched out over time because of disk IO.
Faster disks, sending a compressed dataset in :ref:`plain mode <zfs-background-knowledge-plain-vs-raw-sends>` and the zrepl transport mode all contribute to higher CPU requirements.
* Network bandwidth: Size estimation does not consume meaningful amounts of bandwidth, step execution does.
* :ref:`zrepl ZFS abstractions <zrepl-zfs-abstractions>`: for each replication step zrepl needs to update its ZFS abstractions through the ``zfs`` command which often waits multiple seconds for the zpool to sync.
Thus, if the actual send & recv time of a step is small compared to the time spent on zrepl ZFS abstractions then increasing step execution concurrency will result in a lower overall turnaround time.
docs/configuration/sendrecvoptions.rst
+178 -9
View File
@@ -9,22 +9,62 @@ Send & Recv Options
Send Options
~~~~~~~~~~~~
:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
::
jobs:
- type: push
filesystems: ...
send:
encrypted: true
# flags from the table below go here
...
:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
The following table specifies the list of (boolean) options.
Flags with an entry in the ``zfs send`` column map directly to the zfs send CLI flags.
zrepl does not perform feature checks for these flags.
If you enable a flag that is not supported by the installed version of ZFS, the zfs error will show up at runtime in the logs and zrepl status.
See the `upstream man page <https://openzfs.github.io/openzfs-docs/man/8/zfs-send.8.html>`_ (``man zfs-send``) for their semantics.
``encryption`` option
---------------------
.. list-table::
:widths: 20 10 70
:header-rows: 1
The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
More specifically, if ``encryption=true``, zrepl
* - ``send.``
- ``zfs send``
- Comment
* - ``encrypted``
-
- Specific to zrepl, :ref:`see below <job-send-options-encrypted>`.
* - ``raw``
- ``-w``
- Use ``encrypted`` to only allow encrypted sends.
* - ``send_properties``
- ``-p``
- **Be careful**, read the :ref:`note on property replication below <job-note-property-replication>`.
* - ``backup_properties``
- ``-b``
- **Be careful**, read the :ref:`note on property replication below <job-note-property-replication>`.
* - ``large_blocks``
- ``-L``
- **Potential data loss on OpenZFS < 2.0**, see :ref:`warning below <job-send-options-large-blocks>`.
* - ``compressed``
- ``-c``
-
* - ``embbeded_data``
- ``-e``
-
* - ``saved``
- ``-S``
-
.. _job-send-options-encrypted:
``encrypted``
-------------
The ``encrypted`` option controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
More specifically, if ``encrypted=true``, zrepl
* checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and
* invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and
@@ -32,14 +72,143 @@ More specifically, if ``encryption=true``, zrepl
Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages.
If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
If ``encrypted=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
.. NOTE::
Use ``encrypted`` instead of ``raw`` to make your intent clear that zrepl must only replicate filesystems that are actually encrypted by OpenZFS native encryption.
It is meant as a safeguard to prevent unintended sends of unencrypted filesystems in raw mode.
.. _job-send-options-properties:
``properties``
--------------
Sends the dataset properties along with snapshots.
Please be careful with this option and read the :ref:`note on property replication below <job-note-property-replication>`.
.. _job-send-options-backup-properties:
``backup_properties``
---------------------
When properties are modified on a filesystem that was received from a send stream with ``send.properties=true``, ZFS archives the original received value internally.
This also applies to :ref:`inheriting or overriding properties during zfs receive <job-recv-options--inherit-and-override>`.
When sending those received filesystems another hop, the ``backup_properties`` flag instructs ZFS to send the original property values rather than the current locally set values.
This is useful for replicating properties across multiple levels of backup machines.
**Example:**
Suppose we want to flow snapshots from Machine A to B, then from B to C.
A will enable the :ref:`properties send option <job-send-options-properties>`.
B will want to override :ref:`critical properties such as mountpoint or canmount <job-note-property-replication>`.
But the job that replicates from B to C should be sending the original property values received from A.
Thus, B sets the ``backup_properties`` option.
Please be careful with this option and read the :ref:`note on property replication below <job-note-property-replication>`.
.. _job-send-options-large-blocks:
``large_blocks``
----------------
This flag should not be changed after initial replication.
Prior to `OpenZFS commit 7bcb7f08 <https://github.com/openzfs/zfs/pull/10383/files#diff-4c1e47568f46fb63546e984943b09a3e6b051e2242649523f7835bbdfe2a9110R337-R342>`_
it was possible to change this setting which resulted in **data loss on the receiver**.
The commit in question is included in OpenZFS 2.0 and works around the problem by prohibiting receives of incremental streams with a flipped setting.
.. WARNING::
This bug has **not been fixed in the OpenZFS 0.8 releases** which means that changing this flag after initial replication might cause **data loss** on the receiver.
.. _job-recv-options:
Recv Options
~~~~~~~~~~~~
:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section.
However, there are currently no variables to configure there.
:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section:
::
jobs:
- type: pull
recv:
properties:
inherit:
- "mountpoint"
override: {
"org.openzfs.systemd:ignore": "on"
}
...
.. _job-recv-options--inherit-and-override:
``properties``
--------------
``override`` maps directly to the `zfs recv -o flag <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_.
Property name-value pairs specified in this map will apply to all received filesystems, regardless of whether the send stream contains properties or not.
``inherit`` maps directly to the `zfs recv -x flag <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_.
Property names specified in this list will be inherited from the receiving side's parent filesystem (e.g. ``root_fs``).
With both options, the sending side's property value is still stored on the receiver, but the local override or inherit is the one that takes effect.
You can send the original properties from the first receiver to another receiver using :ref:`send.backup_properties<job-send-options-backup-properties>`.
.. _job-note-property-replication:
A Note on Property Replication
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If a send stream contains properties, as per ``send.properties`` or ``send.backup_properties``,
the default ZFS behavior is to use those properties on the receiving side, verbatim.
In many use cases for zrepl, this can have devastating consequences.
For example, when backing up a filesystem that has ``mountpoint=/`` to a storage server,
that storage server's root filesystem will be shadowed by the received file system on some platforms.
Also, many scripts and tools use ZFS user properties for configuration and do not check the property source (``local`` vs. ``received``).
If they are installed on the receiving side as well as the sending side, property replication could have unintended effects.
**zrepl currently does not provide any automatic safe-guards for property replication:**
* Make sure to read the entire man page on zfs recv (`man zfs recv <https://openzfs.github.io/openzfs-docs/man/8/zfs-recv.8.html>`_) before enabling this feature.
* Use ``recv.properties.override`` whenever possible, e.g. for ``mountpoint=none`` or ``canmount=off``.
* Use ``recv.properties.inherit`` if that makes more sense to you.
Below is an **non-exhaustive list of problematic properties**.
Please open a pull request if you find a property that is missing from this list.
(Both with regards to core ZFS tools and other software in the broader ecosystem.)
Mount behaviour
---------------
* ``mountpoint``
* ``canmount``
* ``overlay``
Note: inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
This is an `issue in OpenZFS <https://github.com/openzfs/zfs/issues/11416>`_ .
As a workaround, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.
Please comment at zrepl :issue:`430` if you encounter this issue and/or would like zrepl to automatically work around it.
Systemd
-------
With systemd, you should also consider the properties processed by the `zfs-mount-generator <https://manpages.debian.org/buster-backports/zfsutils-linux/zfs-mount-generator.8.en.html>`_ .
Most notably:
* ``org.openzfs.systemd:ignore``
* ``org.openzfs.systemd:wanted-by``
* ``org.openzfs.systemd:required-by``
Encryption
----------
If the sender filesystems are encrypted but the sender does :ref:`plain sends <zfs-background-knowledge-plain-vs-raw-sends>`
and property replication is enabled, the receiver must :ref:`inherit the following properties<job-recv-options--inherit-and-override>`:
* ``keylocation``
* ``keyformat``
* ``encryption``
docs/configuration/snapshotting.rst
+2 -2
View File
@@ -15,7 +15,7 @@ A filesystem that does not have snapshots by the snapshotter has lower priority
For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
Note that the ``zrepl signal replication JOB`` subcommand does not trigger snapshotting.
::
@@ -38,7 +38,7 @@ There is also a ``manual`` snapshotting type, which covers the following use cas
* Existing infrastructure for automatic snapshots: you only want to use this zrepl job for replication.
* Handling snapshotting through a separate ``snap`` job.
Note that you will have to trigger replication manually using the ``zrepl signal wakeup JOB`` subcommand in that case.
Note that you will have to trigger replication manually using the ``zrepl signal replication JOB`` subcommand in that case.
::
docs/index.rst
+5 -1
View File
@@ -61,7 +61,11 @@ Main Features
* [x] Automatic ZFS holds during send & receive
* [x] Automatic bookmark \& hold management for guaranteed incremental send & recv
* [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption)
* [ ] Compressed send & receive
* [x] Properties send & receive
* [x] Compressed send & receive
* [x] Large blocks send & receive
* [x] Embedded data send & receive
* [x] Resume state send & receive
* **Automatic snapshot management**
docs/quickstart.rst
+2 -2
View File
@@ -55,8 +55,8 @@ Watch it Work
=============
Run ``zrepl status`` on the active side of the replication setup to monitor snaphotting, replication and pruning activity.
To re-trigger replication (snapshots are separate!), use ``zrepl signal wakeup JOBNAME``.
(refer to the example use case document if you are uncertain which job you want to wake up).
To re-trigger replication (snapshots are separate!), use ``zrepl signal replication JOBNAME``.
(refer to the example use case document if you are uncertain which job you want to start replication).
You can also use basic UNIX tools to inspect see what's going on.
If you like tmux, here is a handy script that works on FreeBSD: ::
docs/usage.rst
+2 -2
View File
@@ -13,7 +13,7 @@ CLI Overview
The zrepl binary is self-documenting:
run ``zrepl help`` for an overview of the available subcommands or ``zrepl SUBCOMMAND --help`` for information on available flags, etc.
.. _cli-signal-wakeup:
.. _cli-signal-replication:
.. list-table::
:widths: 30 70
@@ -29,7 +29,7 @@ CLI Overview
- show job activity, or with ``--raw`` for JSON output
* - ``zrepl stdinserver``
- see :ref:`transport-ssh+stdinserver`
* - ``zrepl signal wakeup JOB``
* - ``zrepl signal replication JOB``
- manually trigger replication + pruning of JOB
* - ``zrepl signal reset JOB``
- manually abort current replication + pruning of JOB
endpoint/endpoint.go
+72 -41
View File
@@ -17,19 +17,28 @@ import (
"github.com/zrepl/zrepl/util/chainedio"
"github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/semaphore"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
type SenderConfig struct {
FSF zfs.DatasetFilter
Encrypt *zfs.NilBool
JobID JobID
FSF zfs.DatasetFilter
JobID JobID
Encrypt *nodefault.Bool
SendRaw bool
SendProperties bool
SendBackupProperties bool
SendLargeBlocks bool
SendCompressed bool
SendEmbeddedData bool
SendSaved bool
}
func (c *SenderConfig) Validate() error {
c.JobID.MustValidate()
if err := c.Encrypt.Validate(); err != nil {
if err := c.Encrypt.ValidateNoDefault(); err != nil {
return errors.Wrap(err, "`Encrypt` field invalid")
}
if _, err := StepHoldTag(c.JobID); err != nil {
@@ -40,9 +49,11 @@ func (c *SenderConfig) Validate() error {
// Sender implements replication.ReplicationEndpoint for a sending side
type Sender struct {
pdu.UnsafeReplicationServer // prefer compilation errors over default 'method X not implemented' impl
FSFilter zfs.DatasetFilter
encrypt *zfs.NilBool
jobId JobID
config SenderConfig
}
func NewSender(conf SenderConfig) *Sender {
@@ -51,8 +62,8 @@ func NewSender(conf SenderConfig) *Sender {
}
return &Sender{
FSFilter: conf.FSF,
encrypt: conf.Encrypt,
jobId: conf.JobID,
config: conf,
}
}
@@ -118,9 +129,6 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
}
var maxConcurrentZFSSend = envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10)
var maxConcurrentZFSSendSemaphore = semaphore.New(maxConcurrentZFSSend)
func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
if fsv == nil {
return nil
@@ -152,12 +160,12 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
// use s.encrypt setting
// ok, fallthrough outer
case pdu.Tri_False:
if s.encrypt.B {
if s.config.Encrypt.B {
return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
}
// fallthrough outer
case pdu.Tri_True:
if !s.encrypt.B {
if !s.config.Encrypt.B {
return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
}
// fallthrough outer
@@ -166,11 +174,20 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
}
sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
FS: r.Filesystem,
From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
Encrypted: s.encrypt,
ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
FS: r.Filesystem,
From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
ZFSSendFlags: zfs.ZFSSendFlags{
ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
Encrypted: s.config.Encrypt,
Properties: s.config.SendProperties,
BackupProperties: s.config.SendBackupProperties,
Raw: s.config.SendRaw,
LargeBlocks: s.config.SendLargeBlocks,
Compressed: s.config.SendCompressed,
EmbeddedData: s.config.SendEmbeddedData,
Saved: s.config.SendSaved,
},
}
sendArgs, err := sendArgsUnvalidated.Validate(ctx)
@@ -178,16 +195,6 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
return nil, nil, errors.Wrap(err, "validate send arguments")
}
getLogger(ctx).Debug("acquire concurrent send semaphore")
// TODO use try-acquire and fail with resource-exhaustion rpc status
// => would require handling on the client-side
// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
guard, err := maxConcurrentZFSSendSemaphore.Acquire(ctx)
if err != nil {
return nil, nil, err
}
defer guard.Release()
si, err := zfs.ZFSSendDry(ctx, sendArgs)
if err != nil {
return nil, nil, errors.Wrap(err, "zfs send dry failed")
@@ -422,19 +429,49 @@ type FSMap interface { // FIXME unused
AsFilter() FSFilter
}
// NOTE: when adding members to this struct, remember
// to add them to `ReceiverConfig.copyIn()`
type ReceiverConfig struct {
JobID JobID
RootWithoutClientComponent *zfs.DatasetPath // TODO use
RootWithoutClientComponent *zfs.DatasetPath
AppendClientIdentity bool
InheritProperties []zfsprop.Property
OverrideProperties map[zfsprop.Property]string
}
func (c *ReceiverConfig) copyIn() {
c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
pInherit := make([]zfsprop.Property, len(c.InheritProperties))
copy(pInherit, c.InheritProperties)
c.InheritProperties = pInherit
pOverride := make(map[zfsprop.Property]string, len(c.OverrideProperties))
for key, value := range c.OverrideProperties {
pOverride[key] = value
}
c.OverrideProperties = pOverride
}
func (c *ReceiverConfig) Validate() error {
c.JobID.MustValidate()
for _, prop := range c.InheritProperties {
err := prop.Validate()
if err != nil {
return errors.Wrapf(err, "inherit property %q", prop)
}
}
for prop := range c.OverrideProperties {
err := prop.Validate()
if err != nil {
return errors.Wrapf(err, "override property %q", prop)
}
}
if c.RootWithoutClientComponent.Length() <= 0 {
return errors.New("RootWithoutClientComponent must not be an empty dataset path")
}
@@ -443,6 +480,8 @@ func (c *ReceiverConfig) Validate() error {
// Receiver implements replication.ReplicationEndpoint for a receiving side
type Receiver struct {
pdu.UnsafeReplicationServer // prefer compilation errors over default 'method X not implemented' impl
conf ReceiverConfig // validated
recvParentCreationMtx *chainlock.L
@@ -484,7 +523,7 @@ func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
clientIdentity, ok := ctx.Value(ClientIdentityKey).(string)
if !ok {
panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
panic("ClientIdentityKey context value must be set")
}
clientRoot, err := clientRoot(s.conf.RootWithoutClientComponent, clientIdentity)
@@ -629,8 +668,6 @@ func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io
return nil, nil, fmt.Errorf("receiver does not implement Send()")
}
var maxConcurrentZFSRecvSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_RECV", 10))
func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
@@ -722,6 +759,10 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
return nil, errors.Wrap(err, "cannot get placeholder state")
}
log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state")
recvOpts.InheritProperties = s.conf.InheritProperties
recvOpts.OverrideProperties = s.conf.OverrideProperties
if ph.FSExists && ph.IsPlaceholder {
recvOpts.RollbackAndForceRecv = true
clearPlaceholderProperty = true
@@ -746,16 +787,6 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
}
log.Debug("acquire concurrent recv semaphore")
// TODO use try-acquire and fail with resource-exhaustion rpc status
// => would require handling on the client-side
// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
guard, err := maxConcurrentZFSRecvSemaphore.Acquire(ctx)
if err != nil {
return nil, err
}
defer guard.Release()
var peek bytes.Buffer
var MaxPeek = envconst.Int64("ZREPL_ENDPOINT_RECV_PEEK_SIZE", 1<<20)
log.WithField("max_peek_bytes", MaxPeek).Info("peeking incoming stream")
endpoint/endpoint_zfs_abstraction.go
+11 -9
View File
@@ -13,6 +13,7 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/util/semaphore"
"github.com/zrepl/zrepl/zfs"
)
@@ -257,7 +258,7 @@ type ListZFSHoldsAndBookmarksQuery struct {
type CreateTXGRangeBound struct {
CreateTXG uint64
Inclusive *zfs.NilBool // must not be nil
Inclusive *nodefault.Bool // must not be nil
}
// A non-empty range of CreateTXGs
@@ -300,7 +301,7 @@ func (q *ListZFSHoldsAndBookmarksQuery) Validate() error {
var createTXGRangeBoundAllowCreateTXG0 = envconst.Bool("ZREPL_ENDPOINT_LIST_ABSTRACTIONS_QUERY_CREATETXG_RANGE_BOUND_ALLOW_0", false)
func (i *CreateTXGRangeBound) Validate() error {
if err := i.Inclusive.Validate(); err != nil {
if err := i.Inclusive.ValidateNoDefault(); err != nil {
return errors.Wrap(err, "Inclusive")
}
if i.CreateTXG == 0 && !createTXGRangeBoundAllowCreateTXG0 {
@@ -419,7 +420,7 @@ func (r *CreateTXGRange) String() string {
if r.Since == nil {
fmt.Fprintf(&buf, "~")
} else {
if err := r.Since.Inclusive.Validate(); err != nil {
if err := r.Since.Inclusive.ValidateNoDefault(); err != nil {
fmt.Fprintf(&buf, "?")
} else if r.Since.Inclusive.B {
fmt.Fprintf(&buf, "[")
@@ -435,7 +436,7 @@ func (r *CreateTXGRange) String() string {
fmt.Fprintf(&buf, "~")
} else {
fmt.Fprintf(&buf, "%d", r.Until.CreateTXG)
if err := r.Until.Inclusive.Validate(); err != nil {
if err := r.Until.Inclusive.ValidateNoDefault(); err != nil {
fmt.Fprintf(&buf, "?")
} else if r.Until.Inclusive.B {
fmt.Fprintf(&buf, "]")
@@ -579,16 +580,17 @@ func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmark
_, add, wait := trace.WithTaskGroup(ctx, "list-abstractions-impl-fs")
defer wait()
for i := range fss {
for _, fs := range fss {
fs := fs // capture by copy
add(func(ctx context.Context) {
g, err := sem.Acquire(ctx)
if err != nil {
errCb(err, fss[i], err.Error())
errCb(err, fs, err.Error())
return
}
func() {
defer g.Release()
listAbstractionsImplFS(ctx, fss[i], &query, emitAbstraction, errCb)
listAbstractionsImplFS(ctx, fs, &query, emitAbstraction, errCb)
}()
})
}
@@ -828,14 +830,14 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
untilBound = &CreateTXGRangeBound{
CreateTXG: (*sfnsc.cursor).GetCreateTXG(),
// if we have a cursor, can throw away step hold on both From and To
Inclusive: &zfs.NilBool{B: true},
Inclusive: &nodefault.Bool{B: true},
}
} else if sfnsc.step != nil {
untilBound = &CreateTXGRangeBound{
CreateTXG: (*sfnsc.step).GetCreateTXG(),
// if we don't have a cursor, the step most recent step hold is our
// initial replication cursor and it's possibly still live (interrupted initial replication)
Inclusive: &zfs.NilBool{B: false},
Inclusive: &nodefault.Bool{B: false},
}
} else {
untilBound = nil // consider everything stale
endpoint/endpoint_zfs_abstraction_test.go
+18 -18
View File
@@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/zfs"
"github.com/zrepl/zrepl/util/nodefault"
)
func TestCreateTXGRange(t *testing.T) {
@@ -47,8 +47,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "wrong order obvious",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{20, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{23, &nodefault.Bool{B: true}},
Until: &CreateTXGRangeBound{20, &nodefault.Bool{B: true}},
},
expectString: "[23,20]",
},
@@ -56,8 +56,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "wrong order edge-case could also be empty",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{22, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{23, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{22, &nodefault.Bool{B: true}},
},
expectString: "(23,22]",
},
@@ -65,8 +65,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "empty",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
},
expectString: "(2,2)",
},
@@ -74,8 +74,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "inclusive-since-exclusive-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: true}},
Until: &CreateTXGRangeBound{5, &nodefault.Bool{B: false}},
},
expectString: "[2,5)",
expect: []testCaseExpectation{
@@ -92,8 +92,8 @@ func TestCreateTXGRange(t *testing.T) {
name: "exclusive-since-inclusive-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: &CreateTXGRangeBound{5, &nodefault.Bool{B: true}},
},
expectString: "(2,5]",
expect: []testCaseExpectation{
@@ -111,7 +111,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: true,
config: &CreateTXGRange{
Since: nil,
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: true}},
},
expectString: "~,0]",
},
@@ -119,7 +119,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "half-open-no-until",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{2, &nodefault.Bool{B: false}},
Until: nil,
},
expectString: "(2,~",
@@ -138,7 +138,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: false,
config: &CreateTXGRange{
Since: nil,
Until: &CreateTXGRangeBound{4, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{4, &nodefault.Bool{B: true}},
},
expectString: "~,4]",
expect: []testCaseExpectation{
@@ -154,7 +154,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeSince",
expectInvalid: false,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: true}},
Since: &CreateTXGRangeBound{math.MaxUint64, &nodefault.Bool{B: true}},
Until: nil,
},
expectString: "[18446744073709551615,~",
@@ -169,7 +169,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeSinceNegative",
expectInvalid: true,
config: &CreateTXGRange{
Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: false}},
Since: &CreateTXGRangeBound{math.MaxUint64, &nodefault.Bool{B: false}},
Until: nil,
},
expectString: "(18446744073709551615,~",
@@ -178,7 +178,7 @@ func TestCreateTXGRange(t *testing.T) {
name: "edgeUntil",
expectInvalid: false,
config: &CreateTXGRange{
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: true}},
},
configAllowZeroCreateTXG: true,
expectString: "~,0]",
@@ -193,7 +193,7 @@ func TestCreateTXGRange(t *testing.T) {
expectInvalid: true,
configAllowZeroCreateTXG: true,
config: &CreateTXGRange{
Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: false}},
Until: &CreateTXGRangeBound{0, &nodefault.Bool{B: false}},
},
expectString: "~,0)",
},
go.mod
+17 -7
View File
@@ -5,13 +5,19 @@ go 1.12
require (
github.com/fatih/color v1.7.0
github.com/gdamore/tcell v1.2.0
github.com/gdamore/tcell/v2 v2.2.0
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909
github.com/go-logfmt/logfmt v0.4.0
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator v9.31.0+incompatible
github.com/go-playground/validator/v10 v10.4.1
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
github.com/golang/protobuf v1.3.2
github.com/google/uuid v1.1.1
github.com/golang/protobuf v1.4.3
github.com/google/uuid v1.1.2
github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
github.com/kisielk/gotool v1.0.0 // indirect
github.com/kr/pretty v0.1.0
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.2.0
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.8
@@ -28,15 +34,19 @@ require (
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
github.com/spf13/cobra v0.0.2
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.6.1
github.com/willf/bitset v1.1.10
github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
gitlab.com/tslocum/cview v1.5.3
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
golang.org/x/text v0.3.5 // indirect
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135
gonum.org/v1/gonum v0.7.0 // indirect
google.golang.org/grpc v1.17.0
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579 // indirect
google.golang.org/grpc v1.35.0
google.golang.org/protobuf v1.25.0
)
go.sum
+113
View File
@@ -19,15 +19,21 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -40,6 +46,11 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
github.com/gdamore/tcell v1.4.0 h1:vUnHwJRvcPQa3tzi+0QI4U9JINXYJlOz9yiaiPQ2wMU=
github.com/gdamore/tcell/v2 v2.0.0-dev/go.mod h1:vSVL/GV5mCSlPC6thFP5kfOFdM9MGZcalipmpTxTgQA=
github.com/gdamore/tcell/v2 v2.1.1-0.20201225194624-29bb185874fd/go.mod h1:vSVL/GV5mCSlPC6thFP5kfOFdM9MGZcalipmpTxTgQA=
github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4=
github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909 h1:9NC8seTx6/zRmMTAdsHj/uOMi0EGHGQtjyLafBjk77Q=
github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909/go.mod h1:lP+DW8LR6Rw3ru9Vo2/y/3iiLaLWmofYql/va+7zJOk=
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
@@ -53,6 +64,15 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA=
github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4 h1:0suja/iKSDbEIYLbrS/8C7iArJiWpgCNcR+zwAHu7Ig=
github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -88,6 +108,17 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
@@ -114,9 +145,14 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -149,11 +185,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lucasb-eyer/go-colorful v1.0.2 h1:mCMFu6PgSozg9tDNMMK3g18oJBX7oYGrC09mS6CXfO4=
github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
@@ -166,6 +207,10 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -234,6 +279,9 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
@@ -276,6 +324,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/theckman/goconstraint v1.11.0 h1:oBUwN5wpE4dwyPhRGraEgJsFTr+JtLWiDnaJZJeeXI0=
github.com/theckman/goconstraint v1.11.0/go.mod h1:zkCR/f2kOULTk/h1ujgyB9BlCNLaqlQ6GN2Zl4mg81g=
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
@@ -298,28 +350,42 @@ github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f/go.mod h1:JmNwis
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QESvbW8Meibz7zCrxshP71U9dH5KOCXM=
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
gitlab.com/tslocum/cbind v0.1.4 h1:cbZXPPcieXspk8cShoT6efz7HAT8yMNQcofYWNizis4=
gitlab.com/tslocum/cbind v0.1.4/go.mod h1:RvwYE3auSjBNlCmWeGspzn+jdLUVQ8C2QGC+0nP9ChI=
gitlab.com/tslocum/cview v1.5.3 h1:6OTCtIUp1EkfGeLqQFRHtW8ynMJ66BhoBwuW8oZ84AQ=
gitlab.com/tslocum/cview v1.5.3/go.mod h1:k/eLWRIF3B26VLDgtRRPkjLUXmcCsy+YCSPEAtNQgIY=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -335,6 +401,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeoexdbeMjttk6Oh1rD10=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -342,25 +409,43 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/nt
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e h1:Io7mpb+aUAGF0MKxbyQ7HQl1VgB+cL6ZJZUFaFNqVV4=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
@@ -368,13 +453,37 @@ gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6d
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579 h1:Iwh0ba2kTgq2Q6mJiXhzrrjD7h11nEVnbMHFmp0/HsQ=
google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -392,7 +501,11 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
lazy.sh
+2 -7
View File
@@ -39,15 +39,10 @@ godep() {
export GOOS="$GOHOSTOS"
export GOARCH="$GOHOSTARCH"
# TODO GOARM=$GOHOSTARM?
go build -v -mod=readonly -o "$GOPATH/bin/stringer" golang.org/x/tools/cmd/stringer
go build -v -mod=readonly -o "$GOPATH/bin/protoc-gen-go" github.com/golang/protobuf/protoc-gen-go
go build -v -mod=readonly -o "$GOPATH/bin/enumer" github.com/alvaroloes/enumer
go build -v -mod=readonly -o "$GOPATH/bin/goimports" golang.org/x/tools/cmd/goimports
go build -v -mod=readonly -o "$GOPATH/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
go build -v -mod=readonly -o "$GOPATH/bin/gocovmerge" github.com/wadey/gocovmerge
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
set +x
popd
if ! type stringer || ! type protoc-gen-go || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
if ! type stringer || ! type protoc-gen-go || ! type protoc-gen-go-grpc || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
exit 1
fi
main.go
+2 -1
View File
@@ -4,12 +4,13 @@ package main
import (
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/client"
"github.com/zrepl/zrepl/client/status"
"github.com/zrepl/zrepl/daemon"
)
func init() {
cli.AddSubcommand(daemon.DaemonCmd)
cli.AddSubcommand(client.StatusCmd)
cli.AddSubcommand(status.Subcommand)
cli.AddSubcommand(client.SignalCmd)
cli.AddSubcommand(client.StdinserverCmd)
cli.AddSubcommand(client.ConfigcheckCmd)
platformtest/platformtest_ops.go
+3 -3
View File
@@ -256,14 +256,14 @@ nextLine:
case string(DestroyRoot):
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
stmts = append(stmts, &DestroyRootOp{rootds})
continue nextLine
case string(CreateRoot):
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
stmts = append(stmts, &FSOp{Op: Add, Path: rootds})
continue nextLine
@@ -319,7 +319,7 @@ nextLine:
}
if comps.Scan() {
return nil, &LineError{scan.Text(), fmt.Sprintf("unexpected tokens at EOL")}
return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
}
}
return stmts, nil
platformtest/tests/generated_cases.go
+1
View File
@@ -24,6 +24,7 @@ var Cases = []Case{BatchDestroy,
ReplicationIsResumableFullSend__both_GuaranteeResumability,
ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication,
ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication,
ReplicationPropertyReplicationWorks,
ReplicationReceiverErrorWhileStillSending,
ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication,
ReplicationStepCompletedLostBehavior__GuaranteeResumability,
platformtest/tests/recvForceIntoEncryptedErr.go
+8 -5
View File
@@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -31,11 +32,13 @@ func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
sfsSnap1 := sendArgVersion(ctx, sfs, "@1")
sendArgs, err := zfs.ZFSSendArgsUnvalidated{
FS: sfs,
Encrypted: &zfs.NilBool{B: false},
From: nil,
To: &sfsSnap1,
ResumeToken: "",
FS: sfs,
From: nil,
To: &sfsSnap1,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: "",
},
}.Validate(ctx)
require.NoError(ctx, err)
platformtest/tests/recvRollback.go
+8 -5
View File
@@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -27,11 +28,13 @@ func ReceiveForceRollbackWorksUnencrypted(ctx *platformtest.Context) {
sfsSnap1 := sendArgVersion(ctx, sfs, "@1")
sendArgs, err := zfs.ZFSSendArgsUnvalidated{
FS: sfs,
Encrypted: &zfs.NilBool{B: false},
From: nil,
To: &sfsSnap1,
ResumeToken: "",
FS: sfs,
From: nil,
To: &sfsSnap1,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: "",
},
}.Validate(ctx)
require.NoError(ctx, err)
platformtest/tests/replication.go
+204 -29
View File
@@ -7,6 +7,7 @@ import (
"os"
"path"
"sort"
"time"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
@@ -15,11 +16,14 @@ import (
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/replication/driver"
"github.com/zrepl/zrepl/replication/logic"
"github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/replication/report"
"github.com/zrepl/zrepl/util/limitio"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
zfsprop "github.com/zrepl/zrepl/zfs/property"
)
// mimics the replication invocations of an active-side job
@@ -29,13 +33,15 @@ import (
// of a new sender and receiver instance and one blocking invocation
// of the replication engine without encryption
type replicationInvocation struct {
sjid, rjid endpoint.JobID
sfs string
sfilter *filters.DatasetMapFilter
rfsRoot string
interceptSender func(e *endpoint.Sender) logic.Sender
interceptReceiver func(e *endpoint.Receiver) logic.Receiver
guarantee pdu.ReplicationConfigProtection
sjid, rjid endpoint.JobID
sfs string
sfilter *filters.DatasetMapFilter
rfsRoot string
interceptSender func(e *endpoint.Sender) logic.Sender
interceptReceiver func(e *endpoint.Receiver) logic.Receiver
guarantee *pdu.ReplicationConfigProtection
senderConfigHook func(*endpoint.SenderConfig)
receiverConfigHook func(*endpoint.ReceiverConfig)
}
func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
@@ -55,25 +61,44 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
err := i.sfilter.Add(i.sfs, "ok")
require.NoError(ctx, err)
}
sender := i.interceptSender(endpoint.NewSender(endpoint.SenderConfig{
senderConfig := endpoint.SenderConfig{
FSF: i.sfilter.AsFilter(),
Encrypt: &zfs.NilBool{B: false},
Encrypt: &nodefault.Bool{B: false},
JobID: i.sjid,
}))
receiver := i.interceptReceiver(endpoint.NewReceiver(endpoint.ReceiverConfig{
}
if i.senderConfigHook != nil {
i.senderConfigHook(&senderConfig)
}
receiverConfig := endpoint.ReceiverConfig{
JobID: i.rjid,
AppendClientIdentity: false,
RootWithoutClientComponent: mustDatasetPath(i.rfsRoot),
}))
}
if i.receiverConfigHook != nil {
i.receiverConfigHook(&receiverConfig)
}
require.Equal(ctx, senderConfig.JobID, i.sjid)
require.Equal(ctx, receiverConfig.JobID, i.rjid)
sender := i.interceptSender(endpoint.NewSender(senderConfig))
receiver := i.interceptReceiver(endpoint.NewReceiver(receiverConfig))
plannerPolicy := logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(false),
ReplicationConfig: pdu.ReplicationConfig{
Protection: &i.guarantee,
ReplicationConfig: &pdu.ReplicationConfig{
Protection: i.guarantee,
},
}
report, wait := replication.Do(
ctx,
driver.Config{
MaxAttempts: 1,
StepQueueConcurrency: 1,
ReconnectHardFailTimeout: 1 * time.Second,
},
logic.NewPlanner(nil, nil, sender, receiver, plannerPolicy),
)
wait(true)
@@ -106,7 +131,7 @@ func ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed(ctx *platformte
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
@@ -170,7 +195,7 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
@@ -343,7 +368,7 @@ func (s *PartialSender) Send(ctx context.Context, r *pdu.SendReq) (r1 *pdu.SendR
func ReplicationIsResumableFullSend__both_GuaranteeResumability(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
},
@@ -358,7 +383,7 @@ func ReplicationIsResumableFullSend__both_GuaranteeResumability(ctx *platformtes
func ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
},
@@ -373,7 +398,7 @@ func ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_G
func ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
setup := replicationIsResumableFullSendSetup{
protection: pdu.ReplicationConfigProtection{
protection: &pdu.ReplicationConfigProtection{
Initial: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
},
@@ -386,7 +411,7 @@ func ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_inc
}
type replicationIsResumableFullSendSetup struct {
protection pdu.ReplicationConfigProtection
protection *pdu.ReplicationConfigProtection
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated bool
expectAllThreeSnapshotsToThreeBePresentAfterLoop bool
expectNoSnapshotsOnReceiverAfterLoop bool
@@ -501,7 +526,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
}
rfs := rep.ReceiveSideFilesystem()
report := rep.Do(ctx)
@@ -528,7 +553,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability), // !
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability), // !
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
},
@@ -574,7 +599,7 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication), // !
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication), // !
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
},
@@ -684,7 +709,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
}
rfs := rep.ReceiveSideFilesystem()
report := rep.Do(ctx)
@@ -702,7 +727,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
interceptSender: func(e *endpoint.Sender) logic.Sender {
return &FailSendCompletedSender{e}
},
@@ -750,7 +775,7 @@ func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaran
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
guarantee: pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
}
report := rep.Do(ctx)
ctx.Logf("expecting failure:\n%s", pretty.Sprint(report))
@@ -818,7 +843,7 @@ func ReplicationReceiverErrorWhileStillSending(ctx *platformtest.Context) {
rjid: rjid,
sfs: sfs,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
},
@@ -880,7 +905,7 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
},
@@ -900,7 +925,7 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
}
require.Contains(ctx, fsByName, fsA)
require.Contains(ctx, fsByName, fsAA)
require.Contains(ctx, fsByName, fsAChild)
require.Contains(ctx, fsByName, fsAA)
checkFS := func(fs string, expectErrMsg string) {
@@ -915,3 +940,153 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.
checkFS(fsAChild, "parent(s) failed during initial replication")
checkFS(fsAA, mockRecvErr.Error()) // fsAA is not treated as a child of fsA
}
func ReplicationPropertyReplicationWorks(ctx *platformtest.Context) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
CREATEROOT
+ "sender"
+ "sender/a"
+ "sender/a@1"
+ "sender/a/child"
+ "sender/a/child@1"
+ "receiver"
`)
sjid := endpoint.MustMakeJobID("sender-job")
rjid := endpoint.MustMakeJobID("receiver-job")
fsA := ctx.RootDataset + "/sender/a"
fsAChild := ctx.RootDataset + "/sender/a/child"
sfilter := filters.NewDatasetMapFilter(2, true)
mustAddToSFilter := func(fs string) {
err := sfilter.Add(fs, "ok")
require.NoError(ctx, err)
}
mustAddToSFilter(fsA)
mustAddToSFilter(fsAChild)
rfsRoot := ctx.RootDataset + "/receiver"
type testPropExpectation struct {
Exists bool
Source zfs.PropertySource
ExpectSpecialValue string
}
type testProp struct {
Name string
SetOnSender map[string]bool
ExpectReceiver map[string]testPropExpectation
}
testProps := []testProp{
{
Name: "zrepl:ignored",
SetOnSender: map[string]bool{fsA: true, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {
Exists: false,
},
fsAChild: {
Exists: false,
},
},
},
{
Name: "zrepl:replicate",
SetOnSender: map[string]bool{fsA: true, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {Exists: true, Source: zfs.SourceReceived},
fsAChild: {Exists: true, Source: zfs.SourceReceived},
},
},
{
Name: "zrepl:overridden",
SetOnSender: map[string]bool{fsA: false, fsAChild: true},
ExpectReceiver: map[string]testPropExpectation{
fsA: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"},
fsAChild: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"},
},
},
}
for _, prop := range testProps {
for fs := range prop.SetOnSender {
err := zfs.ZFSSet(ctx, mustDatasetPath(fs), map[string]string{prop.Name: prop.Name})
require.NoError(ctx, err)
}
}
rep := replicationInvocation{
sjid: sjid,
rjid: rjid,
sfilter: sfilter,
rfsRoot: rfsRoot,
guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
receiverConfigHook: func(c *endpoint.ReceiverConfig) {
c.InheritProperties = []zfsprop.Property{"zrepl:ignored"}
c.OverrideProperties = map[zfsprop.Property]string{
"zrepl:overridden": "overridden value",
}
},
senderConfigHook: func(c *endpoint.SenderConfig) {
c.SendProperties = true // TODO: do another tier with SendBackupProperties
},
}
r := rep.Do(ctx)
ctx.Logf("\n%s", pretty.Sprint(r))
require.Len(ctx, r.Attempts, 1)
attempt := r.Attempts[0]
require.Nil(ctx, attempt.PlanError)
require.Len(ctx, attempt.Filesystems, 2)
fsByName := make(map[string]*report.FilesystemReport, len(attempt.Filesystems))
for _, fs := range attempt.Filesystems {
fsByName[fs.Info.Name] = fs
}
require.Contains(ctx, fsByName, fsA)
require.Contains(ctx, fsByName, fsAChild)
require.Len(ctx, fsByName, 2)
requireFSReportSucceeded := func(fs string) *zfs.DatasetPath {
rep := fsByName[fs]
require.Len(ctx, rep.Steps, 1)
require.Nil(ctx, rep.PlanError)
require.Nil(ctx, rep.StepError)
require.Len(ctx, rep.Steps, 1)
require.Equal(ctx, 1, rep.CurrentStep)
return mustDatasetPath(path.Join(rfsRoot, fs))
}
rfsA := requireFSReportSucceeded(fsA)
rfsAChild := requireFSReportSucceeded(fsAChild)
rfsmap := map[string]*zfs.DatasetPath{
fsA: rfsA,
fsAChild: rfsAChild,
}
for fs, rfs := range rfsmap {
for _, tp := range testProps {
r_, err := zfs.ZFSGet(ctx, rfs, []string{tp.Name})
require.NoError(ctx, err)
r := r_.GetDetails(tp.Name)
expect := tp.ExpectReceiver[fs]
if !expect.Exists {
require.Equal(ctx, zfs.SourceNone, r.Source)
} else {
require.Equal(ctx, expect.Source, r.Source)
if expect.ExpectSpecialValue != "" {
require.Equal(ctx, expect.ExpectSpecialValue, r.Value)
} else {
require.Equal(ctx, tp.Name, r.Value)
}
}
}
}
}
platformtest/tests/resumableRecvAndTokenHandling.go
+7 -4
View File
@@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -26,10 +27,12 @@ func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
src := makeDummyDataSnapshots(ctx, sendFS)
s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: false},
ResumeToken: "",
FS: sendFS,
To: src.snapA,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: "",
},
}, zfs.RecvOptions{
RollbackAndForceRecv: false, // doesnt' exist yet
SavePartialRecvState: true,
platformtest/tests/sendArgsValidation.go
+18 -13
View File
@@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/util/nodefault"
"github.com/zrepl/zrepl/zfs"
)
@@ -42,8 +43,10 @@ func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *plat
RelName: "@a snap",
GUID: props.Guid,
},
Encrypted: &zfs.NilBool{B: true},
ResumeToken: "",
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: true},
ResumeToken: "",
},
}.Validate(ctx)
var stream *zfs.SendStream
@@ -95,18 +98,18 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
src := makeDummyDataSnapshots(ctx, sendFS)
unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: false}, // !
FS: sendFS,
To: src.snapA,
ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}}, // !
}, zfs.RecvOptions{
RollbackAndForceRecv: false,
SavePartialRecvState: true,
})
encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: true}, // !
FS: sendFS,
To: src.snapA,
ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: true}}, // !
}, zfs.RecvOptions{
RollbackAndForceRecv: false,
SavePartialRecvState: true,
@@ -172,9 +175,9 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
src2 := makeDummyDataSnapshots(ctx, sendFS2)
rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS1,
To: src1.snapA,
Encrypted: &zfs.NilBool{B: false},
FS: sendFS1,
To: src1.snapA,
ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}},
}, zfs.RecvOptions{
RollbackAndForceRecv: false,
SavePartialRecvState: true,
@@ -188,8 +191,10 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
RelName: src2.snapA.RelName,
GUID: src2.snapA.GUID,
},
Encrypted: &zfs.NilBool{B: false},
ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
ZFSSendFlags: zfs.ZFSSendFlags{
Encrypted: &nodefault.Bool{B: false},
ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
},
}
_, err = maliciousSend.Validate(ctx)
require.Error(ctx, err)
replication/driver/replication_driver.go
+48 -51
View File
@@ -9,6 +9,7 @@ import (
"sync"
"time"
"github.com/go-playground/validator"
"github.com/kr/pretty"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
@@ -19,7 +20,6 @@ import (
"github.com/zrepl/zrepl/replication/report"
"github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/util/envconst"
)
type interval struct {
@@ -84,6 +84,7 @@ type Planner interface {
// an attempt represents a single planning & execution of fs replications
type attempt struct {
planner Planner
config Config
l *chainlock.L
@@ -181,10 +182,25 @@ type step struct {
type ReportFunc func() *report.Report
type WaitFunc func(block bool) (done bool)
var maxAttempts = envconst.Int64("ZREPL_REPLICATION_MAX_ATTEMPTS", 3)
var reconnectHardFailTimeout = envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute)
type Config struct {
StepQueueConcurrency int `validate:"gte=1"`
MaxAttempts int `validate:"eq=-1|gt=0"`
ReconnectHardFailTimeout time.Duration `validate:"gt=0"`
}
var validate = validator.New()
func (c Config) Validate() error {
return validate.Struct(c)
}
// caller must ensure config.Validate() == nil
func Do(ctx context.Context, config Config, planner Planner) (ReportFunc, WaitFunc) {
if err := config.Validate(); err != nil {
panic(err)
}
func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
log := getLog(ctx)
l := chainlock.New()
run := &run{
@@ -201,7 +217,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
defer log.Debug("run ended")
var prev *attempt
mainLog := log
for ano := 0; ano < int(maxAttempts) || maxAttempts == 0; ano++ {
for ano := 0; ano < int(config.MaxAttempts); ano++ {
log := mainLog.WithField("attempt_number", ano)
log.Debug("start attempt")
@@ -213,6 +229,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
l: l,
startedAt: time.Now(),
planner: planner,
config: config,
}
run.attempts = append(run.attempts, cur)
run.l.DropWhile(func() {
@@ -248,7 +265,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
shouldReconnect := mostRecentErrClass == errorClassTemporaryConnectivityRelated
log.WithField("reconnect_decision", shouldReconnect).Debug("reconnect decision made")
if shouldReconnect {
run.waitReconnect.Set(time.Now(), reconnectHardFailTimeout)
run.waitReconnect.Set(time.Now(), config.ReconnectHardFailTimeout)
log.WithField("deadline", run.waitReconnect.End()).Error("temporary connectivity-related error identified, start waiting for reconnect")
var connectErr error
var connectErrTime time.Time
@@ -421,7 +438,7 @@ func (a *attempt) doFilesystems(ctx context.Context, prevs map[*fs]*fs) {
defer a.l.Lock().Unlock()
stepQueue := newStepQueue()
defer stepQueue.Start(envconst.Int("ZREPL_REPLICATION_EXPERIMENTAL_REPLICATION_CONCURRENCY", 1))() // TODO parallel replication
defer stepQueue.Start(a.config.StepQueueConcurrency)()
var fssesDone sync.WaitGroup
for _, f := range a.fss {
fssesDone.Add(1)
@@ -492,57 +509,37 @@ func (f *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
// => don't set f.planning.done just yet
f.debug("initial len(fs.planned.steps) = %d", len(f.planned.steps))
// for not-first attempts, only allow fs.planned.steps
// up to including the originally planned target snapshot
// for not-first attempts that succeeded in planning, only allow fs.planned.steps
// up to and including the originally planned target snapshot
if prev != nil && prev.planning.done && prev.planning.err == nil {
f.debug("attempting to correlate plan with previous attempt to find out what is left to do")
// find the highest of the previously uncompleted steps for which we can also find a step
// in our current plan
prevUncompleted := prev.planned.steps[prev.planned.step:]
if len(prevUncompleted) == 0 {
f.debug("prevUncompleted is empty")
return
}
if len(f.planned.steps) == 0 {
f.debug("fs.planned.steps is empty")
return
}
prevFailed := prevUncompleted[0]
curFirst := f.planned.steps[0]
// we assume that PlanFS retries prevFailed (using curFirst)
if !prevFailed.step.TargetEquals(curFirst.step) {
f.debug("Targets don't match")
// Two options:
// A: planning algorithm is broken
// B: manual user intervention inbetween
// Neither way will we make progress, so let's error out
stepFmt := func(step *step) string {
r := step.report()
s := r.Info
if r.IsIncremental() {
return fmt.Sprintf("%s=>%s", s.From, s.To)
} else {
return fmt.Sprintf("full=>%s", s.To)
var target struct{ prev, cur int }
target.prev = -1
target.cur = -1
out:
for p := len(prevUncompleted) - 1; p >= 0; p-- {
for q := len(f.planned.steps) - 1; q >= 0; q-- {
if prevUncompleted[p].step.TargetEquals(f.planned.steps[q].step) {
target.prev = p
target.cur = q
break out
}
}
msg := fmt.Sprintf("last attempt's uncompleted step %s does not correspond to this attempt's first planned step %s",
stepFmt(prevFailed), stepFmt(curFirst))
f.planned.stepErr = newTimedError(errors.New(msg), time.Now())
}
if target.prev == -1 || target.cur == -1 {
f.debug("no correlation possible between previous attempt and this attempt's plan")
f.planning.err = newTimedError(fmt.Errorf("cannot correlate previously failed attempt to current plan"), time.Now())
return
}
// only allow until step targets diverge
min := len(prevUncompleted)
if min > len(f.planned.steps) {
min = len(f.planned.steps)
}
diverge := 0
for ; diverge < min; diverge++ {
f.debug("diverge compare iteration %d", diverge)
if !f.planned.steps[diverge].step.TargetEquals(prevUncompleted[diverge].step) {
break
}
}
f.debug("diverge is %d", diverge)
f.planned.steps = f.planned.steps[0:diverge]
f.planned.steps = f.planned.steps[0:target.cur]
f.debug("found correlation, new steps are len(fs.planned.steps) = %d", len(f.planned.steps))
} else {
f.debug("previous attempt does not exist or did not finish planning, no correlation possible, taking this attempt's plan as is")
}
f.debug("post-prev-merge len(fs.planned.steps) = %d", len(f.planned.steps))
// now we are done planning (f.planned.steps won't change from now on)
f.planning.done = true
replication/driver/replication_driver_test.go
+6 -1
View File
@@ -154,7 +154,12 @@ func TestReplication(t *testing.T) {
defer trace.WithTaskFromStackUpdateCtx(&ctx)()
mp := &mockPlanner{}
getReport, wait := Do(ctx, mp)
driverConfig := Config{
StepQueueConcurrency: 1,
MaxAttempts: 1,
ReconnectHardFailTimeout: 1 * time.Second,
}
getReport, wait := Do(ctx, driverConfig, mp)
begin := time.Now()
fireAt := []time.Duration{
// the following values are relative to the start
replication/logic/pdu/pdu.pb.go
+1548 -1140
View File
File diff suppressed because it is too large Load Diff
replication/logic/pdu/pdu.proto
+1 -1
View File
@@ -1,5 +1,5 @@
syntax = "proto3";
option go_package = "pdu";
option go_package = ".;pdu";
service Replication {
rpc Ping(PingReq) returns (PingRes);
replication/logic/pdu/pdu_grpc.pb.go
+281
View File
@@ -0,0 +1,281 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
package pdu
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// ReplicationClient is the client API for Replication service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ReplicationClient interface {
Ping(ctx context.Context, in *PingReq, opts ...grpc.CallOption) (*PingRes, error)
ListFilesystems(ctx context.Context, in *ListFilesystemReq, opts ...grpc.CallOption) (*ListFilesystemRes, error)
ListFilesystemVersions(ctx context.Context, in *ListFilesystemVersionsReq, opts ...grpc.CallOption) (*ListFilesystemVersionsRes, error)
DestroySnapshots(ctx context.Context, in *DestroySnapshotsReq, opts ...grpc.CallOption) (*DestroySnapshotsRes, error)
ReplicationCursor(ctx context.Context, in *ReplicationCursorReq, opts ...grpc.CallOption) (*ReplicationCursorRes, error)
SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error)
}
type replicationClient struct {
cc grpc.ClientConnInterface
}
func NewReplicationClient(cc grpc.ClientConnInterface) ReplicationClient {
return &replicationClient{cc}
}
func (c *replicationClient) Ping(ctx context.Context, in *PingReq, opts ...grpc.CallOption) (*PingRes, error) {
out := new(PingRes)
err := c.cc.Invoke(ctx, "/Replication/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *replicationClient) ListFilesystems(ctx context.Context, in *ListFilesystemReq, opts ...grpc.CallOption) (*ListFilesystemRes, error) {
out := new(ListFilesystemRes)
err := c.cc.Invoke(ctx, "/Replication/ListFilesystems", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *replicationClient) ListFilesystemVersions(ctx context.Context, in *ListFilesystemVersionsReq, opts ...grpc.CallOption) (*ListFilesystemVersionsRes, error) {
out := new(ListFilesystemVersionsRes)
err := c.cc.Invoke(ctx, "/Replication/ListFilesystemVersions", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *replicationClient) DestroySnapshots(ctx context.Context, in *DestroySnapshotsReq, opts ...grpc.CallOption) (*DestroySnapshotsRes, error) {
out := new(DestroySnapshotsRes)
err := c.cc.Invoke(ctx, "/Replication/DestroySnapshots", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *replicationClient) ReplicationCursor(ctx context.Context, in *ReplicationCursorReq, opts ...grpc.CallOption) (*ReplicationCursorRes, error) {
out := new(ReplicationCursorRes)
err := c.cc.Invoke(ctx, "/Replication/ReplicationCursor", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *replicationClient) SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error) {
out := new(SendCompletedRes)
err := c.cc.Invoke(ctx, "/Replication/SendCompleted", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// ReplicationServer is the server API for Replication service.
// All implementations must embed UnimplementedReplicationServer
// for forward compatibility
type ReplicationServer interface {
Ping(context.Context, *PingReq) (*PingRes, error)
ListFilesystems(context.Context, *ListFilesystemReq) (*ListFilesystemRes, error)
ListFilesystemVersions(context.Context, *ListFilesystemVersionsReq) (*ListFilesystemVersionsRes, error)
DestroySnapshots(context.Context, *DestroySnapshotsReq) (*DestroySnapshotsRes, error)
ReplicationCursor(context.Context, *ReplicationCursorReq) (*ReplicationCursorRes, error)
SendCompleted(context.Context, *SendCompletedReq) (*SendCompletedRes, error)
mustEmbedUnimplementedReplicationServer()
}
// UnimplementedReplicationServer must be embedded to have forward compatible implementations.
type UnimplementedReplicationServer struct {
}
func (UnimplementedReplicationServer) Ping(context.Context, *PingReq) (*PingRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedReplicationServer) ListFilesystems(context.Context, *ListFilesystemReq) (*ListFilesystemRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListFilesystems not implemented")
}
func (UnimplementedReplicationServer) ListFilesystemVersions(context.Context, *ListFilesystemVersionsReq) (*ListFilesystemVersionsRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListFilesystemVersions not implemented")
}
func (UnimplementedReplicationServer) DestroySnapshots(context.Context, *DestroySnapshotsReq) (*DestroySnapshotsRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method DestroySnapshots not implemented")
}
func (UnimplementedReplicationServer) ReplicationCursor(context.Context, *ReplicationCursorReq) (*ReplicationCursorRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReplicationCursor not implemented")
}
func (UnimplementedReplicationServer) SendCompleted(context.Context, *SendCompletedReq) (*SendCompletedRes, error) {
return nil, status.Errorf(codes.Unimplemented, "method SendCompleted not implemented")
}
func (UnimplementedReplicationServer) mustEmbedUnimplementedReplicationServer() {}
// UnsafeReplicationServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ReplicationServer will
// result in compilation errors.
type UnsafeReplicationServer interface {
mustEmbedUnimplementedReplicationServer()
}
func RegisterReplicationServer(s grpc.ServiceRegistrar, srv ReplicationServer) {
s.RegisterService(&Replication_ServiceDesc, srv)
}
func _Replication_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).Ping(ctx, req.(*PingReq))
}
return interceptor(ctx, in, info, handler)
}
func _Replication_ListFilesystems_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListFilesystemReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).ListFilesystems(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/ListFilesystems",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).ListFilesystems(ctx, req.(*ListFilesystemReq))
}
return interceptor(ctx, in, info, handler)
}
func _Replication_ListFilesystemVersions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListFilesystemVersionsReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).ListFilesystemVersions(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/ListFilesystemVersions",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).ListFilesystemVersions(ctx, req.(*ListFilesystemVersionsReq))
}
return interceptor(ctx, in, info, handler)
}
func _Replication_DestroySnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DestroySnapshotsReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).DestroySnapshots(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/DestroySnapshots",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).DestroySnapshots(ctx, req.(*DestroySnapshotsReq))
}
return interceptor(ctx, in, info, handler)
}
func _Replication_ReplicationCursor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReplicationCursorReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).ReplicationCursor(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/ReplicationCursor",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).ReplicationCursor(ctx, req.(*ReplicationCursorReq))
}
return interceptor(ctx, in, info, handler)
}
func _Replication_SendCompleted_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SendCompletedReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ReplicationServer).SendCompleted(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/Replication/SendCompleted",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ReplicationServer).SendCompleted(ctx, req.(*SendCompletedReq))
}
return interceptor(ctx, in, info, handler)
}
// Replication_ServiceDesc is the grpc.ServiceDesc for Replication service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Replication_ServiceDesc = grpc.ServiceDesc{
ServiceName: "Replication",
HandlerType: (*ReplicationServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Ping",
Handler: _Replication_Ping_Handler,
},
{
MethodName: "ListFilesystems",
Handler: _Replication_ListFilesystems_Handler,
},
{
MethodName: "ListFilesystemVersions",
Handler: _Replication_ListFilesystemVersions_Handler,
},
{
MethodName: "DestroySnapshots",
Handler: _Replication_DestroySnapshots_Handler,
},
{
MethodName: "ReplicationCursor",
Handler: _Replication_ReplicationCursor_Handler,
},
{
MethodName: "SendCompleted",
Handler: _Replication_SendCompleted_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "pdu.proto",
}
replication/logic/pdu/pdu_test.go
+4 -4
View File
@@ -10,7 +10,7 @@ import (
func TestFilesystemVersion_RelName(t *testing.T) {
type TestCase struct {
In FilesystemVersion
In *FilesystemVersion
Out string
Panic bool
}
@@ -18,7 +18,7 @@ func TestFilesystemVersion_RelName(t *testing.T) {
creat := FilesystemVersionCreation(time.Now())
tcs := []TestCase{
{
In: FilesystemVersion{
In: &FilesystemVersion{
Type: FilesystemVersion_Snapshot,
Name: "foobar",
Creation: creat,
@@ -26,7 +26,7 @@ func TestFilesystemVersion_RelName(t *testing.T) {
Out: "@foobar",
},
{
In: FilesystemVersion{
In: &FilesystemVersion{
Type: FilesystemVersion_Bookmark,
Name: "foobar",
Creation: creat,
@@ -34,7 +34,7 @@ func TestFilesystemVersion_RelName(t *testing.T) {
Out: "#foobar",
},
{
In: FilesystemVersion{
In: &FilesystemVersion{
Type: 2342,
Name: "foobar",
Creation: creat,
replication/logic/replication_logic.go
+8 -5
View File
@@ -19,7 +19,6 @@ import (
"github.com/zrepl/zrepl/replication/report"
"github.com/zrepl/zrepl/util/bytecounter"
"github.com/zrepl/zrepl/util/chainlock"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/semaphore"
"github.com/zrepl/zrepl/zfs"
)
@@ -117,7 +116,7 @@ func (p *Planner) WaitForConnectivity(ctx context.Context) error {
type Filesystem struct {
sender Sender
receiver Receiver
policy PlannerPolicy
policy PlannerPolicy // immutable, it's .ReplicationConfig member is a pointer and copied into messages
Path string // compat
receiverFS, senderFS *pdu.Filesystem // receiverFS may be nil, senderFS never nil
@@ -222,7 +221,11 @@ func (s *Step) ReportInfo() *report.StepInfo {
}
}
// caller must ensure policy.Validate() == nil
func NewPlanner(secsPerState *prometheus.HistogramVec, bytesReplicated *prometheus.CounterVec, sender Sender, receiver Receiver, policy PlannerPolicy) *Planner {
if err := policy.Validate(); err != nil {
panic(err)
}
return &Planner{
sender: sender,
receiver: receiver,
@@ -272,7 +275,7 @@ func (p *Planner) doPlanning(ctx context.Context) ([]*Filesystem, error) {
}
rfss := rlfssres.GetFilesystems()
sizeEstimateRequestSem := semaphore.New(envconst.Int64("ZREPL_REPLICATION_MAX_CONCURRENT_SIZE_ESTIMATE", 4))
sizeEstimateRequestSem := semaphore.New(int64(p.policy.SizeEstimationConcurrency))
q := make([]*Filesystem, 0, len(sfss))
for _, fs := range sfss {
@@ -569,7 +572,7 @@ func (s *Step) buildSendRequest(dryRun bool) (sr *pdu.SendReq) {
Encrypted: s.encrypt.ToPDU(),
ResumeToken: s.resumeToken,
DryRun: dryRun,
ReplicationConfig: &s.parent.policy.ReplicationConfig,
ReplicationConfig: s.parent.policy.ReplicationConfig,
}
return sr
}
@@ -614,7 +617,7 @@ func (s *Step) doReplication(ctx context.Context) error {
Filesystem: fs,
To: sr.GetTo(),
ClearResumeToken: !sres.UsedResumeToken,
ReplicationConfig: &s.parent.policy.ReplicationConfig,
ReplicationConfig: s.parent.policy.ReplicationConfig,
}
log.Debug("initiate receive request")
_, err = s.receiver.Receive(ctx, rr, byteCountingStream)
replication/logic/replication_logic_policy.go
+10 -2
View File
@@ -1,6 +1,7 @@
package logic
import (
"github.com/go-playground/validator"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
@@ -8,8 +9,15 @@ import (
)
type PlannerPolicy struct {
EncryptedSend tri // all sends must be encrypted (send -w, and encryption!=off)
ReplicationConfig pdu.ReplicationConfig
EncryptedSend tri // all sends must be encrypted (send -w, and encryption!=off)
ReplicationConfig *pdu.ReplicationConfig
SizeEstimationConcurrency int `validate:"gte=1"`
}
var validate = validator.New()
func (p PlannerPolicy) Validate() error {
return validate.Struct(p)
}
func ReplicationConfigFromConfig(in *config.Replication) (*pdu.ReplicationConfig, error) {
replication/replication.go
+2 -2
View File
@@ -8,6 +8,6 @@ import (
"github.com/zrepl/zrepl/replication/driver"
)
func Do(ctx context.Context, planner driver.Planner) (driver.ReportFunc, driver.WaitFunc) {
return driver.Do(ctx, planner)
func Do(ctx context.Context, driverConfig driver.Config, planner driver.Planner) (driver.ReportFunc, driver.WaitFunc) {
return driver.Do(ctx, driverConfig, planner)
}
rpc/dataconn/dataconn_client.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"io"
"strings"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"
"github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/rpc/dataconn/stream"
rpc/dataconn/dataconn_debug.go
+1
View File
@@ -5,6 +5,7 @@ import (
"os"
)
//nolint:unused
var debugEnabled bool = false
func init() {
rpc/dataconn/dataconn_server.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"io"
"sync"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/replication/logic/pdu"
rpc/dataconn/frameconn/frameconn.go
+3 -3
View File
@@ -39,7 +39,7 @@ func assertPublicFrameType(frameType uint32) {
func (f *FrameHeader) Unmarshal(buf []byte) {
if len(buf) != 8 {
panic(fmt.Sprintf("frame header is 8 bytes long"))
panic("frame header is 8 bytes long")
}
f.Type = binary.BigEndian.Uint32(buf[0:4])
f.PayloadLen = binary.BigEndian.Uint32(buf[4:8])
@@ -47,7 +47,7 @@ func (f *FrameHeader) Unmarshal(buf []byte) {
type Conn struct {
readMtx, writeMtx sync.Mutex
nc timeoutconn.Conn
nc *timeoutconn.Conn
readNextValid bool
readNext FrameHeader
nextReadErr error
@@ -55,7 +55,7 @@ type Conn struct {
shutdown shutdownFSM
}
func Wrap(nc timeoutconn.Conn) *Conn {
func Wrap(nc *timeoutconn.Conn) *Conn {
return &Conn{
nc: nc,
// ncBuf: bufio.NewReadWriter(bufio.NewReaderSize(nc, 1<<23), bufio.NewWriterSize(nc, 1<<23)),
rpc/dataconn/stream/stream_conn.go
+1
View File
@@ -144,6 +144,7 @@ func (c *Conn) ReadStream(frameType uint32, closeConnOnClose bool) (_ *StreamRea
c.readMtx.Lock()
if !c.readClean {
c.readMtx.Unlock()
return nil, errWriteStreamToErrorUnknownState
}
rpc/dataconn/timeoutconn/timeoutconn.go
+36 -15
View File
@@ -11,7 +11,7 @@ import (
"errors"
"io"
"net"
"sync/atomic"
"sync"
"syscall"
"time"
)
@@ -54,39 +54,60 @@ type Wire interface {
}
type Conn struct {
// immutable state
Wire
renewDeadlinesDisabled int32
idleTimeout time.Duration
idleTimeout time.Duration
// mutable state (protected by mtx)
mtx sync.RWMutex
renewDeadlinesDisabled bool
}
func Wrap(conn Wire, idleTimeout time.Duration) Conn {
return Conn{Wire: conn, idleTimeout: idleTimeout}
func Wrap(conn Wire, idleTimeout time.Duration) *Conn {
return &Conn{
Wire: conn,
idleTimeout: idleTimeout,
}
}
// DisableTimeouts disables the idle timeout behavior provided by this package.
// Existing deadlines are cleared iff the call is the first call to this method.
// Existing deadlines are cleared iff the call is the first call to this method
// or if the previous call produced an error.
func (c *Conn) DisableTimeouts() error {
if atomic.CompareAndSwapInt32(&c.renewDeadlinesDisabled, 0, 1) {
return c.SetDeadline(time.Time{})
c.mtx.Lock()
defer c.mtx.Unlock()
if c.renewDeadlinesDisabled {
return nil
}
err := c.SetDeadline(time.Time{})
if err != nil {
return err
}
c.renewDeadlinesDisabled = true
return nil
}
func (c *Conn) renewReadDeadline() error {
if atomic.LoadInt32(&c.renewDeadlinesDisabled) != 0 {
c.mtx.RLock()
defer c.mtx.RUnlock()
if c.renewDeadlinesDisabled {
return nil
}
return c.SetReadDeadline(time.Now().Add(c.idleTimeout))
}
func (c *Conn) RenewWriteDeadline() error {
if atomic.LoadInt32(&c.renewDeadlinesDisabled) != 0 {
c.mtx.RLock()
defer c.mtx.RUnlock()
if c.renewDeadlinesDisabled {
return nil
}
return c.SetWriteDeadline(time.Now().Add(c.idleTimeout))
}
func (c Conn) Read(p []byte) (n int, err error) {
func (c *Conn) Read(p []byte) (n int, err error) {
n = 0
err = nil
restart:
@@ -103,7 +124,7 @@ restart:
return n, err
}
func (c Conn) Write(p []byte) (n int, err error) {
func (c *Conn) Write(p []byte) (n int, err error) {
n = 0
restart:
if err := c.RenewWriteDeadline(); err != nil {
@@ -123,7 +144,7 @@ restart:
// but is guaranteed to use the writev system call if the wrapped Wire
// support it.
// Note the Conn does not support writev through io.Copy(aConn, aNetBuffers).
func (c Conn) WritevFull(bufs net.Buffers) (n int64, err error) {
func (c *Conn) WritevFull(bufs net.Buffers) (n int64, err error) {
n = 0
restart:
if err := c.RenewWriteDeadline(); err != nil {
@@ -163,12 +184,12 @@ var _ SyscallConner = (*net.TCPConn)(nil)
// If the connection returned io.EOF, the number of bytes written until
// then + io.EOF is returned. This behavior is different to io.ReadFull
// which returns io.ErrUnexpectedEOF.
func (c Conn) ReadvFull(buffers net.Buffers) (n int64, err error) {
func (c *Conn) ReadvFull(buffers net.Buffers) (n int64, err error) {
return c.readv(buffers)
}
// invoked by c.readv if readv system call cannot be used
func (c Conn) readvFallback(nbuffers net.Buffers) (n int64, err error) {
func (c *Conn) readvFallback(nbuffers net.Buffers) (n int64, err error) {
buffers := [][]byte(nbuffers)
for i := range buffers {
curBuf := buffers[i]
rpc/dataconn/timeoutconn/timeoutconn_readv_unsupported.go
+1 -1
View File
@@ -4,7 +4,7 @@ package timeoutconn
import "net"
func (c Conn) readv(buffers net.Buffers) (n int64, err error) {
func (c *Conn) readv(buffers net.Buffers) (n int64, err error) {
// Go does not expose the SYS_READV symbol for Solaris / Illumos - do they have it?
// Anyhow, use the fallback
return c.readvFallback(buffers)
rpc/dataconn/timeoutconn/timoutconn_readv.go
+2 -3
View File
@@ -29,8 +29,7 @@ func buildIovecs(buffers net.Buffers) (totalLen int64, vecs []syscall.Iovec) {
return totalLen, vecs
}
func (c Conn) readv(buffers net.Buffers) (n int64, err error) {
func (c *Conn) readv(buffers net.Buffers) (n int64, err error) {
scc, ok := c.Wire.(SyscallConner)
if !ok {
return c.readvFallback(buffers)
@@ -62,7 +61,7 @@ func (c Conn) readv(buffers net.Buffers) (n int64, err error) {
return n, nil
}
func (c Conn) doOneReadv(rawConn syscall.RawConn, iovecs *[]syscall.Iovec) (n int64, err error) {
func (c *Conn) doOneReadv(rawConn syscall.RawConn, iovecs *[]syscall.Iovec) (n int64, err error) {
rawReadErr := rawConn.Read(func(fd uintptr) (done bool) {
// iovecs, n and err must not be shadowed!
rpc/grpcclientidentity/authlistener_grpc_adaptor.go
+2 -5
View File
@@ -16,7 +16,6 @@ import (
"context"
"fmt"
"net"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
@@ -28,12 +27,10 @@ import (
type Logger = logger.Logger
type GRPCDialFunction = func(string, time.Duration) (net.Conn, error)
type GRPCDialFunction = func(context.Context, string) (net.Conn, error)
func NewDialer(logger Logger, connecter transport.Connecter) GRPCDialFunction {
return func(s string, duration time.Duration) (conn net.Conn, e error) {
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
return func(ctx context.Context, s string) (conn net.Conn, e error) {
nc, err := connecter.Connect(ctx)
// TODO find better place (callback from gRPC?) where to log errors
// we want the users to know, though
rpc/grpcclientidentity/example/grpcauth.proto
+2 -2
View File
@@ -1,8 +1,8 @@
syntax = "proto3";
option go_package = ".;pdu";
package pdu;
service Greeter {
rpc Greet(GreetRequest) returns (GreetResponse) {}
}
@@ -13,4 +13,4 @@ message GreetRequest {
message GreetResponse {
string msg = 1;
}
}
rpc/grpcclientidentity/example/main.go
+2 -1
View File
@@ -94,7 +94,7 @@ func server() {
srv, serve := grpchelper.NewServer(authListener, clientIdentityKey, log, nil)
svc := &greeter{"hello "}
svc := &greeter{prepend: "hello "}
pdu.RegisterGreeterServer(srv, svc)
if err := serve(); err != nil {
@@ -103,6 +103,7 @@ func server() {
}
type greeter struct {
pdu.UnsafeGreeterServer
prepend string
}
rpc/grpcclientidentity/example/pdu/grpcauth.pb.go
+172 -151
View File
@@ -1,193 +1,214 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.25.0
// protoc v3.12.4
// source: grpcauth.proto
package pdu
import (
fmt "fmt"
proto "github.com/golang/protobuf/proto"
context "golang.org/x/net/context"
grpc "google.golang.org/grpc"
math "math"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// This is a compile-time assertion that a sufficiently up-to-date version
// of the legacy proto package is being used.
const _ = proto.ProtoPackageIsVersion4
type GreetRequest struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
}
func (m *GreetRequest) Reset() { *m = GreetRequest{} }
func (m *GreetRequest) String() string { return proto.CompactTextString(m) }
func (*GreetRequest) ProtoMessage() {}
func (x *GreetRequest) Reset() {
*x = GreetRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_grpcauth_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *GreetRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GreetRequest) ProtoMessage() {}
func (x *GreetRequest) ProtoReflect() protoreflect.Message {
mi := &file_grpcauth_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GreetRequest.ProtoReflect.Descriptor instead.
func (*GreetRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_1dfba7be0cf69353, []int{0}
return file_grpcauth_proto_rawDescGZIP(), []int{0}
}
func (m *GreetRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GreetRequest.Unmarshal(m, b)
}
func (m *GreetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GreetRequest.Marshal(b, m, deterministic)
}
func (m *GreetRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_GreetRequest.Merge(m, src)
}
func (m *GreetRequest) XXX_Size() int {
return xxx_messageInfo_GreetRequest.Size(m)
}
func (m *GreetRequest) XXX_DiscardUnknown() {
xxx_messageInfo_GreetRequest.DiscardUnknown(m)
}
var xxx_messageInfo_GreetRequest proto.InternalMessageInfo
func (m *GreetRequest) GetName() string {
if m != nil {
return m.Name
func (x *GreetRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
type GreetResponse struct {
Msg string `protobuf:"bytes,1,opt,name=msg,proto3" json:"msg,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Msg string `protobuf:"bytes,1,opt,name=msg,proto3" json:"msg,omitempty"`
}
func (m *GreetResponse) Reset() { *m = GreetResponse{} }
func (m *GreetResponse) String() string { return proto.CompactTextString(m) }
func (*GreetResponse) ProtoMessage() {}
func (x *GreetResponse) Reset() {
*x = GreetResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_grpcauth_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *GreetResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GreetResponse) ProtoMessage() {}
func (x *GreetResponse) ProtoReflect() protoreflect.Message {
mi := &file_grpcauth_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GreetResponse.ProtoReflect.Descriptor instead.
func (*GreetResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_1dfba7be0cf69353, []int{1}
return file_grpcauth_proto_rawDescGZIP(), []int{1}
}
func (m *GreetResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_GreetResponse.Unmarshal(m, b)
}
func (m *GreetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_GreetResponse.Marshal(b, m, deterministic)
}
func (m *GreetResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_GreetResponse.Merge(m, src)
}
func (m *GreetResponse) XXX_Size() int {
return xxx_messageInfo_GreetResponse.Size(m)
}
func (m *GreetResponse) XXX_DiscardUnknown() {
xxx_messageInfo_GreetResponse.DiscardUnknown(m)
}
var xxx_messageInfo_GreetResponse proto.InternalMessageInfo
func (m *GreetResponse) GetMsg() string {
if m != nil {
return m.Msg
func (x *GreetResponse) GetMsg() string {
if x != nil {
return x.Msg
}
return ""
}
func init() {
proto.RegisterType((*GreetRequest)(nil), "pdu.GreetRequest")
proto.RegisterType((*GreetResponse)(nil), "pdu.GreetResponse")
var File_grpcauth_proto protoreflect.FileDescriptor
var file_grpcauth_proto_rawDesc = []byte{
0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x61, 0x75, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x12, 0x03, 0x70, 0x64, 0x75, 0x22, 0x22, 0x0a, 0x0c, 0x47, 0x72, 0x65, 0x65, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x21, 0x0a, 0x0d, 0x47, 0x72, 0x65,
0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73,
0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x32, 0x3b, 0x0a, 0x07,
0x47, 0x72, 0x65, 0x65, 0x74, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x05, 0x47, 0x72, 0x65, 0x65, 0x74,
0x12, 0x11, 0x2e, 0x70, 0x64, 0x75, 0x2e, 0x47, 0x72, 0x65, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x70, 0x64, 0x75, 0x2e, 0x47, 0x72, 0x65, 0x65, 0x74, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x07, 0x5a, 0x05, 0x2e, 0x3b, 0x70,
0x64, 0x75, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
func init() { proto.RegisterFile("grpcauth.proto", fileDescriptor_1dfba7be0cf69353) }
var (
file_grpcauth_proto_rawDescOnce sync.Once
file_grpcauth_proto_rawDescData = file_grpcauth_proto_rawDesc
)
var fileDescriptor_1dfba7be0cf69353 = []byte{
// 137 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x2f, 0x2a, 0x48,
0x4e, 0x2c, 0x2d, 0xc9, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2e, 0x48, 0x29, 0x55,
0x52, 0xe2, 0xe2, 0x71, 0x2f, 0x4a, 0x4d, 0x2d, 0x09, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11,
0x12, 0xe2, 0x62, 0xc9, 0x4b, 0xcc, 0x4d, 0x95, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x02, 0xb3,
0x95, 0x14, 0xb9, 0x78, 0xa1, 0x6a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0x04, 0xb8, 0x98,
0x73, 0x8b, 0xd3, 0xa1, 0x6a, 0x40, 0x4c, 0x23, 0x6b, 0x2e, 0x76, 0xb0, 0x92, 0xd4, 0x22, 0x21,
0x03, 0x2e, 0x56, 0x30, 0x53, 0x48, 0x50, 0xaf, 0x20, 0xa5, 0x54, 0x0f, 0xd9, 0x74, 0x29, 0x21,
0x64, 0x21, 0x88, 0x61, 0x4a, 0x0c, 0x49, 0x6c, 0x60, 0xf7, 0x18, 0x03, 0x02, 0x00, 0x00, 0xff,
0xff, 0xa8, 0x53, 0x2f, 0x4c, 0xa1, 0x00, 0x00, 0x00,
func file_grpcauth_proto_rawDescGZIP() []byte {
file_grpcauth_proto_rawDescOnce.Do(func() {
file_grpcauth_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpcauth_proto_rawDescData)
})
return file_grpcauth_proto_rawDescData
}
// Reference imports to suppress errors if they are not otherwise used.
var _ context.Context
var _ grpc.ClientConn
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion4
// GreeterClient is the client API for Greeter service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type GreeterClient interface {
Greet(ctx context.Context, in *GreetRequest, opts ...grpc.CallOption) (*GreetResponse, error)
var file_grpcauth_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_grpcauth_proto_goTypes = []interface{}{
(*GreetRequest)(nil), // 0: pdu.GreetRequest
(*GreetResponse)(nil), // 1: pdu.GreetResponse
}
var file_grpcauth_proto_depIdxs = []int32{
0, // 0: pdu.Greeter.Greet:input_type -> pdu.GreetRequest
1, // 1: pdu.Greeter.Greet:output_type -> pdu.GreetResponse
1, // [1:2] is the sub-list for method output_type
0, // [0:1] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
type greeterClient struct {
cc *grpc.ClientConn
}
func NewGreeterClient(cc *grpc.ClientConn) GreeterClient {
return &greeterClient{cc}
}
func (c *greeterClient) Greet(ctx context.Context, in *GreetRequest, opts ...grpc.CallOption) (*GreetResponse, error) {
out := new(GreetResponse)
err := c.cc.Invoke(ctx, "/pdu.Greeter/Greet", in, out, opts...)
if err != nil {
return nil, err
func init() { file_grpcauth_proto_init() }
func file_grpcauth_proto_init() {
if File_grpcauth_proto != nil {
return
}
return out, nil
}
// GreeterServer is the server API for Greeter service.
type GreeterServer interface {
Greet(context.Context, *GreetRequest) (*GreetResponse, error)
}
func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) {
s.RegisterService(&_Greeter_serviceDesc, srv)
}
func _Greeter_Greet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GreetRequest)
if err := dec(in); err != nil {
return nil, err
if !protoimpl.UnsafeEnabled {
file_grpcauth_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GreetRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_grpcauth_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GreetResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
if interceptor == nil {
return srv.(GreeterServer).Greet(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pdu.Greeter/Greet",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GreeterServer).Greet(ctx, req.(*GreetRequest))
}
return interceptor(ctx, in, info, handler)
}
var _Greeter_serviceDesc = grpc.ServiceDesc{
ServiceName: "pdu.Greeter",
HandlerType: (*GreeterServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Greet",
Handler: _Greeter_Greet_Handler,
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_grpcauth_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "grpcauth.proto",
GoTypes: file_grpcauth_proto_goTypes,
DependencyIndexes: file_grpcauth_proto_depIdxs,
MessageInfos: file_grpcauth_proto_msgTypes,
}.Build()
File_grpcauth_proto = out.File
file_grpcauth_proto_rawDesc = nil
file_grpcauth_proto_goTypes = nil
file_grpcauth_proto_depIdxs = nil
}
rpc/grpcclientidentity/example/pdu/grpcauth_grpc.pb.go
+101
View File
@@ -0,0 +1,101 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
package pdu
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// GreeterClient is the client API for Greeter service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type GreeterClient interface {
Greet(ctx context.Context, in *GreetRequest, opts ...grpc.CallOption) (*GreetResponse, error)
}
type greeterClient struct {
cc grpc.ClientConnInterface
}
func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient {
return &greeterClient{cc}
}
func (c *greeterClient) Greet(ctx context.Context, in *GreetRequest, opts ...grpc.CallOption) (*GreetResponse, error) {
out := new(GreetResponse)
err := c.cc.Invoke(ctx, "/pdu.Greeter/Greet", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// GreeterServer is the server API for Greeter service.
// All implementations must embed UnimplementedGreeterServer
// for forward compatibility
type GreeterServer interface {
Greet(context.Context, *GreetRequest) (*GreetResponse, error)
mustEmbedUnimplementedGreeterServer()
}
// UnimplementedGreeterServer must be embedded to have forward compatible implementations.
type UnimplementedGreeterServer struct {
}
func (UnimplementedGreeterServer) Greet(context.Context, *GreetRequest) (*GreetResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Greet not implemented")
}
func (UnimplementedGreeterServer) mustEmbedUnimplementedGreeterServer() {}
// UnsafeGreeterServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to GreeterServer will
// result in compilation errors.
type UnsafeGreeterServer interface {
mustEmbedUnimplementedGreeterServer()
}
func RegisterGreeterServer(s grpc.ServiceRegistrar, srv GreeterServer) {
s.RegisterService(&Greeter_ServiceDesc, srv)
}
func _Greeter_Greet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GreetRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GreeterServer).Greet(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/pdu.Greeter/Greet",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GreeterServer).Greet(ctx, req.(*GreetRequest))
}
return interceptor(ctx, in, info, handler)
}
// Greeter_ServiceDesc is the grpc.ServiceDesc for Greeter service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Greeter_ServiceDesc = grpc.ServiceDesc{
ServiceName: "pdu.Greeter",
HandlerType: (*GreeterServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Greet",
Handler: _Greeter_Greet_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "grpcauth.proto",
}
rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go
+3 -1
View File
@@ -34,8 +34,10 @@ func ClientConn(cn transport.Connecter, log Logger) *grpc.ClientConn {
Timeout: KeepalivePeerTimeout,
PermitWithoutStream: true,
})
dialerOption := grpc.WithDialer(grpcclientidentity.NewDialer(log, cn))
dialerOption := grpc.WithContextDialer(grpcclientidentity.NewDialer(log, cn))
cred := grpc.WithTransportCredentials(grpcclientidentity.NewTransportCredentials(log))
// we use context.Background without a timeout here because we don't set grpc.WithBlock
// => docs: "In the non-blocking case, the ctx does not act against the connection. It only controls the setup steps."
cc, err := grpc.DialContext(context.Background(), "doesn't matter done by dialer", dialerOption, cred, ka)
if err != nil {
log.WithError(err).Error("cannot create gRPC client conn (non-blocking)")
rpc/rpc_client.go
+2 -2
View File
@@ -172,14 +172,14 @@ func (c *Client) WaitForConnectivity(ctx context.Context) error {
}
go func() {
defer wg.Done()
ctrl, ctrlErr := c.controlClient.Ping(ctx, &req, grpc.FailFast(false))
ctrl, ctrlErr := c.controlClient.Ping(ctx, &req, grpc.WaitForReady(true))
checkRes(ctrl, ctrlErr, loggers.Control, &ctrlOk)
}()
go func() {
defer wg.Done()
for ctx.Err() == nil {
data, dataErr := c.dataClient.ReqPing(ctx, &req)
// dataClient uses transport.Connecter, which doesn't expose FailFast(false)
// dataClient uses transport.Connecter, which doesn't expose WaitForReady(true)
// => we need to mask dial timeouts
if err, ok := dataErr.(interface{ Temporary() bool }); ok && err.Temporary() {
// Rate-limit pings here in case Temporary() is a mis-classification
rpc/rpc_debug.go
+1
View File
@@ -5,6 +5,7 @@ import (
"os"
)
//nolint:unused
var debugEnabled bool = false
func init() {
util/choices/choices.go
+98
View File
@@ -0,0 +1,98 @@
// Package choice implements a flag.Value type that accepts a set of choices.
//
// See test cases or grep the code base for usage hints.
package choices
import (
"flag"
"fmt"
"strings"
)
type Choices struct {
choices map[string]interface{}
typeString string
value interface{}
}
var _ flag.Value = (*Choices)(nil)
func new(pairs ...interface{}) Choices {
if (len(pairs) % 2) != 0 {
panic("must provide a sequence of key value pairs")
}
c := Choices{
choices: make(map[string]interface{}, len(pairs)/2),
value: nil,
}
for i := 0; i < len(pairs); {
key, ok := pairs[i].(string)
if !ok {
panic(fmt.Sprintf("argument %d is %T but should be a string, value: %#v", i, pairs[i], pairs[i]))
}
c.choices[key] = pairs[i+1]
i += 2
}
c.typeString = strings.Join(c.choicesList(true), ",") // overrideable by setter
return c
}
func (c *Choices) Init(pairs ...interface{}) {
*c = new(pairs...)
}
func (c Choices) choicesList(escaped bool) []string {
keys := make([]string, len(c.choices))
i := 0
for k := range c.choices {
e := k
if escaped {
e = fmt.Sprintf("%q", k)
}
keys[i] = e
i += 1
}
return keys
}
func (c Choices) Usage() string {
return fmt.Sprintf("one of %s", strings.Join(c.choicesList(true), ","))
}
func (c Choices) InputForChoice(v interface{}) (string, error) {
for input, choice := range c.choices {
if choice == v {
return input, nil
}
}
return "", fmt.Errorf("choice not registered at .Init(): %v", v)
}
func (c *Choices) SetDefaultValue(v interface{}) {
c.value = v
}
func (c Choices) Value() interface{} {
return c.value
}
func (c *Choices) Set(input string) error {
v, ok := c.choices[input]
if !ok {
return fmt.Errorf("invalid value %q: must be one of %s", input, c.Usage())
}
c.value = v
return nil
}
func (c *Choices) String() string {
return "" // c.value.(fmt.Stringer).String()
}
func (c *Choices) SetTypeString(ts string) {
c.typeString = ts
}
func (c *Choices) Type() string {
return c.typeString
}
util/choices/choices_test.go
+50
View File
@@ -0,0 +1,50 @@
package choices_test
import (
"bytes"
"flag"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/util/choices"
)
func TestChoices(t *testing.T) {
var c choices.Choices
fs := flag.NewFlagSet("testset", flag.ContinueOnError)
c.Init("append", os.O_APPEND, "overwrite", os.O_TRUNC|os.O_CREATE)
fs.Var(&c, "mode", c.Usage())
var o bytes.Buffer
fs.SetOutput(&o)
fs.Usage()
usage := o.String()
o.Reset()
t.Logf("usage:\n%s", usage)
require.Contains(t, usage, "\"append\"")
require.Contains(t, usage, "\"overwrite\"")
err := fs.Parse([]string{"-mode", "append"})
require.NoError(t, err)
o.Reset()
require.Equal(t, os.O_APPEND, c.Value())
c.SetDefaultValue(nil)
err = fs.Parse([]string{})
require.NoError(t, err)
o.Reset()
require.Nil(t, c.Value())
// a little whitebox testing: this is allowed ATM, we don't check that the default value was specified as a choice in init
c.SetDefaultValue(os.O_RDWR)
err = fs.Parse([]string{})
require.NoError(t, err)
o.Reset()
require.Equal(t, os.O_RDWR, c.Value())
}
util/nodefault/nodefault.go
+32
View File
@@ -0,0 +1,32 @@
// Package nodefault provides newtypes around builtin Go types
// so that if the newtype is used as a field in a struct and that field
// is zero-initialized (https://golang.org/ref/spec#The_zero_value),
// accessing the field will cause a null pointer deref panic.
// Or in other terms: It soft-enforces that the caller sets the field
// explicitly when constructing the struct.
//
// Example:
//
// type Config struct {
// // This field must be set to a non-nil value,
// // forcing the caller to make their mind up
// // about this field.
// CriticalSetting *nodefault.Bool
// }
//
// An function that takes such a Config should _not_ check for nil-ness:
// and instead unconditionally dereference:
//
// func f(c Config) {
// if (c.CriticalSetting) { }
// }
//
// If the caller of f forgot to specify the .CriticalSetting
// field, the Go runtime will issue a nil-pointer deref panic
// and it'll be clear that the caller did not read the docs of Config.
//
// f(Config{}) // crashes
//
// f Config{ CriticalSetting: &nodefault.Bool{B: false}} // doesn't crash
//
package nodefault
util/nodefault/nodefault_bool.go
+19
View File
@@ -0,0 +1,19 @@
package nodefault
import "fmt"
type Bool struct{ B bool }
func (n *Bool) ValidateNoDefault() error {
if n == nil {
return fmt.Errorf("must explicitly set `true` or `false`")
}
return nil
}
func (n *Bool) String() string {
if n == nil {
return "unset"
}
return fmt.Sprintf("%v", n.B)
}
util/optionaldeadline/optionaldeadline_test.go
+6 -2
View File
@@ -93,8 +93,12 @@ func TestContextWithOptionalDeadlineParentCancellation(t *testing.T) {
}
type testContextKey string
const testContextKeyKey testContextKey = "key"
func TestContextWithOptionalDeadlineValue(t *testing.T) {
pctx := context.WithValue(context.Background(), "key", "value")
pctx := context.WithValue(context.Background(), testContextKeyKey, "value")
cctx, _ := ContextWithOptionalDeadline(pctx)
assert.Equal(t, "value", cctx.Value("key"))
assert.Equal(t, "value", cctx.Value(testContextKeyKey))
}
util/semaphore/semaphore_test.go
+10 -14
View File
@@ -2,7 +2,6 @@ package semaphore
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
@@ -15,7 +14,7 @@ import (
func TestSemaphore(t *testing.T) {
const numGoroutines = 10
const concurrentSemaphore = 5
const concurrentSemaphore = 6
const sleepTime = 1 * time.Second
begin := time.Now()
@@ -26,29 +25,26 @@ func TestSemaphore(t *testing.T) {
beforeT, afterT uint32
}
ctx := context.Background()
defer trace.WithTaskFromStackUpdateCtx(&ctx)()
rootCtx, endRoot := trace.WithTaskFromStack(context.Background())
defer endRoot()
_, add, waitEnd := trace.WithTaskGroup(rootCtx, "TestSemaphore")
var wg sync.WaitGroup
wg.Add(numGoroutines)
for i := 0; i < numGoroutines; i++ {
go func() {
ctx, end := trace.WithTaskFromStack(ctx)
defer end()
defer wg.Done()
// not capturing i so no need for local copy
add(func(ctx context.Context) {
res, err := sem.Acquire(ctx)
require.NoError(t, err)
defer res.Release()
if time.Since(begin) > sleepTime {
atomic.AddUint32(&acquisitions.beforeT, 1)
} else {
atomic.AddUint32(&acquisitions.afterT, 1)
} else {
atomic.AddUint32(&acquisitions.beforeT, 1)
}
time.Sleep(sleepTime)
}()
})
}
wg.Wait()
waitEnd()
assert.True(t, acquisitions.beforeT == concurrentSemaphore)
assert.True(t, acquisitions.afterT == numGoroutines-concurrentSemaphore)
zfs/encryption.go
+1 -1
View File
@@ -51,7 +51,7 @@ func ZFSGetEncryptionEnabled(ctx context.Context, fs string) (enabled bool, err
return false, err
}
props, err := zfsGet(ctx, fs, []string{"encryption"}, sourceAny)
props, err := zfsGet(ctx, fs, []string{"encryption"}, SourceAny)
if err != nil {
return false, errors.Wrap(err, "cannot get `encryption` property")
}
zfs/placeholder.go
+2 -3
View File
@@ -67,7 +67,7 @@ type FilesystemPlaceholderState struct {
func ZFSGetFilesystemPlaceholderState(ctx context.Context, p *DatasetPath) (state *FilesystemPlaceholderState, err error) {
state = &FilesystemPlaceholderState{FS: p.ToString()}
state.FS = p.ToString()
props, err := zfsGet(ctx, p.ToString(), []string{PlaceholderPropertyName}, sourceLocal)
props, err := zfsGet(ctx, p.ToString(), []string{PlaceholderPropertyName}, SourceLocal)
var _ error = (*DatasetDoesNotExist)(nil) // weak assertion on zfsGet's interface
if _, ok := err.(*DatasetDoesNotExist); ok {
return state, nil
@@ -110,12 +110,11 @@ func ZFSCreatePlaceholderFilesystem(ctx context.Context, fs *DatasetPath, parent
}
func ZFSSetPlaceholder(ctx context.Context, p *DatasetPath, isPlaceholder bool) error {
props := NewZFSProperties()
prop := placeholderPropertyOff
if isPlaceholder {
prop = placeholderPropertyOn
}
props.Set(PlaceholderPropertyName, prop)
props := map[string]string{PlaceholderPropertyName: prop}
return zfsSet(ctx, p.ToString(), props)
}
zfs/property/property.go
+33
View File
@@ -0,0 +1,33 @@
package property
import (
"fmt"
"regexp"
)
type Property string
// Check property name conforms to zfsprops(8), section "User Properties"
// Keep regex and error message in sync!
var (
propertyValidNameChars = regexp.MustCompile(`^[0-9a-zA-Z-_\.:]+$`)
propertyValidNameCharsErr = fmt.Errorf("property name must only contain alphanumeric chars and any in %q", "-_.:")
)
func (p Property) Validate() error {
const PROPERTYNAMEMAXLEN int = 256
if len(p) < 1 {
return fmt.Errorf("property name cannot be empty")
}
if len(p) > PROPERTYNAMEMAXLEN {
return fmt.Errorf("property name longer than %d characters", PROPERTYNAMEMAXLEN)
}
if p[0] == '-' {
return fmt.Errorf("property name cannot start with '-'")
}
if !propertyValidNameChars.MatchString(string(p)) {
return propertyValidNameCharsErr
}
return nil
}
zfs/propertysource_enumer.go
+82
View File
@@ -0,0 +1,82 @@
// Code generated by "enumer -type=PropertySource -trimprefix=Source"; DO NOT EDIT.
//
package zfs
import (
"fmt"
)
const (
_PropertySourceName_0 = "LocalDefault"
_PropertySourceName_1 = "Inherited"
_PropertySourceName_2 = "None"
_PropertySourceName_3 = "Temporary"
_PropertySourceName_4 = "Received"
_PropertySourceName_5 = "Any"
)
var (
_PropertySourceIndex_0 = [...]uint8{0, 5, 12}
_PropertySourceIndex_1 = [...]uint8{0, 9}
_PropertySourceIndex_2 = [...]uint8{0, 4}
_PropertySourceIndex_3 = [...]uint8{0, 9}
_PropertySourceIndex_4 = [...]uint8{0, 8}
_PropertySourceIndex_5 = [...]uint8{0, 3}
)
func (i PropertySource) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _PropertySourceName_0[_PropertySourceIndex_0[i]:_PropertySourceIndex_0[i+1]]
case i == 4:
return _PropertySourceName_1
case i == 8:
return _PropertySourceName_2
case i == 16:
return _PropertySourceName_3
case i == 32:
return _PropertySourceName_4
case i == 4294967295:
return _PropertySourceName_5
default:
return fmt.Sprintf("PropertySource(%d)", i)
}
}
var _PropertySourceValues = []PropertySource{1, 2, 4, 8, 16, 32, 4294967295}
var _PropertySourceNameToValueMap = map[string]PropertySource{
_PropertySourceName_0[0:5]: 1,
_PropertySourceName_0[5:12]: 2,
_PropertySourceName_1[0:9]: 4,
_PropertySourceName_2[0:4]: 8,
_PropertySourceName_3[0:9]: 16,
_PropertySourceName_4[0:8]: 32,
_PropertySourceName_5[0:3]: 4294967295,
}
// PropertySourceString retrieves an enum value from the enum constants string name.
// Throws an error if the param is not part of the enum.
func PropertySourceString(s string) (PropertySource, error) {
if val, ok := _PropertySourceNameToValueMap[s]; ok {
return val, nil
}
return 0, fmt.Errorf("%s does not belong to PropertySource values", s)
}
// PropertySourceValues returns all values of the enum
func PropertySourceValues() []PropertySource {
return _PropertySourceValues
}
// IsAPropertySource returns "true" if the value is listed in the enum definition. "false" otherwise
func (i PropertySource) IsAPropertySource() bool {
for _, v := range _PropertySourceValues {
if i == v {
return true
}
}
return false
}
zfs/resume_token.go
+26 -5
View File
@@ -18,11 +18,14 @@ import (
// NOTE: Update ZFSSendARgs.Validate when changing fields (potentially SECURITY SENSITIVE)
type ResumeToken struct {
HasFromGUID, HasToGUID bool
FromGUID, ToGUID uint64
ToName string
HasCompressOK, CompressOK bool
HasRawOk, RawOK bool
HasFromGUID, HasToGUID bool
FromGUID, ToGUID uint64
ToName string
HasCompressOK, CompressOK bool
HasRawOk, RawOK bool
HasLargeBlockOK, LargeBlockOK bool
HasEmbedOk, EmbedOK bool
HasSavedOk, SavedOk bool
}
var resumeTokenNVListRE = regexp.MustCompile(`\t(\S+) = (.*)`)
@@ -240,6 +243,24 @@ func ParseResumeToken(ctx context.Context, token string) (*ResumeToken, error) {
if err != nil {
return nil, ResumeTokenParsingError
}
case "embedok":
rt.HasEmbedOk = true
rt.EmbedOK, err = strconv.ParseBool(val)
if err != nil {
return nil, ResumeTokenParsingError
}
case "largeblockok":
rt.HasLargeBlockOK = true
rt.LargeBlockOK, err = strconv.ParseBool(val)
if err != nil {
return nil, ResumeTokenParsingError
}
case "savedok":
rt.HasSavedOk = true
rt.SavedOk, err = strconv.ParseBool(val)
if err != nil {
return nil, ResumeTokenParsingError
}
}
}
zfs/versions.go
+1 -1
View File
@@ -267,7 +267,7 @@ func ZFSListFilesystemVersions(ctx context.Context, fs *DatasetPath, options Lis
}
func ZFSGetFilesystemVersion(ctx context.Context, ds string) (v FilesystemVersion, _ error) {
props, err := zfsGet(ctx, ds, []string{"createtxg", "guid", "creation", "userrefs"}, sourceAny)
props, err := zfsGet(ctx, ds, []string{"createtxg", "guid", "creation", "userrefs"}, SourceAny)
if err != nil {
return v, err
}
zfs/zfs.go
+238 -138
View File
@@ -21,6 +21,8 @@ import (
"github.com/zrepl/zrepl/util/circlog"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/util/nodefault"
zfsprop "github.com/zrepl/zrepl/zfs/property"
"github.com/zrepl/zrepl/zfs/zfscmd"
)
@@ -325,45 +327,6 @@ func absVersion(fs string, v *ZFSSendArgVersion) (full string, err error) {
return fmt.Sprintf("%s%s", fs, v.RelName), nil
}
// tok is allowed to be nil
// a must already be validated
//
// SECURITY SENSITIVE because Raw must be handled correctly
func (a ZFSSendArgsUnvalidated) buildCommonSendArgs() ([]string, error) {
args := make([]string, 0, 3)
// ResumeToken takes precedence, we assume that it has been validated to reflect
// what is described by the other fields in ZFSSendArgs
if a.ResumeToken != "" {
args = append(args, "-t", a.ResumeToken)
return args, nil
}
if a.Encrypted.B {
args = append(args, "-w")
}
toV, err := absVersion(a.FS, a.To)
if err != nil {
return nil, err
}
fromV := ""
if a.From != nil {
fromV, err = absVersion(a.FS, a.From)
if err != nil {
return nil, err
}
}
if fromV == "" { // Initial
args = append(args, toV)
} else {
args = append(args, "-i", fromV, toV)
}
return args, nil
}
func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) {
if capacity <= 0 {
panic(fmt.Sprintf("capacity must be positive %v", capacity))
@@ -563,30 +526,11 @@ func (v ZFSSendArgVersion) MustBeBookmark() {
}
}
type NilBool struct{ B bool }
func (n *NilBool) Validate() error {
if n == nil {
return fmt.Errorf("must explicitly set `true` or `false`")
}
return nil
}
func (n *NilBool) String() string {
if n == nil {
return "unset"
}
return fmt.Sprintf("%v", n.B)
}
// When updating this struct, check Validate and ValidateCorrespondsToResumeToken (POTENTIALLY SECURITY SENSITIVE)
type ZFSSendArgsUnvalidated struct {
FS string
From, To *ZFSSendArgVersion // From may be nil
Encrypted *NilBool
// Preferred if not empty
ResumeToken string // if not nil, must match what is specified in From, To (covered by ValidateCorrespondsToResumeToken)
FS string
From, To *ZFSSendArgVersion // From may be nil
ZFSSendFlags
}
type ZFSSendArgsValidated struct {
@@ -595,8 +539,22 @@ type ZFSSendArgsValidated struct {
ToVersion FilesystemVersion
}
type ZFSSendFlags struct {
Encrypted *nodefault.Bool
Properties bool
BackupProperties bool
Raw bool
LargeBlocks bool
Compressed bool
EmbeddedData bool
Saved bool
// Preferred if not empty
ResumeToken string // if not nil, must match what is specified in From, To (covered by ValidateCorrespondsToResumeToken)
}
type zfsSendArgsValidationContext struct {
encEnabled *NilBool
encEnabled *nodefault.Bool
}
type ZFSSendArgsValidationErrorCode int
@@ -653,8 +611,8 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali
// fallthrough
}
if err := a.Encrypted.Validate(); err != nil {
return v, newGenericValidationError(a, errors.Wrap(err, "`Raw` invalid"))
if err := a.ZFSSendFlags.Validate(); err != nil {
return v, newGenericValidationError(a, errors.Wrap(err, "send flags invalid"))
}
valCtx := &zfsSendArgsValidationContext{}
@@ -663,17 +621,15 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali
return v, newValidationError(a, ZFSSendArgsFSEncryptionCheckFail,
errors.Wrapf(err, "cannot check whether filesystem %q is encrypted", a.FS))
}
valCtx.encEnabled = &NilBool{fsEncrypted}
valCtx.encEnabled = &nodefault.Bool{B: fsEncrypted}
if a.Encrypted.B && !fsEncrypted {
return v, newValidationError(a, ZFSSendArgsEncryptedSendRequestedButFSUnencrypted,
errors.Errorf("encrypted send requested, but filesystem %q is not encrypted", a.FS))
}
if a.ResumeToken != "" {
if err := a.validateCorrespondsToResumeToken(ctx, valCtx); err != nil {
return v, newValidationError(a, ZFSSendArgsResumeTokenMismatch, err)
}
if err := a.validateEncryptionFlagsCorrespondToResumeToken(ctx, valCtx); err != nil {
return v, newValidationError(a, ZFSSendArgsResumeTokenMismatch, err)
}
return ZFSSendArgsValidated{
@@ -683,6 +639,89 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali
}, nil
}
func (f ZFSSendFlags) Validate() error {
if err := f.Encrypted.ValidateNoDefault(); err != nil {
return errors.Wrap(err, "flag `Encrypted` invalid")
}
return nil
}
// If ResumeToken is empty, builds a command line with the flags specified.
// If ResumeToken is not empty, build a command line with just `-t {{.ResumeToken}}`.
//
// SECURITY SENSITIVE it is the caller's responsibility to ensure that a.Encrypted semantics
// hold for the file system that will be sent with the send flags returned by this function
func (a ZFSSendFlags) buildSendFlagsUnchecked() []string {
args := make([]string, 0)
// ResumeToken takes precedence, we assume that it has been validated
// to reflect what is described by the other fields.
if a.ResumeToken != "" {
args = append(args, "-t", a.ResumeToken)
return args
}
if a.Encrypted.B || a.Raw {
args = append(args, "-w")
}
if a.Properties {
args = append(args, "-p")
}
if a.BackupProperties {
args = append(args, "-b")
}
if a.LargeBlocks {
args = append(args, "-L")
}
if a.Compressed {
args = append(args, "-c")
}
if a.EmbeddedData {
args = append(args, "-e")
}
if a.Saved {
args = append(args, "-S")
}
return args
}
func (a ZFSSendArgsValidated) buildSendCommandLine() ([]string, error) {
flags := a.buildSendFlagsUnchecked()
if a.ZFSSendFlags.ResumeToken != "" {
return flags, nil
}
toV, err := absVersion(a.FS, a.To)
if err != nil {
return nil, err
}
fromV := ""
if a.From != nil {
fromV, err = absVersion(a.FS, a.From)
if err != nil {
return nil, err
}
}
if fromV == "" { // Initial
flags = append(flags, toV)
} else {
flags = append(flags, "-i", fromV, toV)
}
return flags, nil
}
type ZFSSendArgsResumeTokenMismatchError struct {
What ZFSSendArgsResumeTokenMismatchErrorCode
Err error
@@ -707,10 +746,17 @@ func (c ZFSSendArgsResumeTokenMismatchErrorCode) fmt(format string, args ...inte
}
}
// This is SECURITY SENSITIVE and requires exhaustive checking of both side's values
// An attacker requesting a Send with a crafted ResumeToken may encode different parameters in the resume token than expected:
// Validate that the encryption settings specified in `a` correspond to the encryption settings encoded in the resume token.
//
// This is SECURITY SENSITIVE:
// It is possible for an attacker to craft arbitrary resume tokens.
// Those malicious resume tokens could encode different parameters in the resume token than expected:
// for example, they may specify another file system (e.g. the filesystem with secret data) or request unencrypted send instead of encrypted raw send.
func (a ZFSSendArgsUnvalidated) validateCorrespondsToResumeToken(ctx context.Context, valCtx *zfsSendArgsValidationContext) error {
//
// Note that we don't check correspondence of all other send flags because
// a) the resume token does not capture all send flags (e.g. send -p is implemented in libzfs and thus not represented in the resume token)
// b) it would force us to either reject resume tokens with unknown flags.
func (a ZFSSendArgsUnvalidated) validateEncryptionFlagsCorrespondToResumeToken(ctx context.Context, valCtx *zfsSendArgsValidationContext) error {
if a.ResumeToken == "" {
return nil // nothing to do
@@ -735,6 +781,7 @@ func (a ZFSSendArgsUnvalidated) validateCorrespondsToResumeToken(ctx context.Con
"filesystem in resume token field `toname` = %q does not match expected value %q", tokenFS.ToString(), a.FS)
}
// If From is set, it must match.
if (a.From != nil) != t.HasFromGUID { // existence must be same
if t.HasFromGUID {
return gen.fmt("resume token not expected to be incremental, but `fromguid` = %v", t.FromGUID)
@@ -789,18 +836,20 @@ func ZFSSend(ctx context.Context, sendArgs ZFSSendArgsValidated) (*SendStream, e
args = append(args, "send")
// pre-validation of sendArgs for plain ErrEncryptedSendNotSupported error
// TODO go1.13: push this down to sendArgs.Validate
if encryptedSendValid := sendArgs.Encrypted.Validate(); encryptedSendValid == nil && sendArgs.Encrypted.B {
supported, err := EncryptionCLISupported(ctx)
// we tie BackupProperties (send -b) and SendRaw (-w, same as with Encrypted) to this
// since these were released together.
if sendArgs.Encrypted.B || sendArgs.Raw || sendArgs.BackupProperties {
encryptionSupported, err := EncryptionCLISupported(ctx)
if err != nil {
return nil, errors.Wrap(err, "cannot determine CLI native encryption support")
}
if !supported {
if !encryptionSupported {
return nil, ErrEncryptedSendNotSupported
}
}
sargs, err := sendArgs.buildCommonSendArgs()
sargs, err := sendArgs.buildSendCommandLine()
if err != nil {
return nil, err
}
@@ -974,7 +1023,7 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI
args := make([]string, 0)
args = append(args, "send", "-n", "-v", "-P")
sargs, err := sendArgs.buildCommonSendArgs()
sargs, err := sendArgs.buildSendCommandLine()
if err != nil {
return nil, err
}
@@ -992,14 +1041,6 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI
return &si, nil
}
type RecvOptions struct {
// Rollback to the oldest snapshot, destroy it, then perform `recv -F`.
// Note that this doesn't change property values, i.e. an existing local property value will be kept.
RollbackAndForceRecv bool
// Set -s flag used for resumable send & recv
SavePartialRecvState bool
}
type ErrRecvResumeNotSupported struct {
FS string
CheckErr error
@@ -1016,6 +1057,39 @@ func (e *ErrRecvResumeNotSupported) Error() string {
return buf.String()
}
type RecvOptions struct {
// Rollback to the oldest snapshot, destroy it, then perform `recv -F`.
// Note that this doesn't change property values, i.e. an existing local property value will be kept.
RollbackAndForceRecv bool
// Set -s flag used for resumable send & recv
SavePartialRecvState bool
InheritProperties []zfsprop.Property
OverrideProperties map[zfsprop.Property]string
}
func (opts RecvOptions) buildRecvFlags() []string {
args := make([]string, 0)
if opts.RollbackAndForceRecv {
args = append(args, "-F")
}
if opts.SavePartialRecvState {
args = append(args, "-s")
}
if opts.InheritProperties != nil {
for _, prop := range opts.InheritProperties {
args = append(args, "-x", string(prop))
}
}
if opts.OverrideProperties != nil {
for prop, value := range opts.OverrideProperties {
args = append(args, "-o", fmt.Sprintf("%s=%s", prop, value))
}
}
return args
}
const RecvStderrBufSiz = 1 << 15
func ZFSRecv(ctx context.Context, fs string, v *ZFSSendArgVersion, stream io.ReadCloser, opts RecvOptions) (err error) {
@@ -1064,17 +1138,14 @@ func ZFSRecv(ctx context.Context, fs string, v *ZFSSendArgVersion, stream io.Rea
}
}
args := make([]string, 0)
args = append(args, "recv")
if opts.RollbackAndForceRecv {
args = append(args, "-F")
}
if opts.SavePartialRecvState {
if supported, err := ResumeRecvSupported(ctx, fsdp); err != nil || !supported {
return &ErrRecvResumeNotSupported{FS: fs, CheckErr: err}
}
args = append(args, "-s")
}
args := []string{"recv"}
args = append(args, opts.buildRecvFlags()...)
args = append(args, v.FullPath(fs))
ctx, cancelCmd := context.WithCancel(ctx)
@@ -1250,43 +1321,38 @@ func ZFSRecvClearResumeToken(ctx context.Context, fs string) (err error) {
return nil
}
type PropertyValue struct {
Value string
Source PropertySource
}
type ZFSProperties struct {
m map[string]string
m map[string]PropertyValue
}
func NewZFSProperties() *ZFSProperties {
return &ZFSProperties{make(map[string]string, 4)}
}
func (p *ZFSProperties) Set(key, val string) {
p.m[key] = val
return &ZFSProperties{make(map[string]PropertyValue, 4)}
}
func (p *ZFSProperties) Get(key string) string {
return p.m[key].Value
}
func (p *ZFSProperties) GetDetails(key string) PropertyValue {
return p.m[key]
}
func (p *ZFSProperties) appendArgs(args *[]string) (err error) {
for prop, val := range p.m {
func zfsSet(ctx context.Context, path string, props map[string]string) error {
args := make([]string, 0)
args = append(args, "set")
for prop, val := range props {
if strings.Contains(prop, "=") {
return errors.New("prop contains rune '=' which is the delimiter between property name and value")
}
*args = append(*args, fmt.Sprintf("%s=%s", prop, val))
args = append(args, fmt.Sprintf("%s=%s", prop, val))
}
return nil
}
func ZFSSet(ctx context.Context, fs *DatasetPath, props *ZFSProperties) (err error) {
return zfsSet(ctx, fs.ToString(), props)
}
func zfsSet(ctx context.Context, path string, props *ZFSProperties) (err error) {
args := make([]string, 0)
args = append(args, "set")
err = props.appendArgs(&args)
if err != nil {
return err
}
args = append(args, path)
cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...)
@@ -1298,11 +1364,15 @@ func zfsSet(ctx context.Context, path string, props *ZFSProperties) (err error)
}
}
return
return err
}
func ZFSSet(ctx context.Context, fs *DatasetPath, props map[string]string) error {
return zfsSet(ctx, fs.ToString(), props)
}
func ZFSGet(ctx context.Context, fs *DatasetPath, props []string) (*ZFSProperties, error) {
return zfsGet(ctx, fs.ToString(), props, sourceAny)
return zfsGet(ctx, fs.ToString(), props, SourceAny)
}
// The returned error includes requested filesystem and version as quoted strings in its error message
@@ -1322,7 +1392,7 @@ func ZFSGetGUID(ctx context.Context, fs string, version string) (g uint64, err e
return 0, errors.New("version does not start with @ or #")
}
path := fmt.Sprintf("%s%s", fs, version)
props, err := zfsGet(ctx, path, []string{"guid"}, sourceAny) // always local
props, err := zfsGet(ctx, path, []string{"guid"}, SourceAny) // always local
if err != nil {
return 0, err
}
@@ -1338,7 +1408,7 @@ func ZFSGetMountpoint(ctx context.Context, fs string) (*GetMountpointOutput, err
if err := EntityNamecheck(fs, EntityTypeFilesystem); err != nil {
return nil, err
}
props, err := zfsGet(ctx, fs, []string{"mountpoint", "mounted"}, sourceAny)
props, err := zfsGet(ctx, fs, []string{"mountpoint", "mounted"}, SourceAny)
if err != nil {
return nil, err
}
@@ -1355,7 +1425,7 @@ func ZFSGetMountpoint(ctx context.Context, fs string) (*GetMountpointOutput, err
}
func ZFSGetRawAnySource(ctx context.Context, path string, props []string) (*ZFSProperties, error) {
return zfsGet(ctx, path, props, sourceAny)
return zfsGet(ctx, path, props, SourceAny)
}
var zfsGetDatasetDoesNotExistRegexp = regexp.MustCompile(`^cannot open '([^)]+)': (dataset does not exist|no such pool or dataset)`) // verified in platformtest
@@ -1375,46 +1445,68 @@ func tryDatasetDoesNotExist(expectPath string, stderr []byte) *DatasetDoesNotExi
return nil
}
type zfsPropertySource uint
//go:generate enumer -type=PropertySource -trimprefix=Source
type PropertySource uint32
const (
sourceLocal zfsPropertySource = 1 << iota
sourceDefault
sourceInherited
sourceNone
sourceTemporary
sourceReceived
SourceLocal PropertySource = 1 << iota
SourceDefault
SourceInherited
SourceNone
SourceTemporary
SourceReceived
sourceAny zfsPropertySource = ^zfsPropertySource(0)
SourceAny PropertySource = ^PropertySource(0)
)
func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string {
var propertySourceParseLUT = map[string]PropertySource{
"local": SourceLocal,
"default": SourceDefault,
"inherited": SourceInherited,
"-": SourceNone,
"temporary": SourceTemporary,
"received": SourceReceived,
}
func parsePropertySource(s string) (PropertySource, error) {
fields := strings.Fields(s)
if len(fields) > 0 {
v, ok := propertySourceParseLUT[fields[0]]
if ok {
return v, nil
}
// fallthrough
}
return 0, fmt.Errorf("unknown property source %q", s)
}
func (s PropertySource) zfsGetSourceFieldPrefixes() []string {
prefixes := make([]string, 0, 7)
if s&sourceLocal != 0 {
if s&SourceLocal != 0 {
prefixes = append(prefixes, "local")
}
if s&sourceDefault != 0 {
if s&SourceDefault != 0 {
prefixes = append(prefixes, "default")
}
if s&sourceInherited != 0 {
if s&SourceInherited != 0 {
prefixes = append(prefixes, "inherited")
}
if s&sourceNone != 0 {
if s&SourceNone != 0 {
prefixes = append(prefixes, "-")
}
if s&sourceTemporary != 0 {
if s&SourceTemporary != 0 {
prefixes = append(prefixes, "temporary")
}
if s&sourceReceived != 0 {
if s&SourceReceived != 0 {
prefixes = append(prefixes, "received")
}
if s == sourceAny {
if s == SourceAny {
prefixes = append(prefixes, "")
}
return prefixes
}
func zfsGet(ctx context.Context, path string, props []string, allowedSources zfsPropertySource) (*ZFSProperties, error) {
func zfsGet(ctx context.Context, path string, props []string, allowedSources PropertySource) (*ZFSProperties, error) {
args := []string{"get", "-Hp", "-o", "property,value,source", strings.Join(props, ","), path}
cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...)
stdout, err := cmd.Output()
@@ -1440,7 +1532,7 @@ func zfsGet(ctx context.Context, path string, props []string, allowedSources zfs
return nil, fmt.Errorf("zfs get did not return the number of expected property values")
}
res := &ZFSProperties{
make(map[string]string, len(lines)),
make(map[string]PropertyValue, len(lines)),
}
allowedPrefixes := allowedSources.zfsGetSourceFieldPrefixes()
for _, line := range lines[:len(lines)-1] {
@@ -1451,8 +1543,16 @@ func zfsGet(ctx context.Context, path string, props []string, allowedSources zfs
return nil, fmt.Errorf("zfs get did not return property,value,source tuples")
}
for _, p := range allowedPrefixes {
// prefix-match so that SourceAny (= "") works
if strings.HasPrefix(fields[2], p) {
res.m[fields[0]] = fields[1]
source, err := parsePropertySource(fields[2])
if err != nil {
return nil, errors.Wrap(err, "parse property source")
}
res.m[fields[0]] = PropertyValue{
Value: fields[1],
Source: source,
}
break
}
}
zfs/zfs_test.go
+150 -4
View File
@@ -5,6 +5,9 @@ import (
"strings"
"testing"
"github.com/zrepl/zrepl/util/nodefault"
zfsprop "github.com/zrepl/zrepl/zfs/property"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -41,20 +44,20 @@ func TestDatasetPathTrimNPrefixComps(t *testing.T) {
func TestZFSPropertySource(t *testing.T) {
tcs := []struct {
in zfsPropertySource
in PropertySource
exp []string
}{
{
in: sourceAny,
in: SourceAny,
// although empty prefix matches any source
exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""},
},
{
in: sourceTemporary,
in: SourceTemporary,
exp: []string{"temporary"},
},
{
in: sourceLocal | sourceInherited,
in: SourceLocal | SourceInherited,
exp: []string{"local", "inherited"},
},
}
@@ -69,6 +72,21 @@ func TestZFSPropertySource(t *testing.T) {
for _, tc := range tcs {
t.Logf("TEST CASE %v", tc)
// give the parsing code some coverage
for _, e := range tc.exp {
if e == "" {
continue // "" is the prefix that matches SourceAny
}
s, err := parsePropertySource(e)
assert.NoError(t, err)
t.Logf("s: %x %s", s, s)
t.Logf("in: %x %s", tc.in, tc.in)
assert.True(t, s&tc.in != 0)
}
// prefix matching
res := tc.in.zfsGetSourceFieldPrefixes()
resSet := toSet(res)
expSet := toSet(tc.exp)
@@ -304,3 +322,131 @@ func TestTryRecvDestroyOrOverwriteEncryptedErr(t *testing.T) {
require.NotNil(t, err)
assert.EqualError(t, err, strings.TrimSpace(msg))
}
func TestZFSSendArgsBuildSendFlags(t *testing.T) {
type args = ZFSSendFlags
type SendTest struct {
conf args
exactMatch bool
flagsInclude []string
flagsExclude []string
}
withEncrypted := func(justFlags ZFSSendFlags) ZFSSendFlags {
if justFlags.Encrypted == nil {
justFlags.Encrypted = &nodefault.Bool{B: false}
}
return justFlags
}
var sendTests = map[string]SendTest{
"Empty Args": {
conf: withEncrypted(args{}),
flagsInclude: []string{},
flagsExclude: []string{"-w", "-p", "-b"},
},
"Raw": {
conf: withEncrypted(args{Raw: true}),
flagsInclude: []string{"-w"},
flagsExclude: []string{},
},
"Encrypted": {
conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: true}}),
flagsInclude: []string{"-w"},
flagsExclude: []string{},
},
"Unencrypted_And_Raw": {
conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: false}, Raw: true}),
flagsInclude: []string{"-w"},
flagsExclude: []string{},
},
"Encrypted_And_Raw": {
conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: true}, Raw: true}),
flagsInclude: []string{"-w"},
flagsExclude: []string{},
},
"Send properties": {
conf: withEncrypted(args{Properties: true}),
flagsInclude: []string{"-p"},
flagsExclude: []string{},
},
"Send backup properties": {
conf: withEncrypted(args{BackupProperties: true}),
flagsInclude: []string{"-b"},
flagsExclude: []string{},
},
"Send -b and -p": {
conf: withEncrypted(args{Properties: true, BackupProperties: true}),
flagsInclude: []string{"-p", "-b"},
flagsExclude: []string{},
},
"Send resume state": {
conf: withEncrypted(args{Saved: true}),
flagsInclude: []string{"-S"},
},
"Resume token wins if not empty": {
conf: withEncrypted(args{ResumeToken: "$theresumetoken$", Compressed: true}),
flagsInclude: []string{"-t", "$theresumetoken$"},
exactMatch: true,
},
}
for testName, test := range sendTests {
t.Run(testName, func(t *testing.T) {
flags := test.conf.buildSendFlagsUnchecked()
assert.GreaterOrEqual(t, len(flags), len(test.flagsInclude))
assert.Subset(t, flags, test.flagsInclude)
if test.exactMatch {
assert.Equal(t, flags, test.flagsInclude)
}
for flag := range flags {
assert.NotContains(t, test.flagsExclude, flag)
}
})
}
}
func TestZFSCommonRecvArgsBuild(t *testing.T) {
type RecvTest struct {
conf RecvOptions
flagsInclude []string
flagsExclude []string
}
var recvTests = map[string]RecvTest{
"Empty Args": {
conf: RecvOptions{},
flagsInclude: []string{},
flagsExclude: []string{"-x", "-o", "-F", "-s"},
},
"ForceRollback": {
conf: RecvOptions{RollbackAndForceRecv: true},
flagsInclude: []string{"-F"},
flagsExclude: []string{"-x", "-o", "-s"},
},
"PartialSend": {
conf: RecvOptions{SavePartialRecvState: true},
flagsInclude: []string{"-s"},
flagsExclude: []string{"-x", "-o", "-F"},
},
"Override properties": {
conf: RecvOptions{OverrideProperties: map[zfsprop.Property]string{zfsprop.Property("abc"): "123"}},
flagsInclude: []string{"-o", "abc=123"},
flagsExclude: []string{"-x", "-F", "-s"},
},
"Exclude/inherit properties": {
conf: RecvOptions{InheritProperties: []zfsprop.Property{"abc", "123"}},
flagsInclude: []string{"-x", "abc", "123"}, flagsExclude: []string{"-o", "-F", "-s"},
},
}
for testName, test := range recvTests {
t.Run(testName, func(t *testing.T) {
flags := test.conf.buildRecvFlags()
assert.Subset(t, flags, test.flagsInclude)
for flag := range flags {
assert.NotContains(t, test.flagsExclude, flag)
}
})
}
}