adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

format source tree using goimports

Browse Source
This commit is contained in:
Christian Schwarz
2019-03-22 19:41:12 +01:00
parent 5324f29693
commit afed762774
93 changed files with 585 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/control.go
+14 -13
View File
@@ -12,6 +12,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/nethelpers"
"github.com/zrepl/zrepl/logger"
@@ -43,24 +44,24 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
var promControl struct {
requestBegin *prometheus.CounterVec
requestBegin *prometheus.CounterVec
requestFinished *prometheus.HistogramVec
}
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "zrepl",
Subsystem: "control",
Name: "request_begin",
Help: "number of request we started to handle",
Namespace: "zrepl",
Subsystem: "control",
Name: "request_begin",
Help: "number of request we started to handle",
}, []string{"endpoint"})
promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "zrepl",
Subsystem: "control",
Name: "request_finished",
Help: "time it took a request to finih",
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20},
Namespace: "zrepl",
Subsystem: "control",
Name: "request_finished",
Help: "time it took a request to finih",
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
}, []string{"endpoint"})
registerer.MustRegister(promControl.requestBegin)
registerer.MustRegister(promControl.requestFinished)
@@ -114,7 +115,7 @@ func (j *controlJob) Run(ctx context.Context) {
requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) {
type reqT struct {
Name string
Op string
Op string
}
var req reqT
if decoder(&req) != nil {
@@ -136,8 +137,8 @@ func (j *controlJob) Run(ctx context.Context) {
server := http.Server{
Handler: mux,
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
WriteTimeout: 1*time.Second,
ReadTimeout: 1*time.Second,
WriteTimeout: 1 * time.Second,
ReadTimeout: 1 * time.Second,
}
outer:
daemon/daemon.go
+10 -9
View File
@@ -3,8 +3,16 @@ package daemon
import (
"context"
"fmt"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/job/reset"
@@ -12,12 +20,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/version"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
)
func Run(conf *config.Config) error {
@@ -74,12 +76,11 @@ func Run(conf *config.Config) error {
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
}
if err != nil {
return errors.Wrapf(err,"cannot build monitorin gjob #%d", i)
return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
}
jobs.start(ctx, job, true)
}
log.Info("starting daemon")
// start regular jobs
@@ -103,7 +104,7 @@ type jobs struct {
// m protects all fields below it
m sync.RWMutex
wakeups map[string]wakeup.Func // by Job.Name
resets map[string]reset.Func // by Job.Name
resets map[string]reset.Func // by Job.Name
jobs map[string]job.Job
}
daemon/filters/fsmapfilter.go
+3 -1
View File
@@ -2,10 +2,12 @@ package filters
import (
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/zfs"
"strings"
)
type DatasetMapFilter struct {
daemon/filters/fsvfilter.go
+2 -2
View File
@@ -1,8 +1,9 @@
package filters
import (
"github.com/zrepl/zrepl/zfs"
"strings"
"github.com/zrepl/zrepl/zfs"
)
type AnyFSVFilter struct{}
@@ -17,7 +18,6 @@ func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err err
return true, nil
}
type PrefixFilter struct {
prefix string
fstype zfs.VersionType
daemon/job/build_jobs.go
+1
View File
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
)
daemon/job/job.go
+13 -11
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs"
)
@@ -29,7 +30,6 @@ func WithLogger(ctx context.Context, l Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, l)
}
type Job interface {
Name() string
Run(ctx context.Context)
@@ -44,15 +44,15 @@ type Type string
const (
TypeInternal Type = "internal"
TypeSnap Type = "snap"
TypePush Type = "push"
TypeSink Type = "sink"
TypePull Type = "pull"
TypeSource Type = "source"
TypeSnap Type = "snap"
TypePush Type = "push"
TypeSink Type = "sink"
TypePull Type = "pull"
TypeSource Type = "source"
)
type Status struct {
Type Type
Type Type
JobSpecific interface{}
}
@@ -65,8 +65,8 @@ func (s *Status) MarshalJSON() ([]byte, error) {
if err != nil {
return nil, err
}
m := map[string]json.RawMessage {
"type": typeJson,
m := map[string]json.RawMessage{
"type": typeJson,
string(s.Type): jobJSON,
}
return json.Marshal(m)
@@ -94,12 +94,14 @@ func (s *Status) UnmarshalJSON(in []byte) (err error) {
var st SnapJobStatus
err = json.Unmarshal(jobJSON, &st)
s.JobSpecific = &st
case TypePull: fallthrough
case TypePull:
fallthrough
case TypePush:
var st ActiveSideStatus
err = json.Unmarshal(jobJSON, &st)
s.JobSpecific = &st
case TypeSource: fallthrough
case TypeSource:
fallthrough
case TypeSink:
var st PassiveStatus
err = json.Unmarshal(jobJSON, &st)
daemon/job/snapjob.go
+1
View File
@@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/wakeup"
daemon/logging/logging_formatters.go
+3 -1
View File
@@ -4,11 +4,13 @@ import (
"bytes"
"encoding/json"
"fmt"
"time"
"github.com/fatih/color"
"github.com/go-logfmt/logfmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
"time"
)
const (
daemon/logging/logging_outlets.go
+4 -2
View File
@@ -4,12 +4,14 @@ import (
"bytes"
"context"
"crypto/tls"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
"io"
"log/syslog"
"net"
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
)
type EntryFormatter interface {
daemon/main.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
type Logger = logger.Logger
var DaemonCmd = &cli.Subcommand {
var DaemonCmd = &cli.Subcommand{
Use: "daemon",
Short: "run the zrepl daemon",
Run: func(subcommand *cli.Subcommand, args []string) error {
daemon/nethelpers/helpers.go
+2 -1
View File
@@ -1,10 +1,11 @@
package nethelpers
import (
"github.com/pkg/errors"
"net"
"os"
"path/filepath"
"github.com/pkg/errors"
)
func PreparePrivateSockpath(sockpath string) error {
daemon/prometheus.go
+5 -4
View File
@@ -2,15 +2,17 @@ package daemon
import (
"context"
"net"
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
"github.com/zrepl/zrepl/zfs"
"net"
"net/http"
)
type prometheusJob struct {
@@ -25,7 +27,7 @@ func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob
}
var prom struct {
taskLogEntries *prometheus.CounterVec
taskLogEntries *prometheus.CounterVec
}
func init() {
@@ -93,4 +95,3 @@ func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
return nil
}
daemon/pruner/pruner.go
+38 -37
View File
@@ -3,17 +3,19 @@ package pruner
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/pruning"
"github.com/zrepl/zrepl/replication/logic/pdu"
"github.com/zrepl/zrepl/util/envconst"
"sort"
"strings"
"sync"
"time"
)
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
@@ -53,7 +55,7 @@ type args struct {
rules []pruning.KeepRule
retryWait time.Duration
considerSnapAtCursorReplicated bool
promPruneSecs prometheus.Observer
promPruneSecs prometheus.Observer
}
type Pruner struct {
@@ -64,7 +66,7 @@ type Pruner struct {
state State
// State PlanErr
err error
err error
// State Exec
execQueue *execQueue
@@ -75,7 +77,7 @@ type PrunerFactory struct {
receiverRules []pruning.KeepRule
retryWait time.Duration
considerSnapAtCursorReplicated bool
promPruneSecs *prometheus.HistogramVec
promPruneSecs *prometheus.HistogramVec
}
type LocalPrunerFactory struct {
@@ -137,11 +139,11 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
}
f := &PrunerFactory{
senderRules: keepRulesSender,
receiverRules: keepRulesReceiver,
retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10 * time.Second),
senderRules: keepRulesSender,
receiverRules: keepRulesReceiver,
retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
promPruneSecs: promPruneSecs,
promPruneSecs: promPruneSecs,
}
return f, nil
}
@@ -213,17 +215,17 @@ func (p *Pruner) Prune() {
func (p *Pruner) prune(args args) {
u := func(f func(*Pruner)) {
p.mtx.Lock()
defer p.mtx.Unlock()
f(p)
}
p.mtx.Lock()
defer p.mtx.Unlock()
f(p)
}
// TODO support automatic retries
// It is advisable to merge this code with package replication/driver before
// That will likely require re-modelling struct fs like replication/driver.attempt,
// including figuring out how to resume a plan after being interrupted by network errors
// The non-retrying code in this package should move straight to replication/logic.
doOneAttempt(&args, u)
}
}
type Report struct {
State string
@@ -239,9 +241,9 @@ type FSReport struct {
}
type SnapshotReport struct {
Name string
Name string
Replicated bool
Date time.Time
Date time.Time
}
func (p *Pruner) Report() *Report {
@@ -250,9 +252,9 @@ func (p *Pruner) Report() *Report {
r := Report{State: p.state.String()}
if p.err != nil {
r.Error = p.err.Error()
}
if p.err != nil {
r.Error = p.err.Error()
}
if p.execQueue != nil {
r.Pending, r.Completed = p.execQueue.Report()
@@ -268,7 +270,7 @@ func (p *Pruner) State() State {
}
type fs struct {
path string
path string
// permanent error during planning
planErr error
@@ -316,7 +318,7 @@ func (f *fs) Report() FSReport {
if f.planErr != nil {
r.LastError = f.planErr.Error()
} else if f.execErrLast != nil {
} else if f.execErrLast != nil {
r.LastError = f.execErrLast.Error()
}
@@ -326,7 +328,7 @@ func (f *fs) Report() FSReport {
}
r.DestroyList = make([]SnapshotReport, len(f.destroyList))
for i, snap := range f.destroyList{
for i, snap := range f.destroyList {
r.DestroyList[i] = snap.(snapshot).Report()
}
@@ -490,9 +492,9 @@ tfss_loop:
})
for {
var pfs *fs
var pfs *fs
u(func(pruner *Pruner) {
pfs = pruner.execQueue.Pop()
pfs = pruner.execQueue.Pop()
})
if pfs == nil {
break
@@ -516,16 +518,15 @@ tfss_loop:
hadErr := false
for _, fsr := range rep.Completed {
hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
}
}
if hadErr {
p.state = ExecErr
} else {
p.state = Done
}
})
}
}
// attempts to exec pfs, puts it back into the queue with the result
func doOneAttemptExec(a *args, u updater, pfs *fs) {
@@ -558,20 +559,20 @@ func doOneAttemptExec(a *args, u updater, pfs *fs) {
err = nil
destroyFails := make([]*pdu.DestroySnapshotRes, 0)
for _, reqDestroy := range destroyList {
res, ok := destroyResults[reqDestroy.Name]
if !ok {
err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
break
} else if res.Error != "" {
destroyFails = append(destroyFails, res)
}
res, ok := destroyResults[reqDestroy.Name]
if !ok {
err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
break
} else if res.Error != "" {
destroyFails = append(destroyFails, res)
}
}
if err == nil && len(destroyFails) > 0 {
names := make([]string, len(destroyFails))
pairs := make([]string, len(destroyFails))
allSame := true
lastMsg := destroyFails[0].Error
for i := 0; i < len(destroyFails); i++{
for i := 0; i < len(destroyFails); i++ {
allSame = allSame && destroyFails[i].Error == lastMsg
relname := destroyFails[i].Snapshot.RelName()
names[i] = relname
daemon/pruner/pruner_queue.go
+4 -5
View File
@@ -7,13 +7,13 @@ import (
)
type execQueue struct {
mtx sync.Mutex
mtx sync.Mutex
pending, completed []*fs
}
func newExecQueue(cap int) *execQueue {
q := execQueue{
pending: make([]*fs, 0, cap),
pending: make([]*fs, 0, cap),
completed: make([]*fs, 0, cap),
}
return &q
@@ -55,7 +55,7 @@ func (q *execQueue) Pop() *fs {
return fs
}
func(q *execQueue) Put(fs *fs, err error, done bool) {
func (q *execQueue) Put(fs *fs, err error, done bool) {
fs.mtx.Lock()
fs.execErrLast = err
if done || err != nil {
@@ -79,5 +79,4 @@ func(q *execQueue) Put(fs *fs, err error, done bool) {
})
q.mtx.Unlock()
}
}
daemon/snapper/snapper.go
+21 -21
View File
@@ -1,18 +1,19 @@
package snapper
import (
"github.com/zrepl/zrepl/config"
"github.com/pkg/errors"
"time"
"context"
"github.com/zrepl/zrepl/daemon/filters"
"fmt"
"github.com/zrepl/zrepl/zfs"
"sort"
"github.com/zrepl/zrepl/logger"
"sync"
)
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs"
)
//go:generate stringer -type=SnapState
type SnapState uint
@@ -28,7 +29,7 @@ type snapProgress struct {
state SnapState
// SnapStarted, SnapDone, SnapError
name string
name string
startAt time.Time
// SnapDone
@@ -44,13 +45,13 @@ type args struct {
prefix string
interval time.Duration
fsf *filters.DatasetMapFilter
snapshotsTaken chan<-struct{}
snapshotsTaken chan<- struct{}
}
type Snapper struct {
args args
mtx sync.Mutex
mtx sync.Mutex
state State
// set in state Plan, used in Waiting
@@ -70,7 +71,7 @@ type Snapper struct {
type State uint
const (
SyncUp State = 1<<iota
SyncUp State = 1 << iota
SyncUpErrWait
Planning
Snapshotting
@@ -81,13 +82,13 @@ const (
func (s State) sf() state {
m := map[State]state{
SyncUp: syncUp,
SyncUp: syncUp,
SyncUpErrWait: wait,
Planning: plan,
Snapshotting: snapshot,
Waiting: wait,
ErrorWait: wait,
Stopped: nil,
Planning: plan,
Snapshotting: snapshot,
Waiting: wait,
ErrorWait: wait,
Stopped: nil,
}
return m[s]
}
@@ -123,9 +124,9 @@ func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *con
}
args := args{
prefix: in.Prefix,
prefix: in.Prefix,
interval: in.Interval,
fsf: fsf,
fsf: fsf,
// ctx and log is set in Run()
}
@@ -199,7 +200,7 @@ func syncUp(a args, u updater) state {
if err != nil {
return onErr(err, u)
}
u(func(s *Snapper){
u(func(s *Snapper) {
s.sleepUntil = syncPoint
})
t := time.NewTimer(syncPoint.Sub(time.Now()))
@@ -386,4 +387,3 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
return snaptimes[0].time, nil
}
daemon/snapper/snapper_all.go
+2 -1
View File
@@ -3,6 +3,7 @@ package snapper
import (
"context"
"fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
)
@@ -17,7 +18,7 @@ type PeriodicOrManual struct {
s *Snapper
}
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan <- struct{}) {
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
if s.s != nil {
s.s.Run(ctx, wakeUpCommon)
}