adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rpc rewrite: control RPCs using gRPC + separate RPC for data transfer

Browse Source 
transport/ssh: update go-netssh to new version
    => supports CloseWrite and Deadlines
    => build: require Go 1.11 (netssh requires it)
This commit is contained in:
Christian Schwarz
2018-12-11 22:01:50 +01:00
parent d281fb00e3
commit 796c5ad42d
100 changed files with 6460 additions and 1485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/job/passive.go
+35 -93
View File
@@ -2,28 +2,30 @@ package job
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/daemon/transport/serve"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/transport/fromconfig"
"github.com/zrepl/zrepl/zfs"
"path"
)
type PassiveSide struct {
mode passiveMode
name string
l serve.ListenerFactory
rpcConf *streamrpc.ConnConfig
mode passiveMode
name string
listen transport.AuthenticatedListenerFactory
}
type passiveMode interface {
ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc
Handler() rpc.Handler
RunPeriodic(ctx context.Context)
Type() Type
}
@@ -34,26 +36,8 @@ type modeSink struct {
func (m *modeSink) Type() Type { return TypeSink }
func (m *modeSink) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc {
log := GetLogger(ctx)
clientRootStr := path.Join(m.rootDataset.ToString(), conn.ClientIdentity())
clientRoot, err := zfs.NewDatasetPath(clientRootStr)
if err != nil {
log.WithError(err).
WithField("client_identity", conn.ClientIdentity()).
Error("cannot build client filesystem map (client identity must be a valid ZFS FS name")
}
log.WithField("client_root", clientRoot).Debug("client root")
local, err := endpoint.NewReceiver(clientRoot)
if err != nil {
log.WithError(err).Error("unexpected error: cannot convert mapping to filter")
return nil
}
h := endpoint.NewHandler(local)
return h.Handle
func (m *modeSink) Handler() rpc.Handler {
return endpoint.NewReceiver(m.rootDataset, true)
}
func (m *modeSink) RunPeriodic(_ context.Context) {}
@@ -72,7 +56,7 @@ func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err
type modeSource struct {
fsfilter zfs.DatasetFilter
snapper *snapper.PeriodicOrManual
snapper *snapper.PeriodicOrManual
}
func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
@@ -93,10 +77,8 @@ func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource
func (m *modeSource) Type() Type { return TypeSource }
func (m *modeSource) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc {
sender := endpoint.NewSender(m.fsfilter)
h := endpoint.NewHandler(sender)
return h.Handle
func (m *modeSource) Handler() rpc.Handler {
return endpoint.NewSender(m.fsfilter)
}
func (m *modeSource) RunPeriodic(ctx context.Context) {
@@ -106,8 +88,8 @@ func (m *modeSource) RunPeriodic(ctx context.Context) {
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
s = &PassiveSide{mode: mode, name: in.Name}
if s.l, s.rpcConf, err = serve.FromConfig(g, in.Serve); err != nil {
return nil, errors.Wrap(err, "cannot build server")
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
return nil, errors.Wrap(err, "cannot build listener factory")
}
return s, nil
@@ -115,7 +97,7 @@ func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passive
func (j *PassiveSide) Name() string { return j.name }
type PassiveStatus struct {}
type PassiveStatus struct{}
func (s *PassiveSide) Status() *Status {
return &Status{Type: s.mode.Type()} // FIXME PassiveStatus
@@ -127,70 +109,30 @@ func (j *PassiveSide) Run(ctx context.Context) {
log := GetLogger(ctx)
defer log.Info("job exiting")
l, err := j.l.Listen()
if err != nil {
log.WithError(err).Error("cannot listen")
return
}
defer l.Close()
ctx = logging.WithSubsystemLoggers(ctx, log)
{
ctx, cancel := context.WithCancel(logging.WithSubsystemLoggers(ctx, log)) // shadowing
ctx, cancel := context.WithCancel(ctx) // shadowing
defer cancel()
go j.mode.RunPeriodic(ctx)
}
log.WithField("addr", l.Addr()).Debug("accepting connections")
var connId int
outer:
for {
select {
case res := <-accept(ctx, l):
if res.err != nil {
log.WithError(res.err).Info("accept error")
continue
}
conn := res.conn
connId++
connLog := log.
WithField("connID", connId)
connLog.
WithField("addr", conn.RemoteAddr()).
WithField("client_identity", conn.ClientIdentity()).
Info("handling connection")
go func() {
defer connLog.Info("finished handling connection")
defer conn.Close()
ctx := logging.WithSubsystemLoggers(ctx, connLog)
handleFunc := j.mode.ConnHandleFunc(ctx, conn)
if handleFunc == nil {
return
}
if err := streamrpc.ServeConn(ctx, conn, j.rpcConf, handleFunc); err != nil {
log.WithError(err).Error("error serving client")
}
}()
case <-ctx.Done():
break outer
}
handler := j.mode.Handler()
if handler == nil {
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
}
}
ctxInterceptor := func(handlerCtx context.Context) context.Context {
return logging.WithSubsystemLoggers(handlerCtx, log)
}
type acceptResult struct {
conn serve.AuthenticatedConn
err error
}
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
func accept(ctx context.Context, listener serve.AuthenticatedListener) <-chan acceptResult {
c := make(chan acceptResult, 1)
go func() {
conn, err := listener.Accept(ctx)
c <- acceptResult{conn, err}
}()
return c
listener, err := j.listen()
if err != nil {
log.WithError(err).Error("cannot listen")
return
}
server.Serve(ctx, listener)
}