adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

streamrpc 0.3 + config from daemon/config

Browse Source
This commit is contained in:
Christian Schwarz
2018-08-31 21:51:44 +02:00
parent d55a271ac7
commit 3d8e552c6a
9 changed files with 144 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/connecter/connecter.go
+31 -4
View File
@@ -4,17 +4,44 @@ import (
"fmt"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/streamrpcconfig"
)
func FromConfig(g *config.Global, in config.ConnectEnum) (streamrpc.Connecter, error) {
func FromConfig(g *config.Global, in config.ConnectEnum) (*ClientFactory, error) {
var (
connecter streamrpc.Connecter
errConnecter, errRPC error
connConf *streamrpc.ConnConfig
)
switch v := in.Ret.(type) {
case *config.SSHStdinserverConnect:
return SSHStdinserverConnecterFromConfig(v)
connecter, errConnecter = SSHStdinserverConnecterFromConfig(v)
connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC)
case *config.TCPConnect:
return TCPConnecterFromConfig(v)
connecter, errConnecter = TCPConnecterFromConfig(v)
connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC)
case *config.TLSConnect:
return TLSConnecterFromConfig(v)
connecter, errConnecter = TLSConnecterFromConfig(v)
connConf, errRPC = streamrpcconfig.FromDaemonConfig(g, v.RPC)
default:
panic(fmt.Sprintf("implementation error: unknown connecter type %T", v))
}
if errConnecter != nil {
return nil, errConnecter
}
if errRPC != nil {
return nil, errRPC
}
return &ClientFactory{connecter: connecter, config: &streamrpc.ClientConfig{connConf}}, nil
}
type ClientFactory struct {
connecter streamrpc.Connecter
config *streamrpc.ClientConfig
}
func (f ClientFactory) NewClient() (*streamrpc.Client, error) {
return streamrpc.NewClient(f.connecter, f.config)
}
daemon/job/job.go
-11
View File
@@ -3,9 +3,7 @@ package job
import (
"context"
"errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/logger"
"time"
)
type Logger = logger.Logger
@@ -59,12 +57,3 @@ func WaitWakeup(ctx context.Context) <-chan struct{} {
return wc
}
var STREAMRPC_CONFIG = &streamrpc.ConnConfig{ // FIXME oversight and configurability
RxHeaderMaxLen: 4096,
RxStructuredMaxLen: 4096 * 4096,
RxStreamMaxChunkSize: 4096 * 4096,
TxChunkSize: 4096 * 4096,
Timeout: streamrpc.Timeout{
Progress: 10 * time.Second,
},
}
daemon/job/push.go
+7 -5
View File
@@ -3,7 +3,6 @@ package job
import (
"context"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/connecter"
"github.com/zrepl/zrepl/daemon/filters"
@@ -16,7 +15,7 @@ import (
type Push struct {
name string
connecter streamrpc.Connecter
clientFactory *connecter.ClientFactory
fsfilter endpoint.FSFilter
prunerFactory *pruner.PrunerFactory
@@ -30,7 +29,10 @@ func PushFromConfig(g *config.Global, in *config.PushJob) (j *Push, err error) {
j = &Push{}
j.name = in.Name
j.connecter, err = connecter.FromConfig(g, in.Replication.Connect)
j.clientFactory, err = connecter.FromConfig(g, in.Replication.Connect)
if err != nil {
return nil, errors.Wrap(err, "cannot build client")
}
if j.fsfilter, err = filters.DatasetMapFilterFromConfig(in.Replication.Filesystems); err != nil {
return nil, errors.Wrap(err, "cannnot build filesystem filter")
@@ -87,9 +89,9 @@ func (j *Push) do(ctx context.Context) {
log := GetLogger(ctx)
ctx = logging.WithSubsystemLoggers(ctx, log)
client, err := streamrpc.NewClient(j.connecter, &streamrpc.ClientConfig{STREAMRPC_CONFIG})
client, err := j.clientFactory.NewClient()
if err != nil {
log.WithError(err).Error("cannot create streamrpc client")
log.WithError(err).Error("factory cannot instantiate streamrpc client")
}
defer client.Close(ctx)
daemon/job/sink.go
+3 -2
View File
@@ -15,6 +15,7 @@ import (
type Sink struct {
name string
l serve.ListenerFactory
rpcConf *streamrpc.ConnConfig
fsmap endpoint.FSMap
fsmapInv endpoint.FSFilter
}
@@ -24,7 +25,7 @@ func SinkFromConfig(g *config.Global, in *config.SinkJob) (s *Sink, err error) {
// FIXME multi client support
s = &Sink{name: in.Name}
if s.l, err = serve.FromConfig(g, in.Replication.Serve); err != nil {
if s.l, s.rpcConf, err = serve.FromConfig(g, in.Replication.Serve); err != nil {
return nil, errors.Wrap(err, "cannot build server")
}
@@ -95,7 +96,7 @@ func (j *Sink) handleConnection(ctx context.Context, conn net.Conn) {
}
handler := endpoint.NewHandler(local)
if err := streamrpc.ServeConn(ctx, conn, STREAMRPC_CONFIG, handler.Handle); err != nil {
if err := streamrpc.ServeConn(ctx, conn, j.rpcConf, handler.Handle); err != nil {
log.WithError(err).Error("error serving client")
}
}
daemon/serve/serve.go
+24 -5
View File
@@ -4,23 +4,42 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"net"
"github.com/zrepl/zrepl/daemon/streamrpcconfig"
"github.com/problame/go-streamrpc"
)
type ListenerFactory interface {
Listen() (net.Listener, error)
}
func FromConfig(g *config.Global, in config.ServeEnum) (ListenerFactory, error) {
func FromConfig(g *config.Global, in config.ServeEnum) (lf ListenerFactory, conf *streamrpc.ConnConfig, _ error) {
var (
lfError, rpcErr error
)
switch v := in.Ret.(type) {
case *config.TCPServe:
return TCPListenerFactoryFromConfig(g, v)
lf, lfError = TCPListenerFactoryFromConfig(g, v)
conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC)
case *config.TLSServe:
return TLSListenerFactoryFromConfig(g, v)
lf, lfError = TLSListenerFactoryFromConfig(g, v)
conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC)
case *config.StdinserverServer:
return StdinserverListenerFactoryFromConfig(g, v)
lf, lfError = StdinserverListenerFactoryFromConfig(g, v)
conf, rpcErr = streamrpcconfig.FromDaemonConfig(g, v.RPC)
default:
return nil, errors.Errorf("internal error: unknown serve type %T", v)
return nil, nil, errors.Errorf("internal error: unknown serve type %T", v)
}
if lfError != nil {
return nil, nil, lfError
}
if rpcErr != nil {
return nil, nil, rpcErr
}
return lf, conf, nil
}
daemon/streamrpcconfig/streamrpcconfig.go
+22
View File
@@ -0,0 +1,22 @@
package streamrpcconfig
import (
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/config"
)
func FromDaemonConfig(g *config.Global, in *config.RPCConfig) (*streamrpc.ConnConfig, error) {
conf := in
if conf == nil {
conf = g.RPC
}
return &streamrpc.ConnConfig{
RxHeaderMaxLen: conf.RxHeaderMaxLen,
RxStructuredMaxLen: conf.RxStructuredMaxLen,
RxStreamMaxChunkSize: conf.RxStreamChunkMaxLen,
TxChunkSize: conf.TxChunkSize,
Timeout: streamrpc.Timeout{
Progress: conf.Timeout,
},
}, nil
}