adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reimplement io.ReadWriteCloser based RPC mechanism

Browse Source 
The existing ByteStreamRPC requires writing RPC stub + server code
for each RPC endpoint. Does not scale well.

Goal: adding a new RPC call should

- not require writing an RPC stub / handler
- not require modifications to the RPC lib

The wire format is inspired by HTTP2, the API by net/rpc.

Frames are used for framing messages, i.e. a message is made of multiple
frames which are glued together using a frame-bridging reader / writer.
This roughly corresponds to HTTP2 streams, although we're happy with
just one stream at any time and the resulting non-need for flow control,
etc.

Frames are typed using a header. The two most important types are
'Header' and 'Data'.

The RPC protocol is built on top of this:

- Client sends a header         => multiple frames of type 'header'
- Client sends request body     => mulitiple frames of type 'data'
- Server reads a header         => multiple frames of type 'header'
- Server reads request body     => mulitiple frames of type 'data'
- Server sends response header  => ...
- Server sends response body    => ...

An RPC header is serialized JSON and always the same structure.
The body is of the type specified in the header.

The RPC server and client use some semi-fancy reflection tequniques to
automatically infer the data type of the request/response body based on
the method signature of the server handler; or the client parameters,
respectively.
This boils down to a special-case for io.Reader, which are just dumped
into a series of data frames as efficiently as possible.
All other types are (de)serialized using encoding/json.

The RPC layer and Frame Layer log some arbitrary messages that proved
useful during debugging. By default, they log to a non-logger, which
should not have a big impact on performance.

pprof analysis shows the implementation spends its CPU time
        60% waiting for syscalls
        30% in memmove
        10% ...

On a Intel(R) Core(TM) i7-6600U CPU @ 2.60GHz CPU, Linux 4.12, the
implementation achieved ~3.6GiB/s.

Future optimization may include spice(2) / vmspice(2) on Linux, although
this doesn't fit so well with the heavy use of io.Reader / io.Writer
throughout the codebase.

The existing hackaround for local calls was re-implemented to fit the
new interface of PRCServer and RPCClient.
The 'R'PC method invocation is a bit slower because reflection is
involved inbetween, but otherwise performance should be no different.

The RPC code currently does not support multipart requests and thus does
not support the equivalent of a POST.

Thus, the switch to the new rpc code had the following fallout:

- Move request objects + constants from rpc package to main app code
- Sacrifice the hacky 'push = pull me' way of doing push
-> need to further extend RPC to support multipart requests or
     something to implement this properly with additional interfaces
-> should be done after replication is abstracted better than separate
     algorithms for doPull() and doPush()
This commit is contained in:
Christian Schwarz
2017-08-19 22:37:14 +02:00
parent e5b713ce5b
commit 6ab05ee1fa
12 changed files with 938 additions and 815 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/config.go
+27 -23
View File
@@ -32,10 +32,9 @@ type Remote struct {
}
type Transport interface {
Connect(rpcLog Logger) (rpc.RPCRequester, error)
Connect(rpcLog Logger) (rpc.RPCClient, error)
}
type LocalTransport struct {
Handler rpc.RPCHandler
}
type SSHTransport struct {
Host string
@@ -53,14 +52,14 @@ type Push struct {
JobName string // for use with jobrun package
To *Remote
Filter zfs.DatasetFilter
InitialReplPolicy rpc.InitialReplPolicy
InitialReplPolicy InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type Pull struct {
JobName string // for use with jobrun package
From *Remote
Mapping DatasetMapFilter
InitialReplPolicy rpc.InitialReplPolicy
InitialReplPolicy InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
@@ -161,8 +160,8 @@ func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
remotes = make(map[string]*Remote, len(asMap))
for name, p := range asMap {
if name == rpc.LOCAL_TRANSPORT_IDENTITY {
err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
if name == LOCAL_TRANSPORT_IDENTITY {
err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", LOCAL_TRANSPORT_IDENTITY))
return
}
@@ -238,7 +237,7 @@ func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error)
return
}
if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
return
}
@@ -276,9 +275,9 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error)
var fromRemote *Remote
if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
if e.From == LOCAL_TRANSPORT_IDENTITY {
fromRemote = &Remote{
Name: rpc.LOCAL_TRANSPORT_IDENTITY,
Name: LOCAL_TRANSPORT_IDENTITY,
Transport: LocalTransport{},
}
} else {
@@ -296,7 +295,7 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error)
if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
return
}
if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
return
}
if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
@@ -309,7 +308,7 @@ func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error)
return
}
func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
s, ok := v.(string)
if !ok {
goto err
@@ -319,9 +318,9 @@ func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy)
case s == "":
p = defaultPolicy
case s == "most_recent":
p = rpc.InitialReplPolicyMostRecent
p = InitialReplPolicyMostRecent
case s == "all":
p = rpc.InitialReplPolicyAll
p = InitialReplPolicyAll
default:
goto err
}
@@ -434,7 +433,7 @@ func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter,
return
}
func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCClient, err error) {
var stream io.ReadWriteCloser
var rpcTransport sshbytestream.SSHTransport
if err = copier.Copy(&rpcTransport, t); err != nil {
@@ -447,18 +446,23 @@ func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
if err != nil {
return
}
return rpc.ConnectByteStreamRPC(stream, rpcLog)
client := rpc.NewClient(stream)
return client, nil
}
func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
if t.Handler == nil {
panic("local transport with uninitialized handler")
func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCClient, err error) {
local := rpc.NewLocalRPC()
handler := Handler{
Logger: log,
// Allow access to any dataset since we control what mapping
// is passed to the pull routine.
// All local datasets will be passed to its Map() function,
// but only those for which a mapping exists will actually be pulled.
// We can pay this small performance penalty for now.
PullACL: localPullACL{},
}
return rpc.ConnectLocalRPC(t.Handler), nil
}
func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
t.Handler = handler
registerEndpoints(local, handler)
return local, nil
}
func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
cmd/handler.go
+60 -45
View File
@@ -2,38 +2,79 @@ package cmd
import (
"fmt"
"io"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/zfs"
"io"
)
type DatasetMapping interface {
Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
}
type FilesystemRequest struct {
Roots []string // may be nil, indicating interest in all filesystems
}
type FilesystemVersionsRequest struct {
Filesystem *zfs.DatasetPath
}
type InitialTransferRequest struct {
Filesystem *zfs.DatasetPath
FilesystemVersion zfs.FilesystemVersion
}
type IncrementalTransferRequest struct {
Filesystem *zfs.DatasetPath
From zfs.FilesystemVersion
To zfs.FilesystemVersion
}
type Handler struct {
Logger Logger
PullACL zfs.DatasetFilter
SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
}
func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
if err != nil {
panic(err)
}
err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
if err != nil {
panic(err)
}
err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
if err != nil {
panic(err)
}
err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
if err != nil {
panic(err)
}
return nil
}
func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
h.Logger.Printf("handling fsr: %#v", r)
h.Logger.Printf("using PullACL: %#v", h.PullACL)
if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
allowed, err := zfs.ZFSListMapping(h.PullACL)
if err != nil {
h.Logger.Printf("handle fsr err: %v\n", err)
return
}
h.Logger.Printf("returning: %#v", roots)
h.Logger.Printf("returning: %#v", allowed)
*roots = allowed
return
}
func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
h.Logger.Printf("handling filesystem versions request: %#v", r)
@@ -43,17 +84,20 @@ func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest
}
// find our versions
if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, nil)
if err != nil {
h.Logger.Printf("our versions error: %#v\n", err)
return
}
h.Logger.Printf("our versions: %#v\n", versions)
h.Logger.Printf("our versions: %#v\n", vs)
*versions = vs
return
}
func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
h.Logger.Printf("handling initial transfer request: %#v", r)
if err = h.pullACLCheck(r.Filesystem); err != nil {
@@ -62,15 +106,17 @@ func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (str
h.Logger.Printf("invoking zfs send")
if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
if err != nil {
h.Logger.Printf("error sending filesystem: %#v", err)
}
*stream = s
return
}
func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
h.Logger.Printf("handling incremental transfer request: %#v", r)
if err = h.pullACLCheck(r.Filesystem); err != nil {
@@ -79,47 +125,16 @@ func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferReque
h.Logger.Printf("invoking zfs send")
if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
if err != nil {
h.Logger.Printf("error sending filesystem: %#v", err)
}
*stream = s
return
}
func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
// Check if we have a sink for this request
// Use that mapping to do what happens in doPull
h.Logger.Printf("handling PullMeRequest: %#v", r)
var sinkMapping DatasetMapping
sinkMapping, err = h.SinkMappingFunc(clientIdentity)
if err != nil {
h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
return
}
h.Logger.Printf("doing pull...")
err = doPull(PullContext{
Remote: client,
Log: h.Logger,
Mapping: sinkMapping,
InitialReplPolicy: r.InitialReplPolicy,
})
if err != nil {
h.Logger.Printf("PullMeRequest failed with error: %s", err)
return
}
h.Logger.Printf("finished handling PullMeRequest: %#v", r)
return
}
func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
var allowed bool
allowed, err = h.PullACL.Filter(p)
cmd/replication.go
+30 -45
View File
@@ -148,24 +148,20 @@ func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
return true, nil
}
const LOCAL_TRANSPORT_IDENTITY string = "local"
const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
type InitialReplPolicy string
const (
InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
InitialReplPolicyAll InitialReplPolicy = "all"
)
func jobPull(pull *Pull, log jobrun.Logger) (err error) {
if lt, ok := pull.From.Transport.(LocalTransport); ok {
lt.SetHandler(Handler{
Logger: log,
// Allow access to any dataset since we control what mapping
// is passed to the pull routine.
// All local datasets will be passed to its Map() function,
// but only those for which a mapping exists will actually be pulled.
// We can pay this small performance penalty for now.
PullACL: localPullACL{},
})
pull.From.Transport = lt
log.Printf("fixing up local transport: %#v", pull.From.Transport)
}
var remote rpc.RPCRequester
var remote rpc.RPCClient
if remote, err = pull.From.Transport.Connect(log); err != nil {
return
@@ -182,7 +178,7 @@ func jobPush(push *Push, log jobrun.Logger) (err error) {
panic("no support for local pushs")
}
var remote rpc.RPCRequester
var remote rpc.RPCClient
if remote, err = push.To.Transport.Connect(log); err != nil {
return err
}
@@ -197,27 +193,19 @@ func jobPush(push *Push, log jobrun.Logger) (err error) {
}
log.Printf("handler: %#v", handler)
r := rpc.PullMeRequest{
InitialReplPolicy: push.InitialReplPolicy,
}
log.Printf("doing PullMeRequest: %#v", r)
if err = remote.PullMeRequest(r, handler); err != nil {
log.Printf("PullMeRequest failed: %s", err)
return
}
panic("no support for push atm")
log.Printf("push job finished")
return
}
func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
log.Printf("closing rpc connection")
ch := make(chan error)
go func() {
ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
ch <- remote.Close()
close(ch)
}()
@@ -231,19 +219,15 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Durat
if err != nil {
log.Printf("error closing connection: %s", err)
err = remote.ForceClose()
if err != nil {
log.Printf("error force-closing connection: %s", err)
}
}
return
}
type PullContext struct {
Remote rpc.RPCRequester
Remote rpc.RPCClient
Log Logger
Mapping DatasetMapping
InitialReplPolicy rpc.InitialReplPolicy
InitialReplPolicy InitialReplPolicy
}
func doPull(pull PullContext) (err error) {
@@ -252,9 +236,9 @@ func doPull(pull PullContext) (err error) {
log := pull.Log
log.Printf("requesting remote filesystem list")
fsr := rpc.FilesystemRequest{}
fsr := FilesystemRequest{}
var remoteFilesystems []*zfs.DatasetPath
if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
if err = remote.Call("FilesystemRequest", &fsr, &remoteFilesystems); err != nil {
return
}
@@ -335,11 +319,11 @@ func doPull(pull PullContext) (err error) {
}
log("requesting remote filesystem versions")
var theirVersions []zfs.FilesystemVersion
theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
r := FilesystemVersionsRequest{
Filesystem: m.Remote,
})
if err != nil {
}
var theirVersions []zfs.FilesystemVersion
if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
log("error requesting remote filesystem versions: %s", err)
log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
return false
@@ -358,7 +342,7 @@ func doPull(pull PullContext) (err error) {
log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
if pull.InitialReplPolicy != InitialReplPolicyMostRecent {
panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
}
@@ -374,7 +358,7 @@ func doPull(pull PullContext) (err error) {
return false
}
r := rpc.InitialTransferRequest{
r := InitialTransferRequest{
Filesystem: m.Remote,
FilesystemVersion: snapsOnly[len(snapsOnly)-1],
}
@@ -382,7 +366,8 @@ func doPull(pull PullContext) (err error) {
log("requesting snapshot stream for %s", r.FilesystemVersion)
var stream io.Reader
if stream, err = remote.InitialTransferRequest(r); err != nil {
if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
log("error requesting initial transfer: %s", err)
return false
}
@@ -434,13 +419,13 @@ func doPull(pull PullContext) (err error) {
}
log("requesting incremental snapshot stream")
r := rpc.IncrementalTransferRequest{
r := IncrementalTransferRequest{
Filesystem: m.Remote,
From: from,
To: to,
}
var stream io.Reader
if stream, err = remote.IncrementalTransferRequest(r); err != nil {
if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
log("error requesting incremental snapshot stream: %s", err)
return false
}
cmd/stdinserver.go
+8 -5
View File
@@ -2,12 +2,13 @@ package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/sshbytestream"
"io"
golog "log"
"os"
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/sshbytestream"
)
var StdinserverCmd = &cobra.Command{
@@ -62,8 +63,10 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
PullACL: pullACL,
}
if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
log.Printf("listenbytestreamerror: %#v\n", err)
server := rpc.NewServer(sshByteStream)
registerEndpoints(server, handler)
if err = server.Serve(); err != nil {
log.Printf("error serving connection: %s", err)
os.Exit(1)
}