rpc rewrite: control RPCs using gRPC + separate RPC for data transfer

transport/ssh: update go-netssh to new version
    => supports CloseWrite and Deadlines
    => build: require Go 1.11 (netssh requires it)
This commit is contained in:
Christian Schwarz
2018-12-11 22:01:50 +01:00
parent d281fb00e3
commit 796c5ad42d
100 changed files with 6460 additions and 1485 deletions
@@ -0,0 +1,58 @@
// Package fromconfig instantiates transports based on zrepl config structures
// (see package config).
package fromconfig
import (
"fmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/transport/local"
"github.com/zrepl/zrepl/transport/ssh"
"github.com/zrepl/zrepl/transport/tcp"
"github.com/zrepl/zrepl/transport/tls"
)
func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport.AuthenticatedListenerFactory,error) {
var (
l transport.AuthenticatedListenerFactory
err error
)
switch v := in.Ret.(type) {
case *config.TCPServe:
l, err = tcp.TCPListenerFactoryFromConfig(g, v)
case *config.TLSServe:
l, err = tls.TLSListenerFactoryFromConfig(g, v)
case *config.StdinserverServer:
l, err = ssh.MultiStdinserverListenerFactoryFromConfig(g, v)
case *config.LocalServe:
l, err = local.LocalListenerFactoryFromConfig(g, v)
default:
return nil, errors.Errorf("internal error: unknown serve type %T", v)
}
return l, err
}
func ConnecterFromConfig(g *config.Global, in config.ConnectEnum) (transport.Connecter, error) {
var (
connecter transport.Connecter
err error
)
switch v := in.Ret.(type) {
case *config.SSHStdinserverConnect:
connecter, err = ssh.SSHStdinserverConnecterFromConfig(v)
case *config.TCPConnect:
connecter, err = tcp.TCPConnecterFromConfig(v)
case *config.TLSConnect:
connecter, err = tls.TLSConnecterFromConfig(v)
case *config.LocalConnect:
connecter, err = local.LocalConnecterFromConfig(v)
default:
panic(fmt.Sprintf("implementation error: unknown connecter type %T", v))
}
return connecter, err
}
+29
View File
@@ -0,0 +1,29 @@
package local
import (
"context"
"fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
)
type LocalConnecter struct {
listenerName string
clientIdentity string
}
func LocalConnecterFromConfig(in *config.LocalConnect) (*LocalConnecter, error) {
if in.ClientIdentity == "" {
return nil, fmt.Errorf("ClientIdentity must not be empty")
}
if in.ListenerName == "" {
return nil, fmt.Errorf("ListenerName must not be empty")
}
return &LocalConnecter{listenerName: in.ListenerName, clientIdentity: in.ClientIdentity}, nil
}
func (c *LocalConnecter) Connect(dialCtx context.Context) (transport.Wire, error) {
l := GetLocalListener(c.listenerName)
return l.Connect(dialCtx, c.clientIdentity)
}
+175
View File
@@ -0,0 +1,175 @@
package local
import (
"context"
"fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/util/socketpair"
"net"
"sync"
"github.com/zrepl/zrepl/transport"
)
var localListeners struct {
m map[string]*LocalListener // listenerName -> listener
init sync.Once
mtx sync.Mutex
}
func GetLocalListener(listenerName string) (*LocalListener) {
localListeners.init.Do(func() {
localListeners.m = make(map[string]*LocalListener)
})
localListeners.mtx.Lock()
defer localListeners.mtx.Unlock()
l, ok := localListeners.m[listenerName]
if !ok {
l = newLocalListener()
localListeners.m[listenerName] = l
}
return l
}
type connectRequest struct {
clientIdentity string
callback chan connectResult
}
type connectResult struct {
conn transport.Wire
err error
}
type LocalListener struct {
connects chan connectRequest
}
func newLocalListener() *LocalListener {
return &LocalListener{
connects: make(chan connectRequest),
}
}
// Connect to the LocalListener from a client with identity clientIdentity
func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string) (conn transport.Wire, err error) {
// place request
req := connectRequest{
clientIdentity: clientIdentity,
callback: make(chan connectResult),
}
select {
case l.connects <- req:
case <-dialCtx.Done():
return nil, dialCtx.Err()
}
// wait for listener response
select {
case connRes := <- req.callback:
conn, err = connRes.conn, connRes.err
case <-dialCtx.Done():
close(req.callback) // sending to the channel afterwards will panic, the listener has to catch this
conn, err = nil, dialCtx.Err()
}
return conn, err
}
type localAddr struct {
S string
}
func (localAddr) Network() string { return "local" }
func (a localAddr) String() string { return a.S }
func (l *LocalListener) Addr() (net.Addr) { return localAddr{"<listening>"} }
func (l *LocalListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
respondToRequest := func(req connectRequest, res connectResult) (err error) {
transport.GetLogger(ctx).
WithField("res.conn", res.conn).WithField("res.err", res.err).
Debug("responding to client request")
defer func() {
errv := recover()
transport.GetLogger(ctx).WithField("recover_err", errv).
Debug("panic on send to client callback, likely a legitimate client-side timeout")
}()
select {
case req.callback <- res:
err = nil
default:
err = fmt.Errorf("client-provided callback did block on send")
}
close(req.callback)
return err
}
transport.GetLogger(ctx).Debug("waiting for local client connect requests")
var req connectRequest
select {
case req = <-l.connects:
case <-ctx.Done():
return nil, ctx.Err()
}
transport.GetLogger(ctx).WithField("client_identity", req.clientIdentity).Debug("got connect request")
if req.clientIdentity == "" {
res := connectResult{nil, fmt.Errorf("client identity must not be empty")}
if err := respondToRequest(req, res); err != nil {
return nil, err
}
return nil, fmt.Errorf("client connected with empty client identity")
}
transport.GetLogger(ctx).Debug("creating socketpair")
left, right, err := socketpair.SocketPair()
if err != nil {
res := connectResult{nil, fmt.Errorf("server error: %s", err)}
if respErr := respondToRequest(req, res); respErr != nil {
// returning the socketpair error properly is more important than the error sent to the client
transport.GetLogger(ctx).WithError(respErr).Error("error responding to client")
}
return nil, err
}
transport.GetLogger(ctx).Debug("responding with left side of socketpair")
res := connectResult{left, nil}
if err := respondToRequest(req, res); err != nil {
transport.GetLogger(ctx).WithError(err).Error("error responding to client")
if err := left.Close(); err != nil {
transport.GetLogger(ctx).WithError(err).Error("cannot close left side of socketpair")
}
if err := right.Close(); err != nil {
transport.GetLogger(ctx).WithError(err).Error("cannot close right side of socketpair")
}
return nil, err
}
return transport.NewAuthConn(right, req.clientIdentity), nil
}
func (l *LocalListener) Close() error {
// FIXME: make sure concurrent Accepts return with error, and further Accepts return that error, too
// Example impl: for each accept, do context.WithCancel, and store the cancel in a list
// When closing, set a member variable to state=closed, make sure accept will exit early
// and then call all cancels in the list
// The code path from Accept entry over check if state=closed to list entry must be protected by a mutex.
return nil
}
func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (transport.AuthenticatedListenerFactory,error) {
if in.ListenerName == "" {
return nil, fmt.Errorf("ListenerName must not be empty")
}
listenerName := in.ListenerName
lf := func() (transport.AuthenticatedListener,error) {
return GetLocalListener(listenerName), nil
}
return lf, nil
}
+55
View File
@@ -0,0 +1,55 @@
package ssh
import (
"context"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
"time"
)
type SSHStdinserverConnecter struct {
Host string
User string
Port uint16
IdentityFile string
TransportOpenCommand []string
SSHCommand string
Options []string
dialTimeout time.Duration
}
func SSHStdinserverConnecterFromConfig(in *config.SSHStdinserverConnect) (c *SSHStdinserverConnecter, err error) {
c = &SSHStdinserverConnecter{
Host: in.Host,
User: in.User,
Port: in.Port,
IdentityFile: in.IdentityFile,
SSHCommand: in.SSHCommand,
Options: in.Options,
dialTimeout: in.DialTimeout,
}
return
}
func (c *SSHStdinserverConnecter) Connect(dialCtx context.Context) (transport.Wire, error) {
var endpoint netssh.Endpoint
if err := copier.Copy(&endpoint, c); err != nil {
return nil, errors.WithStack(err)
}
dialCtx, dialCancel := context.WithTimeout(dialCtx, c.dialTimeout) // context.TODO tied to error handling below
defer dialCancel()
nconn, err := netssh.Dial(dialCtx, endpoint)
if err != nil {
if err == context.DeadlineExceeded {
err = errors.Errorf("dial_timeout of %s exceeded", c.dialTimeout)
}
return nil, err
}
return nconn, nil
}
+147
View File
@@ -0,0 +1,147 @@
package ssh
import (
"github.com/problame/go-netssh"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/nethelpers"
"github.com/zrepl/zrepl/transport"
"fmt"
"net"
"path"
"context"
"github.com/pkg/errors"
"sync/atomic"
)
func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (transport.AuthenticatedListenerFactory,error) {
for _, ci := range in.ClientIdentities {
if err := transport.ValidateClientIdentity(ci); err != nil {
return nil, errors.Wrapf(err, "invalid client identity %q", ci)
}
}
clientIdentities := in.ClientIdentities
sockdir := g.Serve.StdinServer.SockDir
lf := func() (transport.AuthenticatedListener,error) {
return multiStdinserverListenerFromClientIdentities(sockdir, clientIdentities)
}
return lf, nil
}
type multiStdinserverAcceptRes struct {
conn *transport.AuthConn
err error
}
type MultiStdinserverListener struct {
listeners []*stdinserverListener
accepts chan multiStdinserverAcceptRes
closed int32
}
// client identities must be validated
func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string) (*MultiStdinserverListener, error) {
listeners := make([]*stdinserverListener, 0, len(cis))
var err error
for _, ci := range cis {
sockpath := path.Join(sockdir, ci)
l := &stdinserverListener{clientIdentity: ci}
if err = nethelpers.PreparePrivateSockpath(sockpath); err != nil {
break
}
if l.l, err = netssh.Listen(sockpath); err != nil {
break
}
listeners = append(listeners, l)
}
if err != nil {
for _, l := range listeners {
l.Close() // FIXME error reporting?
}
return nil, err
}
return &MultiStdinserverListener{listeners: listeners}, nil
}
func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error){
if m.accepts == nil {
m.accepts = make(chan multiStdinserverAcceptRes, len(m.listeners))
for i := range m.listeners {
go func(i int) {
for atomic.LoadInt32(&m.closed) == 0 {
conn, err := m.listeners[i].Accept(context.TODO())
m.accepts <- multiStdinserverAcceptRes{conn, err}
}
}(i)
}
}
res := <- m.accepts
return res.conn, res.err
}
type multiListenerAddr struct {
clients []string
}
func (multiListenerAddr) Network() string { return "netssh" }
func (l multiListenerAddr) String() string {
return fmt.Sprintf("netssh:clients=%v", l.clients)
}
func (m *MultiStdinserverListener) Addr() net.Addr {
cis := make([]string, len(m.listeners))
for i := range cis {
cis[i] = m.listeners[i].clientIdentity
}
return multiListenerAddr{cis}
}
func (m *MultiStdinserverListener) Close() error {
atomic.StoreInt32(&m.closed, 1)
var oneErr error
for _, l := range m.listeners {
if err := l.Close(); err != nil && oneErr == nil {
oneErr = err
}
}
return oneErr
}
// a single stdinserverListener (part of multiStinserverListener)
type stdinserverListener struct {
l *netssh.Listener
clientIdentity string
}
type listenerAddr struct {
clientIdentity string
}
func (listenerAddr) Network() string { return "netssh" }
func (a listenerAddr) String() string {
return fmt.Sprintf("netssh:client=%q", a.clientIdentity)
}
func (l stdinserverListener) Addr() net.Addr {
return listenerAddr{l.clientIdentity}
}
func (l stdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
c, err := l.l.Accept()
if err != nil {
return nil, err
}
return transport.NewAuthConn(c, l.clientIdentity), nil
}
func (l stdinserverListener) Close() (err error) {
return l.l.Close()
}
+30
View File
@@ -0,0 +1,30 @@
package tcp
import (
"context"
"net"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
)
type TCPConnecter struct {
Address string
dialer net.Dialer
}
func TCPConnecterFromConfig(in *config.TCPConnect) (*TCPConnecter, error) {
dialer := net.Dialer{
Timeout: in.DialTimeout,
}
return &TCPConnecter{in.Address, dialer}, nil
}
func (c *TCPConnecter) Connect(dialCtx context.Context) (transport.Wire, error) {
conn, err := c.dialer.DialContext(dialCtx, "tcp", c.Address)
if err != nil {
return nil, err
}
return conn.(*net.TCPConn), nil
}
+82
View File
@@ -0,0 +1,82 @@
package tcp
import (
"github.com/zrepl/zrepl/config"
"net"
"github.com/pkg/errors"
"context"
"github.com/zrepl/zrepl/transport"
)
type ipMapEntry struct {
ip net.IP
ident string
}
type ipMap struct {
entries []ipMapEntry
}
func ipMapFromConfig(clients map[string]string) (*ipMap, error) {
entries := make([]ipMapEntry, 0, len(clients))
for clientIPString, clientIdent := range clients {
clientIP := net.ParseIP(clientIPString)
if clientIP == nil {
return nil, errors.Errorf("cannot parse client IP %q", clientIPString)
}
if err := transport.ValidateClientIdentity(clientIdent); err != nil {
return nil, errors.Wrapf(err,"invalid client identity for IP %q", clientIPString)
}
entries = append(entries, ipMapEntry{clientIP, clientIdent})
}
return &ipMap{entries: entries}, nil
}
func (m *ipMap) Get(ip net.IP) (string, error) {
for _, e := range m.entries {
if e.ip.Equal(ip) {
return e.ident, nil
}
}
return "", errors.Errorf("no identity mapping for client IP %s", ip)
}
func TCPListenerFactoryFromConfig(c *config.Global, in *config.TCPServe) (transport.AuthenticatedListenerFactory, error) {
addr, err := net.ResolveTCPAddr("tcp", in.Listen)
if err != nil {
return nil, errors.Wrap(err, "cannot parse listen address")
}
clientMap, err := ipMapFromConfig(in.Clients)
if err != nil {
return nil, errors.Wrap(err, "cannot parse client IP map")
}
lf := func() (transport.AuthenticatedListener, error) {
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return nil, err
}
return &TCPAuthListener{l, clientMap}, nil
}
return lf, nil
}
type TCPAuthListener struct {
*net.TCPListener
clientMap *ipMap
}
func (f *TCPAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
nc, err := f.TCPListener.AcceptTCP()
if err != nil {
return nil, err
}
clientIP := nc.RemoteAddr().(*net.TCPAddr).IP
clientIdent, err := f.clientMap.Get(clientIP)
if err != nil {
transport.GetLogger(ctx).WithField("ip", clientIP).Error("client IP not in client map")
nc.Close()
return nil, err
}
return transport.NewAuthConn(nc, clientIdent), nil
}
+51
View File
@@ -0,0 +1,51 @@
package tls
import (
"context"
"crypto/tls"
"net"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/tlsconf"
"github.com/zrepl/zrepl/transport"
)
type TLSConnecter struct {
Address string
dialer net.Dialer
tlsConfig *tls.Config
}
func TLSConnecterFromConfig(in *config.TLSConnect) (*TLSConnecter, error) {
dialer := net.Dialer{
Timeout: in.DialTimeout,
}
ca, err := tlsconf.ParseCAFile(in.Ca)
if err != nil {
return nil, errors.Wrap(err, "cannot parse ca file")
}
cert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
if err != nil {
return nil, errors.Wrap(err, "cannot parse cert/key pair")
}
tlsConfig, err := tlsconf.ClientAuthClient(in.ServerCN, ca, cert)
if err != nil {
return nil, errors.Wrap(err, "cannot build tls config")
}
return &TLSConnecter{in.Address, dialer, tlsConfig}, nil
}
func (c *TLSConnecter) Connect(dialCtx context.Context) (transport.Wire, error) {
conn, err := c.dialer.DialContext(dialCtx, "tcp", c.Address)
if err != nil {
return nil, err
}
tcpConn := conn.(*net.TCPConn)
tlsConn := tls.Client(conn, c.tlsConfig)
return newWireAdaptor(tlsConn, tcpConn), nil
}
+89
View File
@@ -0,0 +1,89 @@
package tls
import (
"crypto/tls"
"crypto/x509"
"fmt"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/tlsconf"
"net"
"time"
"context"
)
type TLSListenerFactory struct {
address string
clientCA *x509.CertPool
serverCert tls.Certificate
handshakeTimeout time.Duration
clientCNs map[string]struct{}
}
func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory,error) {
address := in.Listen
handshakeTimeout := in.HandshakeTimeout
if in.Ca == "" || in.Cert == "" || in.Key == "" {
return nil, errors.New("fields 'ca', 'cert' and 'key'must be specified")
}
clientCA, err := tlsconf.ParseCAFile(in.Ca)
if err != nil {
return nil, errors.Wrap(err, "cannot parse ca file")
}
serverCert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
if err != nil {
return nil, errors.Wrap(err, "cannot parse cer/key pair")
}
clientCNs := make(map[string]struct{}, len(in.ClientCNs))
for i, cn := range in.ClientCNs {
if err := transport.ValidateClientIdentity(cn); err != nil {
return nil, errors.Wrapf(err, "unsuitable client_cn #%d %q", i, cn)
}
// dupes are ok fr now
clientCNs[cn] = struct{}{}
}
lf := func() (transport.AuthenticatedListener, error) {
l, err := net.Listen("tcp", address)
if err != nil {
return nil, err
}
tcpL := l.(*net.TCPListener)
tl := tlsconf.NewClientAuthListener(tcpL, clientCA, serverCert, handshakeTimeout)
return &tlsAuthListener{tl, clientCNs}, nil
}
return lf, nil
}
type tlsAuthListener struct {
*tlsconf.ClientAuthListener
clientCNs map[string]struct{}
}
func (l tlsAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
tcpConn, tlsConn, cn, err := l.ClientAuthListener.Accept()
if err != nil {
return nil, err
}
if _, ok := l.clientCNs[cn]; !ok {
if dl, ok := ctx.Deadline(); ok {
defer tlsConn.SetDeadline(time.Time{})
tlsConn.SetDeadline(dl)
}
if err := tlsConn.Close(); err != nil {
transport.GetLogger(ctx).WithError(err).Error("error closing connection with unauthorized common name")
}
return nil, fmt.Errorf("unauthorized client common name %q from %s", cn, tlsConn.RemoteAddr())
}
adaptor := newWireAdaptor(tlsConn, tcpConn)
return transport.NewAuthConn(adaptor, cn), nil
}
+47
View File
@@ -0,0 +1,47 @@
package tls
import (
"crypto/tls"
"fmt"
"net"
"os"
)
// adapts a tls.Conn and its underlying net.TCPConn into a valid transport.Wire
type transportWireAdaptor struct {
*tls.Conn
tcpConn *net.TCPConn
}
func newWireAdaptor(tlsConn *tls.Conn, tcpConn *net.TCPConn) transportWireAdaptor {
return transportWireAdaptor{tlsConn, tcpConn}
}
// CloseWrite implements transport.Wire.CloseWrite which is different from *tls.Conn.CloseWrite:
// the former requires that the other side observes io.EOF, but *tls.Conn.CloseWrite does not
// close the underlying connection so no io.EOF would be observed.
func (w transportWireAdaptor) CloseWrite() error {
if err := w.Conn.CloseWrite(); err != nil {
// TODO log error
fmt.Fprintf(os.Stderr, "transport/tls.CloseWrite() error: %s\n", err)
}
return w.tcpConn.CloseWrite()
}
// Close implements transport.Wire.Close which is different from a *tls.Conn.Close:
// At the time of writing (Go 1.11), closing tls.Conn closes the TCP connection immediately,
// which results in io.ErrUnexpectedEOF on the other side.
// We assume that w.Conn has a deadline set for the close, so the CloseWrite will time out if it blocks,
// falling through to the actual Close()
func (w transportWireAdaptor) Close() error {
// var buf [1<<15]byte
// w.Conn.Write(buf[:])
// CloseWrite will send a TLS alert record down the line which
// in the Go implementation acts like a flush...?
// if err := w.Conn.CloseWrite(); err != nil {
// // TODO log error
// fmt.Fprintf(os.Stderr, "transport/tls.Close() close write error: %s\n", err)
// }
// time.Sleep(1 * time.Second)
return w.Conn.Close()
}
+84
View File
@@ -0,0 +1,84 @@
// Package transport defines a common interface for
// network connections that have an associated client identity.
package transport
import (
"context"
"errors"
"net"
"syscall"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"
"github.com/zrepl/zrepl/zfs"
)
type AuthConn struct {
Wire
clientIdentity string
}
var _ timeoutconn.SyscallConner = AuthConn{}
func (a AuthConn) SyscallConn() (rawConn syscall.RawConn, err error) {
scc, ok := a.Wire.(timeoutconn.SyscallConner)
if !ok {
return nil, timeoutconn.SyscallConnNotSupported
}
return scc.SyscallConn()
}
func NewAuthConn(conn Wire, clientIdentity string) *AuthConn {
return &AuthConn{conn, clientIdentity}
}
func (c *AuthConn) ClientIdentity() string {
if err := ValidateClientIdentity(c.clientIdentity); err != nil {
panic(err)
}
return c.clientIdentity
}
// like net.Listener, but with an AuthenticatedConn instead of net.Conn
type AuthenticatedListener interface {
Addr() net.Addr
Accept(ctx context.Context) (*AuthConn, error)
Close() error
}
type AuthenticatedListenerFactory func() (AuthenticatedListener, error)
type Wire = timeoutconn.Wire
type Connecter interface {
Connect(ctx context.Context) (Wire, error)
}
// A client identity must be a single component in a ZFS filesystem path
func ValidateClientIdentity(in string) (err error) {
path, err := zfs.NewDatasetPath(in)
if err != nil {
return err
}
if path.Length() != 1 {
return errors.New("client identity must be a single path comonent (not empty, no '/')")
}
return nil
}
type contextKey int
const contextKeyLog contextKey = 0
type Logger = logger.Logger
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, log)
}
func GetLogger(ctx context.Context) Logger {
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
return log
}
return logger.NewNullLogger()
}