rpc rewrite: control RPCs using gRPC + separate RPC for data transfer

transport/ssh: update go-netssh to new version
    => supports CloseWrite and Deadlines
    => build: require Go 1.11 (netssh requires it)

transport/tls/connect_tls.go

@@ -0,0 +1,51 @@
+package tls
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/tlsconf"
+	"github.com/zrepl/zrepl/transport"
+)
+
+type TLSConnecter struct {
+	Address   string
+	dialer    net.Dialer
+	tlsConfig *tls.Config
+}
+
+func TLSConnecterFromConfig(in *config.TLSConnect) (*TLSConnecter, error) {
+	dialer := net.Dialer{
+		Timeout: in.DialTimeout,
+	}
+
+	ca, err := tlsconf.ParseCAFile(in.Ca)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse ca file")
+	}
+
+	cert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse cert/key pair")
+	}
+
+	tlsConfig, err := tlsconf.ClientAuthClient(in.ServerCN, ca, cert)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build tls config")
+	}
+
+	return &TLSConnecter{in.Address, dialer, tlsConfig}, nil
+}
+
+func (c *TLSConnecter) Connect(dialCtx context.Context) (transport.Wire, error) {
+	conn, err := c.dialer.DialContext(dialCtx, "tcp", c.Address)
+	if err != nil {
+		return nil, err
+	}
+	tcpConn := conn.(*net.TCPConn)
+	tlsConn := tls.Client(conn, c.tlsConfig)
+	return newWireAdaptor(tlsConn, tcpConn), nil
+}

transport/tls/serve_tls.go

@@ -0,0 +1,89 @@
+package tls
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/tlsconf"
+	"net"
+	"time"
+	"context"
+)
+
+type TLSListenerFactory struct {
+	address          string
+	clientCA         *x509.CertPool
+	serverCert       tls.Certificate
+	handshakeTimeout time.Duration
+	clientCNs map[string]struct{}
+}
+
+func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory,error) {
+
+	address := in.Listen
+	handshakeTimeout := in.HandshakeTimeout
+
+	if in.Ca == "" || in.Cert == "" || in.Key == "" {
+		return nil, errors.New("fields 'ca', 'cert' and 'key'must be specified")
+	}
+
+	clientCA, err := tlsconf.ParseCAFile(in.Ca)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse ca file")
+	}
+
+	serverCert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse cer/key pair")
+	}
+
+	clientCNs := make(map[string]struct{}, len(in.ClientCNs))
+	for i, cn := range in.ClientCNs {
+		if err := transport.ValidateClientIdentity(cn); err != nil {
+			return nil, errors.Wrapf(err, "unsuitable client_cn #%d %q", i, cn)
+		}
+		// dupes are ok fr now
+		clientCNs[cn] = struct{}{}
+	}
+
+	lf := func() (transport.AuthenticatedListener, error) {
+		l, err := net.Listen("tcp", address)
+		if err != nil {
+			return nil, err
+		}
+		tcpL := l.(*net.TCPListener)
+		tl := tlsconf.NewClientAuthListener(tcpL, clientCA, serverCert, handshakeTimeout)
+		return &tlsAuthListener{tl, clientCNs}, nil
+	}
+
+	return lf, nil
+}
+
+type tlsAuthListener struct {
+	*tlsconf.ClientAuthListener
+	clientCNs map[string]struct{}
+}
+
+func (l tlsAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
+	tcpConn, tlsConn, cn, err := l.ClientAuthListener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	if _, ok := l.clientCNs[cn]; !ok {
+		if dl, ok := ctx.Deadline(); ok {
+			defer tlsConn.SetDeadline(time.Time{})
+			tlsConn.SetDeadline(dl)
+		}
+		if err := tlsConn.Close(); err != nil {
+			transport.GetLogger(ctx).WithError(err).Error("error closing connection with unauthorized common name")
+		}
+		return nil, fmt.Errorf("unauthorized client common name %q from %s", cn, tlsConn.RemoteAddr())
+	}
+	adaptor := newWireAdaptor(tlsConn, tcpConn)
+	return transport.NewAuthConn(adaptor, cn), nil
+}
+
+

transport/tls/tls_wire_adaptor.go

@@ -0,0 +1,47 @@
+package tls
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"os"
+)
+
+// adapts a tls.Conn and its underlying net.TCPConn into a valid transport.Wire
+type transportWireAdaptor struct {
+	*tls.Conn
+	tcpConn *net.TCPConn
+}
+
+func newWireAdaptor(tlsConn *tls.Conn, tcpConn *net.TCPConn) transportWireAdaptor {
+	return transportWireAdaptor{tlsConn, tcpConn}
+}
+
+// CloseWrite implements transport.Wire.CloseWrite which is different from *tls.Conn.CloseWrite:
+// the former requires that the other side observes io.EOF, but *tls.Conn.CloseWrite does not
+// close the underlying connection so no io.EOF would be observed.
+func (w transportWireAdaptor) CloseWrite() error {
+	if err := w.Conn.CloseWrite(); err != nil {
+		// TODO log error
+		fmt.Fprintf(os.Stderr, "transport/tls.CloseWrite() error: %s\n", err)
+	}
+	return w.tcpConn.CloseWrite()
+}
+
+// Close implements transport.Wire.Close which is different from a *tls.Conn.Close:
+// At the time of writing (Go 1.11), closing tls.Conn closes the TCP connection immediately,
+// which results in io.ErrUnexpectedEOF on the other side.
+// We assume that w.Conn has a deadline set for the close, so the CloseWrite will time out if it blocks,
+// falling through to the actual Close()
+func (w transportWireAdaptor) Close() error {
+	//	var buf [1<<15]byte
+	//	w.Conn.Write(buf[:])
+	// CloseWrite will send a TLS alert record down the line which
+	// in the Go implementation acts like a flush...?
+	//	if err := w.Conn.CloseWrite(); err != nil {
+	//		// TODO log error
+	//		fmt.Fprintf(os.Stderr, "transport/tls.Close() close write error: %s\n", err)
+	//	}
+	//	time.Sleep(1 * time.Second)
+	return w.Conn.Close()
+}