dump logrus and roll our own logger instead

2017-09-23 18:20:22 +02:00
parent c31ec8c646
commit e0e362c4ff
10 changed files with 491 additions and 251 deletions
@@ -3,32 +3,18 @@ package cmd
 import (
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	"github.com/zrepl/zrepl/logger"
 	"os"
 	"time"
 )

 type LoggingConfig struct {
-	Stdout struct {
-		Level  logrus.Level
-		Format LogFormat
-	}
-	TCP *TCPLoggingConfig
-}
-
-type TCPLoggingConfig struct {
-	Level         logrus.Level
-	Format        LogFormat
-	Net           string
-	Address       string
-	RetryInterval time.Duration
+	Outlets logger.Outlets
 }

 func parseLogging(i interface{}) (c *LoggingConfig, err error) {

 	c = &LoggingConfig{}
-	c.Stdout.Level = logrus.WarnLevel
-	c.Stdout.Format = LogFormatHuman
 	if i == nil {
 		return c, nil
 	}
@@ -50,89 +36,75 @@ func parseLogging(i interface{}) (c *LoggingConfig, err error) {
 		return nil, errors.Wrap(err, "mapstructure error")
 	}

+	c.Outlets = logger.NewOutlets()
+
 	if asMap.Stdout.Level != "" {
-		lvl, err := logrus.ParseLevel(asMap.Stdout.Level)
+
+		out := WriterOutlet{
+			HumanFormatter{},
+			os.Stdout,
+		}
+
+		level, err := logger.ParseLevel(asMap.Stdout.Level)
 		if err != nil {
 			return nil, errors.Wrap(err, "cannot parse stdout log level")
 		}
-		c.Stdout.Level = lvl
-	}
-	if asMap.Stdout.Format != "" {
-		format, err := parseLogFormat(asMap.Stdout.Format)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse log format")
+
+		if asMap.Stdout.Format != "" {
+			out.Formatter, err = parseLogFormat(asMap.Stdout.Format)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot parse log format")
+			}
 		}
-		c.Stdout.Format = format
+
+		c.Outlets.Add(out, level)
+
 	}

 	if asMap.TCP.Address != "" {
-		c.TCP = &TCPLoggingConfig{}
-		c.TCP.Format, err = parseLogFormat(asMap.TCP.Format)
+
+		out := &TCPOutlet{}
+
+		out.Formatter, err = parseLogFormat(asMap.TCP.Format)
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse log format")
+			return nil, errors.Wrap(err, "cannot parse log 'format'")
 		}
-		c.TCP.Level, err = logrus.ParseLevel(asMap.TCP.Level)
+
+		lvl, err := logger.ParseLevel(asMap.TCP.Level)
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse level")
+			return nil, errors.Wrap(err, "cannot parse 'level'")
 		}
-		c.TCP.RetryInterval, err = time.ParseDuration(asMap.TCP.RetryInterval)
-		c.TCP.Net, c.TCP.Address = asMap.TCP.Net, asMap.TCP.Address
+
+		out.RetryInterval, err = time.ParseDuration(asMap.TCP.RetryInterval)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+		}
+
+		out.Net, out.Address = asMap.TCP.Net, asMap.TCP.Address
+
+		c.Outlets.Add(out, lvl)
 	}

 	return c, nil

 }

-type LogFormat string
-
-const (
-	LogFormatHuman  LogFormat = "human"
-	LogFormatLogfmt LogFormat = "logfmt"
-	LogFormatJSON   LogFormat = "json"
-)
-
-func (f LogFormat) Formatter() logrus.Formatter {
-	switch f {
-	case LogFormatHuman:
-		return HumanFormatter{}
-	case LogFormatLogfmt:
-		return &logrus.TextFormatter{}
-	case LogFormatJSON:
-		return &logrus.JSONFormatter{}
-	default:
-		panic("incomplete implementation")
-	}
-}
-
-var LogFormats []LogFormat = []LogFormat{LogFormatHuman, LogFormatLogfmt, LogFormatJSON}
-
-func parseLogFormat(i interface{}) (f LogFormat, err error) {
+func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
 	var is string
 	switch j := i.(type) {
 	case string:
 		is = j
 	default:
-		return "", errors.Errorf("invalid log format: wrong type: %T", i)
+		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
 	}

-	for _, f := range LogFormats {
-		if string(f) == is {
-			return f, nil
-		}
+	switch is {
+	case "human":
+		return HumanFormatter{}, nil
+	case "json":
+		return &JSONFormatter{}, nil
+	default:
+		return nil, errors.Errorf("invalid log format: '%s'", is)
 	}
-	return "", errors.Errorf("invalid log format: '%s'", is)
-}
-
-func (c *LoggingConfig) MakeLogrus() (l logrus.FieldLogger) {
-
-	log := logrus.New()
-	log.Out = os.Stdout
-	log.Level = logrus.DebugLevel // FIXTHIS IN LOGRUS
-	log.Formatter = c.Stdout.Format.Formatter()
-
-	th := &TCPHook{Formatter: JSONFormatter{}, MinLevel: c.TCP.Level, Net: c.TCP.Net, Address: c.TCP.Address, RetryInterval: c.TCP.RetryInterval}
-	log.Hooks.Add(th)
-
-	return log

 }
@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 )

 // daemonCmd represents the daemon command
@@ -33,7 +35,8 @@ func doDaemon(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	log := conf.Global.logging.MakeLogrus()
+	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
+
 	log.Debug("starting daemon")
 	ctx := context.WithValue(context.Background(), contextKeyLog, log)
 	ctx = context.WithValue(ctx, contextKeyLog, log)
@@ -0,0 +1,114 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"strings"
+	"time"
+)
+
+type EntryFormatter interface {
+	Format(e *logger.Entry) ([]byte, error)
+}
+
+const (
+	FieldLevel   = "level"
+	FieldMessage = "msg"
+	FieldTime    = "time"
+)
+
+const (
+	logJobField     string = "job"
+	logTaskField    string = "task"
+	logFSField      string = "filesystem"
+	logMapFromField string = "map_from"
+	logMapToField   string = "map_to"
+	logIncFromField string = "inc_from"
+	logIncToField   string = "inc_to"
+)
+
+type NoFormatter struct{}
+
+func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
+	return []byte(e.Message), nil
+}
+
+type HumanFormatter struct{}
+
+func (f HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
+
+	var line bytes.Buffer
+
+	fmt.Fprintf(&line, "[%s]", e.Level.Short())
+
+	prefixFields := []string{logJobField, logTaskField, logFSField}
+	prefixed := make(map[string]bool, len(prefixFields)+2)
+	for _, field := range prefixFields {
+		val, ok := e.Fields[field].(string)
+		if ok {
+			fmt.Fprintf(&line, "[%s]", val)
+			prefixed[field] = true
+		} else {
+			break
+		}
+	}
+	// even more prefix fields
+	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
+	mapTo, mapToOk := e.Fields[logMapToField].(string)
+	if mapFromOk && mapToOk {
+		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
+		prefixed[logMapFromField], prefixed[logMapToField] = true, true
+	}
+	incFrom, incFromOk := e.Fields[logIncFromField].(string)
+	incTo, incToOk := e.Fields[logIncToField].(string)
+	if incFromOk && incToOk {
+		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
+		prefixed[logIncFromField], prefixed[logIncToField] = true, true
+	}
+
+	fmt.Fprintf(&line, ": %s", e.Message)
+
+	for field, value := range e.Fields {
+
+		if prefixed[field] {
+			continue
+		}
+
+		if strings.ContainsAny(field, " \t") {
+			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
+		}
+		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
+	}
+
+	return line.Bytes(), nil
+}
+
+type JSONFormatter struct{}
+
+func (f JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
+	data := make(logger.Fields, len(e.Fields)+3)
+	for k, v := range e.Fields {
+		switch v := v.(type) {
+		case error:
+			// Otherwise errors are ignored by `encoding/json`
+			// https://github.com/sirupsen/logrus/issues/137
+			data[k] = v.Error()
+		default:
+			_, err := json.Marshal(v)
+			if err != nil {
+				return nil, errors.Errorf("field is not JSON encodable: %s", k)
+			}
+			data[k] = v
+		}
+	}
+
+	data[FieldMessage] = e.Message
+	data[FieldTime] = e.Time.Format(time.RFC3339)
+	data[FieldLevel] = e.Level
+
+	return json.Marshal(data)
+
+}
@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"io"
+	"net"
+	"time"
+)
+
+type WriterOutlet struct {
+	Formatter EntryFormatter
+	Writer    io.Writer
+}
+
+func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+	bytes, err := h.Formatter.Format(&entry)
+	if err != nil {
+		return err
+	}
+	_, err = h.Writer.Write(bytes)
+	h.Writer.Write([]byte("\n"))
+	return err
+}
+
+type TCPOutlet struct {
+	Formatter     EntryFormatter
+	Net, Address  string
+	Dialer        net.Dialer
+	RetryInterval time.Duration
+	conn          net.Conn
+	retry         time.Time
+}
+
+func (h *TCPOutlet) WriteEntry(ctx context.Context, e logger.Entry) error {
+
+	b, err := h.Formatter.Format(&e)
+	if err != nil {
+		return err
+	}
+
+	if h.conn == nil {
+		if time.Now().Sub(h.retry) < h.RetryInterval {
+			return nil // this is not an error toward the logger
+			//return errors.New("TCP hook reconnect prohibited by retry interval")
+		}
+		h.conn, err = h.Dialer.DialContext(ctx, h.Net, h.Address)
+		if err != nil {
+			h.retry = time.Now()
+			return errors.Wrap(err, "cannot dial")
+		}
+	}
+
+	_, err = h.conn.Write(b)
+	if err != nil {
+		h.conn.Close()
+		h.conn = nil
+		return err
+	}
+
+	return nil
+}
@@ -1,171 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"net"
-	"strings"
-	"time"
-)
-
-const (
-	logJobField     string = "job"
-	logTaskField    string = "task"
-	logFSField      string = "filesystem"
-	logMapFromField string = "map_from"
-	logMapToField   string = "map_to"
-	logIncFromField string = "inc_from"
-	logIncToField   string = "inc_to"
-)
-
-type NoFormatter struct{}
-
-func (f NoFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	return []byte(e.Message), nil
-}
-
-type HumanFormatter struct{}
-
-func (f HumanFormatter) shortLevel(l logrus.Level) string {
-	switch l {
-	case logrus.DebugLevel:
-		return "DBG"
-	case logrus.InfoLevel:
-		return "INF"
-	case logrus.WarnLevel:
-		return "WRN"
-	case logrus.ErrorLevel:
-		return "ERR"
-	case logrus.PanicLevel:
-		return "PNC"
-	}
-	panic("incomplete implementation")
-}
-
-func (f HumanFormatter) Format(e *logrus.Entry) (out []byte, err error) {
-
-	var line bytes.Buffer
-
-	fmt.Fprintf(&line, "[%s]", f.shortLevel(e.Level))
-
-	prefixFields := []string{logJobField, logTaskField, logFSField}
-	prefixed := make(map[string]bool, len(prefixFields)+2)
-	for _, field := range prefixFields {
-		val, ok := e.Data[field].(string)
-		if ok {
-			fmt.Fprintf(&line, "[%s]", val)
-			prefixed[field] = true
-		} else {
-			break
-		}
-	}
-	// even more prefix fields
-	mapFrom, mapFromOk := e.Data[logMapFromField].(string)
-	mapTo, mapToOk := e.Data[logMapToField].(string)
-	if mapFromOk && mapToOk {
-		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
-		prefixed[logMapFromField], prefixed[logMapToField] = true, true
-	}
-	incFrom, incFromOk := e.Data[logIncFromField].(string)
-	incTo, incToOk := e.Data[logIncToField].(string)
-	if incFromOk && incToOk {
-		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
-		prefixed[logIncFromField], prefixed[logIncToField] = true, true
-	}
-
-	fmt.Fprintf(&line, ": %s", e.Message)
-
-	for field, value := range e.Data {
-
-		if prefixed[field] {
-			continue
-		}
-
-		if strings.ContainsAny(field, " \t") {
-			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
-		}
-		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
-	}
-
-	fmt.Fprintf(&line, "\n")
-
-	return line.Bytes(), nil
-}
-
-type JSONFormatter struct{}
-
-func (f JSONFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	data := make(logrus.Fields, len(e.Data)+3)
-	for k, v := range e.Data {
-		switch v := v.(type) {
-		case error:
-			// Otherwise errors are ignored by `encoding/json`
-			// https://github.com/sirupsen/logrus/issues/137
-			data[k] = v.Error()
-		default:
-			_, err := json.Marshal(v)
-			if err != nil {
-				return nil, errors.Errorf("field is not JSON encodable: %s", k)
-			}
-			data[k] = v
-		}
-	}
-
-	data["msg"] = e.Message
-	data["time"] = e.Time.Format(time.RFC3339)
-	data["level"] = e.Level
-
-	return json.Marshal(data)
-
-}
-
-type TCPHook struct {
-	Formatter     logrus.Formatter
-	MinLevel      logrus.Level
-	Net, Address  string
-	RetryInterval time.Duration
-	conn          net.Conn
-	retry         time.Time
-}
-
-func (h *TCPHook) Levels() []logrus.Level {
-	for i := range logrus.AllLevels { // assume it's ordered
-		if logrus.AllLevels[i] == h.MinLevel {
-			return logrus.AllLevels[:i]
-		}
-	}
-	return logrus.AllLevels
-}
-
-func (h *TCPHook) Fire(e *logrus.Entry) error {
-	b, err := h.Formatter.Format(e)
-	if err != nil {
-		return err
-	}
-
-	if h.conn == nil {
-		if time.Now().Sub(h.retry) < h.RetryInterval {
-			return errors.New("TCP hook reconnect prohibited by retry interval")
-		}
-		h.conn, err = net.Dial(h.Net, h.Address)
-		if err != nil {
-			h.retry = time.Now()
-			return errors.Wrap(err, "cannot dial")
-		}
-	}
-
-	_, err = h.conn.Write(b)
-	if err != nil {
-		h.conn.Close()
-		h.conn = nil
-	}
-
-	return nil
-}
-
-type nopWriter int
-
-func (w nopWriter) Write(p []byte) (n int, err error) { return len(p), nil }
@@ -11,8 +11,8 @@
 package cmd

 import (
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
 )

 //
@@ -20,7 +20,7 @@ import (
 //	Printf(format string, v ...interface{})
 //}

-type Logger logrus.FieldLogger
+type Logger = *logger.Logger

 var RootCmd = &cobra.Command{
 	Use:   "zrepl",
@@ -10,9 +10,10 @@ import (
 	"strings"

 	"github.com/kr/pretty"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
+	"time"
 )

 var testCmd = &cobra.Command{
@@ -64,8 +65,9 @@ func init() {

 func testCmdGlobalInit(cmd *cobra.Command, args []string) {

-	log := logrus.New()
-	log.Formatter = NoFormatter{}
+	out := logger.NewOutlets()
+	out.Add(WriterOutlet{NoFormatter{}, os.Stdout}, logger.Info)
+	log := logger.NewLogger(out, 1*time.Second)
 	testCmdGlobal.log = log

 	var err error