WIP rewrite the daemon

cmd subdir does not build on purpose, it's only left in tree to grab old code and move it to github.com/zrepl/zrepl/daemon
2018-08-27 22:21:45 +02:00
parent df6e1bc64d
commit c69ebd3806
55 changed files with 1133 additions and 84 deletions
@@ -0,0 +1,142 @@
+package daemon
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/cmd/daemon/job"
+	"github.com/zrepl/zrepl/cmd/helpers"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/version"
+	"io"
+	"net"
+	"net/http"
+)
+
+type controlJob struct {
+	sockaddr *net.UnixAddr
+	jobs     *jobs
+}
+
+func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
+	j = &controlJob{jobs: jobs}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *controlJob) Name() string { return jobNameControl }
+
+func (j *controlJob) Status() interface{} { return nil }
+
+const (
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
+)
+
+func (j *controlJob) Run(ctx context.Context) {
+
+	log := job.GetLogger(ctx)
+	defer log.Info("control job finished")
+
+	l, err := helpers.ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	pprofServer := NewPProfServer(ctx)
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
+		var msg PprofServerControlMsg
+		err := json.NewDecoder(r.Body).Decode(&msg)
+		if err != nil {
+			log.WithError(err).Error("bad pprof request from client")
+			w.WriteHeader(http.StatusBadRequest)
+		}
+		pprofServer.Control(msg)
+		w.WriteHeader(200)
+	}})
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			return version.NewZreplVersionInformation(), nil
+		}}})
+	mux.Handle(ControlJobEndpointStatus,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			s := j.jobs.status()
+			return s, nil
+		}}})
+	server := http.Server{Handler: mux}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			server.Shutdown(context.Background())
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type jsonResponder struct {
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	res, err := j.producer()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+	} else {
+		io.Copy(w, &buf)
+	}
+}
+
+type requestLogger struct {
+	log         logger.Logger
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Info("start")
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
+	log.Info("finish")
+}
@@ -0,0 +1,174 @@
+package daemon
+
+import (
+	"context"
+	"github.com/zrepl/zrepl/logger"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+	"github.com/zrepl/zrepl/version"
+	"fmt"
+)
+
+.daesdfadsfsafjlsjfda
+
+import (
+	"context"
+	"fmt"
+	"github.com/zrepl/zrepl/cmd/daemon/job"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/version"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+func Run(ctx context.Context, controlSockpath string, outlets *logger.Outlets, confJobs []job.Job) {
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		cancel()
+	}()
+
+	log := logger.NewLogger(outlets, 1*time.Second)
+	log.Info(version.NewZreplVersionInformation().String())
+
+	// parse config
+	for _, job := range confJobs {
+		if IsInternalJobName(job.Name()) {
+			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
+		}
+	}
+
+	ctx = job.WithLogger(ctx, log)
+
+	jobs := newJobs()
+
+	// start control socket
+	controlJob, err := newControlJob(controlSockpath, jobs)
+	if err != nil {
+		panic(err) // FIXME
+	}
+	jobs.start(ctx, controlJob, true)
+
+	// start prometheus
+	//var promJob *prometheusJob // FIXME
+	//jobs.start(ctx, promJob, true)
+
+	log.Info("starting daemon")
+
+	// start regular jobs
+	for _, j := range confJobs {
+		jobs.start(ctx, j, false)
+	}
+
+	select {
+	case <-jobs.wait():
+		log.Info("all jobs finished")
+	case <-ctx.Done():
+		log.WithError(ctx.Err()).Info("context finished")
+	}
+	log.Info("daemon exiting")
+}
+
+type jobs struct {
+	wg sync.WaitGroup
+
+	// m protects all fields below it
+	m       sync.RWMutex
+	wakeups map[string]job.WakeupChan // by JobName
+	jobs    map[string]job.Job
+}
+
+func newJobs() *jobs {
+	return &jobs{
+		wakeups: make(map[string]job.WakeupChan),
+		jobs:    make(map[string]job.Job),
+	}
+}
+
+const (
+	logJobField    string = "job"
+	logTaskField   string = "task"
+	logSubsysField string = "subsystem"
+)
+
+func (s *jobs) wait() <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		s.wg.Wait()
+	}()
+	return ch
+}
+
+func (s *jobs) status() map[string]interface{} {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	type res struct {
+		name   string
+		status interface{}
+	}
+	var wg sync.WaitGroup
+	c := make(chan res, len(s.jobs))
+	for name, j := range s.jobs {
+		wg.Add(1)
+		go func(name string, j job.Job) {
+			defer wg.Done()
+			c <- res{name: name, status: j.Status()}
+		}(name, j)
+	}
+	wg.Wait()
+	close(c)
+	ret := make(map[string]interface{}, len(s.jobs))
+	for res := range c {
+		ret[res.name] = res.status
+	}
+	return ret
+}
+
+const (
+	jobNamePrometheus = "_prometheus"
+	jobNameControl    = "_control"
+)
+
+func IsInternalJobName(s string) bool {
+	return strings.HasPrefix(s, "_")
+}
+
+func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	jobLog := job.GetLogger(ctx).WithField(logJobField, j.Name())
+	jobName := j.Name()
+	if !internal && IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
+	}
+	if internal && !IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
+	}
+	if _, ok := s.jobs[jobName]; ok {
+		panic(fmt.Sprintf("duplicate job name %s", jobName))
+	}
+	s.jobs[jobName] = j
+	ctx = job.WithLogger(ctx, jobLog)
+	ctx, wakeupChan := job.WithWakeup(ctx)
+	s.wakeups[jobName] = wakeupChan
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		jobLog.Info("starting job")
+		defer jobLog.Info("job exited")
+		j.Run(ctx)
+	}()
+}
@@ -0,0 +1,47 @@
+package job
+
+import (
+	"context"
+	"github.com/zrepl/zrepl/logger"
+)
+
+type Logger = logger.Logger
+
+type contextKey int
+
+const (
+	contextKeyLog contextKey = iota
+	contextKeyWakeup
+)
+
+func GetLogger(ctx context.Context) Logger {
+	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
+		return l
+	}
+	return logger.NewNullLogger()
+}
+
+func WithLogger(ctx context.Context, l Logger) context.Context {
+	return context.WithValue(ctx, contextKeyLog, l)
+}
+
+func WithWakeup(ctx context.Context) (context.Context, WakeupChan) {
+	wc := make(chan struct{}, 1)
+	return context.WithValue(ctx, contextKeyWakeup, wc), wc
+}
+
+type Job interface {
+	Name() string
+	Run(ctx context.Context)
+	Status() interface{}
+}
+
+type WakeupChan <-chan struct{}
+
+func WaitWakeup(ctx context.Context) WakeupChan {
+	wc, ok := ctx.Value(contextKeyWakeup).(WakeupChan)
+	if !ok {
+		wc = make(chan struct{})
+	}
+	return wc
+}
@@ -0,0 +1,80 @@
+package daemon
+
+import (
+	"net/http"
+	// FIXME: importing this package has the side-effect of poisoning the http.DefaultServeMux
+	// FIXME: with the /debug/pprof endpoints
+	"context"
+	"net"
+	"net/http/pprof"
+)
+
+type PProfServer struct {
+	cc       chan PprofServerControlMsg
+	state    PprofServerControlMsg
+	listener net.Listener
+}
+
+type PprofServerControlMsg struct {
+	// Whether the server should listen for requests on the given address
+	Run bool
+	// Must be set if Run is true, undefined otherwise
+	HttpListenAddress string
+}
+
+func NewPProfServer(ctx context.Context) *PProfServer {
+
+	s := &PProfServer{
+		cc: make(chan PprofServerControlMsg),
+	}
+
+	go s.controlLoop(ctx)
+	return s
+}
+
+func (s *PProfServer) controlLoop(ctx context.Context) {
+outer:
+	for {
+
+		var msg PprofServerControlMsg
+		select {
+		case <-ctx.Done():
+			if s.listener != nil {
+				s.listener.Close()
+			}
+			break outer
+		case msg = <-s.cc:
+			// proceed
+		}
+
+		var err error
+		if msg.Run && s.listener == nil {
+
+			s.listener, err = net.Listen("tcp", msg.HttpListenAddress)
+			if err != nil {
+				s.listener = nil
+				continue
+			}
+
+			// FIXME: because net/http/pprof does not provide a mux,
+			mux := http.NewServeMux()
+			mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+			go http.Serve(s.listener, mux)
+			continue
+		}
+
+		if !msg.Run && s.listener != nil {
+			s.listener.Close()
+			s.listener = nil
+			continue
+		}
+	}
+}
+
+func (s *PProfServer) Control(msg PprofServerControlMsg) {
+	s.cc <- msg
+}
@@ -0,0 +1,82 @@
+package daemon
+
+import (
+	"context"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/zrepl/zrepl/cmd/daemon/job"
+	"github.com/zrepl/zrepl/zfs"
+	"net"
+	"net/http"
+)
+
+type prometheusJob struct {
+	listen string
+}
+
+func newPrometheusJob(listen string) *prometheusJob {
+	return &prometheusJob{listen}
+}
+
+var prom struct {
+	taskLastActiveStart    *prometheus.GaugeVec
+	taskLastActiveDuration *prometheus.GaugeVec
+	taskLogEntries         *prometheus.CounterVec
+}
+
+func init() {
+	prom.taskLastActiveStart = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "daemon",
+		Name:      "task_last_active_start",
+		Help:      "point in time at which the job task last left idle state",
+	}, []string{"zrepl_job", "job_type", "task"})
+	prom.taskLastActiveDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "daemon",
+		Name:      "task_last_active_duration",
+		Help:      "seconds that the last run ob a job task spent between leaving and re-entering idle state",
+	}, []string{"zrepl_job", "job_type", "task"})
+	prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "zrepl",
+		Subsystem: "daemon",
+		Name:      "task_log_entries",
+		Help:      "number of log entries per job task and level",
+	}, []string{"zrepl_job", "job_type", "task", "level"})
+	prometheus.MustRegister(prom.taskLastActiveStart)
+	prometheus.MustRegister(prom.taskLastActiveDuration)
+	prometheus.MustRegister(prom.taskLogEntries)
+}
+
+func (j *prometheusJob) Name() string { return jobNamePrometheus }
+
+func (j *prometheusJob) Status() interface{} { return nil }
+
+func (j *prometheusJob) Run(ctx context.Context) {
+
+	if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
+		panic(err)
+	}
+
+	log := job.GetLogger(ctx)
+
+	l, err := net.Listen("tcp", j.listen)
+	if err != nil {
+		log.WithError(err).Error("cannot listen")
+	}
+	go func() {
+		select {
+		case <-ctx.Done():
+			l.Close()
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+
+	err = http.Serve(l, mux)
+	if err != nil {
+		log.WithError(err).Error("error while serving")
+	}
+
+}