4 files changed, 305 insertions, 0 deletions
diff --git a/service/watcher/config.go b/service/watcher/config.go
new file mode 100644
index 00000000..74be517a
--- /dev/null
+++ b/service/watcher/config.go
@@ -0,0 +1,48 @@
+package watcher
+
+import (
+	"github.com/spiral/roadrunner"
+	"github.com/spiral/roadrunner/service"
+	"time"
+)
+
+// Configures set of Services.
+type Config struct {
+	// Interval defines the update duration for underlying watchers, default 1s.
+	Interval time.Duration
+
+	// Services declares list of services to be watched.
+	Services map[string]*watcherConfig
+}
+
+// Hydrate must populate Config values using given Config source. Must return error if Config is not valid.
+func (c *Config) Hydrate(cfg service.Config) error {
+	if err := cfg.Unmarshal(c); err != nil {
+		return err
+	}
+
+	// Always use second based definition for time durations
+	if c.Interval < time.Microsecond {
+		c.Interval = time.Second * time.Duration(c.Interval.Nanoseconds())
+	}
+
+	return nil
+}
+
+// InitDefaults sets missing values to their default values.
+func (c *Config) InitDefaults() error {
+	c.Interval = time.Second
+
+	return nil
+}
+
+// Watchers returns list of defined Services
+func (c *Config) Watchers(l listener) (watchers map[string]roadrunner.Watcher) {
+	watchers = make(map[string]roadrunner.Watcher)
+
+	for name, cfg := range c.Services {
+		watchers[name] = &watcher{lsn: l, tick: c.Interval, cfg: cfg}
+	}
+
+	return watchers
+}
diff --git a/service/watcher/service.go b/service/watcher/service.go
new file mode 100644
index 00000000..c81ff3f5
--- /dev/null
+++ b/service/watcher/service.go
@@ -0,0 +1,46 @@
+package watcher
+
+import (
+	"github.com/spiral/roadrunner"
+	"github.com/spiral/roadrunner/service"
+)
+
+// ID defines watcher service name.
+const ID = "watch"
+
+// Watchable defines the ability to attach roadrunner watcher.
+type Watchable interface {
+	// Watch attaches watcher to the service.
+	Watch(w roadrunner.Watcher)
+}
+
+// Services to watch the state of roadrunner service inside other services.
+type Service struct {
+	cfg  *Config
+	lsns []func(event int, ctx interface{})
+}
+
+// Init watcher service
+func (s *Service) Init(cfg *Config, c service.Container) (bool, error) {
+	// mount Services to designated services
+	for id, watcher := range cfg.Watchers(s.throw) {
+		svc, _ := c.Get(id)
+		if watchable, ok := svc.(Watchable); ok {
+			watchable.Watch(watcher)
+		}
+	}
+
+	return true, nil
+}
+
+// AddListener attaches server event watcher.
+func (s *Service) AddListener(l func(event int, ctx interface{})) {
+	s.lsns = append(s.lsns, l)
+}
+
+// throw handles service, server and pool events.
+func (s *Service) throw(event int, ctx interface{}) {
+	for _, l := range s.lsns {
+		l(event, ctx)
+	}
+}
diff --git a/service/watcher/state_watch.go b/service/watcher/state_watch.go
new file mode 100644
index 00000000..3090d15d
--- /dev/null
+++ b/service/watcher/state_watch.go
@@ -0,0 +1,58 @@
+package watcher
+
+import (
+	"github.com/spiral/roadrunner"
+	"time"
+)
+
+type stateWatcher struct {
+	prev map[*roadrunner.Worker]state
+	next map[*roadrunner.Worker]state
+}
+
+type state struct {
+	state    int64
+	numExecs int64
+	since    time.Time
+}
+
+func newStateWatcher() *stateWatcher {
+	return &stateWatcher{
+		prev: make(map[*roadrunner.Worker]state),
+		next: make(map[*roadrunner.Worker]state),
+	}
+}
+
+// add new worker to be watched
+func (sw *stateWatcher) push(w *roadrunner.Worker) {
+	sw.next[w] = state{state: w.State().Value(), numExecs: w.State().NumExecs()}
+}
+
+// update worker states.
+func (sw *stateWatcher) sync(t time.Time) {
+	for w := range sw.prev {
+		if _, ok := sw.next[w]; !ok {
+			delete(sw.prev, w)
+		}
+	}
+
+	for w, s := range sw.next {
+		ps, ok := sw.prev[w]
+		if !ok || ps.state != s.state || ps.numExecs != s.numExecs {
+			sw.prev[w] = state{state: s.state, numExecs: s.numExecs, since: t}
+		}
+
+		delete(sw.next, w)
+	}
+}
+
+// find all workers which spend given amount of time in a specific state.
+func (sw *stateWatcher) find(state int64, since time.Time) (workers []*roadrunner.Worker) {
+	for w, s := range sw.prev {
+		if s.state == state && s.since.Before(since) {
+			workers = append(workers, w)
+		}
+	}
+
+	return
+}
diff --git a/service/watcher/watcher.go b/service/watcher/watcher.go
new file mode 100644
index 00000000..08d477fa
--- /dev/null
+++ b/service/watcher/watcher.go
@@ -0,0 +1,153 @@
+package watcher
+
+import (
+	"fmt"
+	"github.com/spiral/roadrunner"
+	"github.com/spiral/roadrunner/util"
+	"time"
+)
+
+const (
+	// EventMaxMemory caused when worker consumes more memory than allowed.
+	EventMaxMemory = iota + 8000
+
+	// EventMaxTTL thrown when worker is removed due TTL being reached. Context is roadrunner.WorkerError
+	EventMaxTTL
+
+	// EventMaxIdleTTL triggered when worker spends too much time at rest.
+	EventMaxIdleTTL
+
+	// EventMaxIdleTTL triggered when worker spends too much time doing the task (max_execution_time).
+	EventMaxExecTTL
+)
+
+// handles watcher events
+type listener func(event int, ctx interface{})
+
+// defines the watcher behaviour
+type watcherConfig struct {
+	// MaxMemory defines maximum amount of memory allowed for worker. In megabytes.
+	MaxMemory uint64
+
+	// TTL defines maximum time worker is allowed to live.
+	TTL int64
+
+	// MaxIdleTTL defines maximum duration worker can spend in idle mode.
+	MaxIdleTTL int64
+
+	// MaxExecTTL defines maximum lifetime per job.
+	MaxExecTTL int64
+}
+
+type watcher struct {
+	lsn  listener
+	tick time.Duration
+	cfg  *watcherConfig
+
+	// list of workers which are currently working
+	sw *stateWatcher
+
+	stop chan interface{}
+}
+
+// watch the pool state
+func (wch *watcher) watch(p roadrunner.Pool) {
+	now := time.Now()
+
+	for _, w := range p.Workers() {
+		if w.State().Value() == roadrunner.StateInvalid {
+			// skip duplicate assessment
+			continue
+		}
+
+		s, err := util.WorkerState(w)
+		if err != nil {
+			continue
+		}
+
+		if wch.cfg.TTL != 0 && now.Sub(w.Created).Seconds() >= float64(wch.cfg.TTL) {
+			err := fmt.Errorf("max TTL reached (%vs)", wch.cfg.TTL)
+			if p.Remove(w, err) {
+				wch.report(EventMaxTTL, w, err)
+			}
+			continue
+		}
+
+		if wch.cfg.MaxMemory != 0 && s.MemoryUsage >= wch.cfg.MaxMemory*1024*1024 {
+			err := fmt.Errorf("max allowed memory reached (%vMB)", wch.cfg.MaxMemory)
+			if p.Remove(w, err) {
+				wch.report(EventMaxMemory, w, err)
+			}
+			continue
+		}
+
+		// watch the worker state changes
+		wch.sw.push(w)
+	}
+
+	wch.sw.sync(now)
+
+	if wch.cfg.MaxExecTTL != 0 {
+		for _, w := range wch.sw.find(
+			roadrunner.StateWorking,
+			now.Add(-time.Second*time.Duration(wch.cfg.MaxExecTTL)),
+		) {
+			err := fmt.Errorf("max exec time reached (%vs)", wch.cfg.MaxExecTTL)
+			if p.Remove(w, err) {
+				// brutally
+				go w.Kill()
+				wch.report(EventMaxExecTTL, w, err)
+			}
+		}
+	}
+
+	// locale workers which are in idle mode for too long
+	if wch.cfg.MaxIdleTTL != 0 {
+		for _, w := range wch.sw.find(
+			roadrunner.StateReady,
+			now.Add(-time.Second*time.Duration(wch.cfg.MaxIdleTTL)),
+		) {
+			err := fmt.Errorf("max idle time reached (%vs)", wch.cfg.MaxIdleTTL)
+			if p.Remove(w, err) {
+				wch.report(EventMaxIdleTTL, w, err)
+			}
+		}
+	}
+}
+
+// throw watcher event
+func (wch *watcher) report(event int, worker *roadrunner.Worker, caused error) {
+	if wch.lsn != nil {
+		wch.lsn(event, roadrunner.WorkerError{Worker: worker, Caused: caused})
+	}
+}
+
+// Attach watcher to the pool
+func (wch *watcher) Attach(pool roadrunner.Pool) roadrunner.Watcher {
+	wp := &watcher{
+		tick: wch.tick,
+		lsn:  wch.lsn,
+		cfg:  wch.cfg,
+		sw:   newStateWatcher(),
+		stop: make(chan interface{}),
+	}
+
+	go func(wp *watcher, pool roadrunner.Pool) {
+		ticker := time.NewTicker(wp.tick)
+		for {
+			select {
+			case <-ticker.C:
+				wp.watch(pool)
+			case <-wp.stop:
+				return
+			}
+		}
+	}(wp, pool)
+
+	return wp
+}
+
+// Detach watcher from the pool.
+func (wch *watcher) Detach() {
+	close(wch.stop)
+}