1 files changed, 84 insertions, 40 deletions

diff --git a/service/watcher/watcher.go b/service/watcher/watcher.go
index 63dce3d5..08d477fa 100644
--- a/service/watcher/watcher.go
+++ b/service/watcher/watcher.go
@@ -8,11 +8,17 @@ import (
 )
 
 const (
-	// EventMaxTTL thrown when worker is removed due MaxTTL being reached. Context is roadrunner.WorkerError
-	EventMaxTTL = iota + 8000
-
 	// EventMaxMemory caused when worker consumes more memory than allowed.
-	EventMaxMemory
+	EventMaxMemory = iota + 8000
+
+	// EventMaxTTL thrown when worker is removed due TTL being reached. Context is roadrunner.WorkerError
+	EventMaxTTL
+
+	// EventMaxIdleTTL triggered when worker spends too much time at rest.
+	EventMaxIdleTTL
+
+	// EventMaxIdleTTL triggered when worker spends too much time doing the task (max_execution_time).
+	EventMaxExecTTL
 )
 
 // handles watcher events
@@ -20,76 +26,114 @@ type listener func(event int, ctx interface{})
 
 // defines the watcher behaviour
 type watcherConfig struct {
-	// MaxTTL defines maximum time worker is allowed to live.
-	MaxTTL time.Duration
-
 	// MaxMemory defines maximum amount of memory allowed for worker. In megabytes.
 	MaxMemory uint64
-}
 
-// Normalize watcher config and upscale the durations.
-func (c *watcherConfig) Normalize() error {
-	// Always use second based definition for time durations
-	if c.MaxTTL < time.Microsecond {
-		c.MaxTTL = time.Second * time.Duration(c.MaxTTL.Nanoseconds())
-	}
+	// TTL defines maximum time worker is allowed to live.
+	TTL int64
+
+	// MaxIdleTTL defines maximum duration worker can spend in idle mode.
+	MaxIdleTTL int64
 
-	return nil
+	// MaxExecTTL defines maximum lifetime per job.
+	MaxExecTTL int64
 }
 
 type watcher struct {
-	lsn      listener
-	interval time.Duration
-	cfg      *watcherConfig
-	stop     chan interface{}
+	lsn  listener
+	tick time.Duration
+	cfg  *watcherConfig
 
 	// list of workers which are currently working
-	//working map[*roadrunner.Worker]time.Time
+	sw *stateWatcher
+
+	stop chan interface{}
 }
 
 // watch the pool state
-func (watch *watcher) watch(p roadrunner.Pool) {
+func (wch *watcher) watch(p roadrunner.Pool) {
 	now := time.Now()
+
 	for _, w := range p.Workers() {
-		if watch.cfg.MaxTTL != 0 && now.Sub(w.Created) >= watch.cfg.MaxTTL {
-			err := fmt.Errorf("max TTL reached (%s)", watch.cfg.MaxTTL)
+		if w.State().Value() == roadrunner.StateInvalid {
+			// skip duplicate assessment
+			continue
+		}
+
+		s, err := util.WorkerState(w)
+		if err != nil {
+			continue
+		}
+
+		if wch.cfg.TTL != 0 && now.Sub(w.Created).Seconds() >= float64(wch.cfg.TTL) {
+			err := fmt.Errorf("max TTL reached (%vs)", wch.cfg.TTL)
 			if p.Remove(w, err) {
-				watch.report(EventMaxTTL, w, err)
+				wch.report(EventMaxTTL, w, err)
 			}
+			continue
 		}
 
-		state, err := util.WorkerState(w)
-		if err != nil {
+		if wch.cfg.MaxMemory != 0 && s.MemoryUsage >= wch.cfg.MaxMemory*1024*1024 {
+			err := fmt.Errorf("max allowed memory reached (%vMB)", wch.cfg.MaxMemory)
+			if p.Remove(w, err) {
+				wch.report(EventMaxMemory, w, err)
+			}
 			continue
 		}
 
-		if watch.cfg.MaxMemory != 0 && state.MemoryUsage >= watch.cfg.MaxMemory*1024*1024 {
-			err := fmt.Errorf("max allowed memory reached (%vMB)", watch.cfg.MaxMemory)
+		// watch the worker state changes
+		wch.sw.push(w)
+	}
+
+	wch.sw.sync(now)
+
+	if wch.cfg.MaxExecTTL != 0 {
+		for _, w := range wch.sw.find(
+			roadrunner.StateWorking,
+			now.Add(-time.Second*time.Duration(wch.cfg.MaxExecTTL)),
+		) {
+			err := fmt.Errorf("max exec time reached (%vs)", wch.cfg.MaxExecTTL)
+			if p.Remove(w, err) {
+				// brutally
+				go w.Kill()
+				wch.report(EventMaxExecTTL, w, err)
+			}
+		}
+	}
+
+	// locale workers which are in idle mode for too long
+	if wch.cfg.MaxIdleTTL != 0 {
+		for _, w := range wch.sw.find(
+			roadrunner.StateReady,
+			now.Add(-time.Second*time.Duration(wch.cfg.MaxIdleTTL)),
+		) {
+			err := fmt.Errorf("max idle time reached (%vs)", wch.cfg.MaxIdleTTL)
 			if p.Remove(w, err) {
-				watch.report(EventMaxMemory, w, err)
+				wch.report(EventMaxIdleTTL, w, err)
 			}
 		}
 	}
 }
 
 // throw watcher event
-func (watch *watcher) report(event int, worker *roadrunner.Worker, caused error) {
-	if watch.lsn != nil {
-		watch.lsn(event, roadrunner.WorkerError{Worker: worker, Caused: caused})
+func (wch *watcher) report(event int, worker *roadrunner.Worker, caused error) {
+	if wch.lsn != nil {
+		wch.lsn(event, roadrunner.WorkerError{Worker: worker, Caused: caused})
 	}
 }
 
 // Attach watcher to the pool
-func (watch *watcher) Attach(pool roadrunner.Pool) roadrunner.Watcher {
+func (wch *watcher) Attach(pool roadrunner.Pool) roadrunner.Watcher {
 	wp := &watcher{
-		interval: watch.interval,
-		lsn:      watch.lsn,
-		cfg:      watch.cfg,
-		stop:     make(chan interface{}),
+		tick: wch.tick,
+		lsn:  wch.lsn,
+		cfg:  wch.cfg,
+		sw:   newStateWatcher(),
+		stop: make(chan interface{}),
 	}
 
 	go func(wp *watcher, pool roadrunner.Pool) {
-		ticker := time.NewTicker(wp.interval)
+		ticker := time.NewTicker(wp.tick)
 		for {
 			select {
 			case <-ticker.C:
@@ -104,6 +148,6 @@ func (watch *watcher) Attach(pool roadrunner.Pool) roadrunner.Watcher {
 }
 
 // Detach watcher from the pool.
-func (watch *watcher) Detach() {
-	close(watch.stop)
+func (wch *watcher) Detach() {
+	close(wch.stop)
 }