2 files changed, 29 insertions, 49 deletions

diff --git a/plugins/jobs/drivers/ephemeral/consumer.go b/plugins/jobs/drivers/ephemeral/consumer.go
index 03959b49..95ad6ecd 100644
--- a/plugins/jobs/drivers/ephemeral/consumer.go
+++ b/plugins/jobs/drivers/ephemeral/consumer.go
@@ -16,7 +16,8 @@ import (
 )
 
 const (
-	prefetch string = "prefetch"
+	prefetch      string = "prefetch"
+	goroutinesMax uint64 = 1000
 )
 
 type Config struct {
@@ -32,7 +33,7 @@ type JobConsumer struct {
 	localPrefetch chan *Item
 
 	// time.sleep goroutines max number
-	goroutinesMaxNum uint64
+	goroutines uint64
 
 	stopCh chan struct{}
 }
@@ -41,11 +42,11 @@ func NewJobBroker(configKey string, log logger.Logger, cfg config.Configurer, eh
 	const op = errors.Op("new_ephemeral_pipeline")
 
 	jb := &JobConsumer{
-		log:              log,
-		pq:               pq,
-		eh:               eh,
-		goroutinesMaxNum: 1000,
-		stopCh:           make(chan struct{}, 1),
+		log:        log,
+		pq:         pq,
+		eh:         eh,
+		goroutines: 0,
+		stopCh:     make(chan struct{}, 1),
 	}
 
 	err := cfg.UnmarshalKey(configKey, &jb.cfg)
@@ -68,11 +69,11 @@ func NewJobBroker(configKey string, log logger.Logger, cfg config.Configurer, eh
 
 func FromPipeline(pipeline *pipeline.Pipeline, log logger.Logger, eh events.Handler, pq priorityqueue.Queue) (*JobConsumer, error) {
 	jb := &JobConsumer{
-		log:              log,
-		pq:               pq,
-		eh:               eh,
-		goroutinesMaxNum: 1000,
-		stopCh:           make(chan struct{}, 1),
+		log:        log,
+		pq:         pq,
+		eh:         eh,
+		goroutines: 0,
+		stopCh:     make(chan struct{}, 1),
 	}
 
 	// initialize a local queue
@@ -112,18 +113,18 @@ func (j *JobConsumer) handleItem(ctx context.Context, msg *Item) error {
 	// goroutines here. We should limit goroutines here.
 	if msg.Options.Delay > 0 {
 		// if we have 1000 goroutines waiting on the delay - reject 1001
-		if atomic.LoadUint64(&j.goroutinesMaxNum) >= 1000 {
+		if atomic.LoadUint64(&j.goroutines) >= goroutinesMax {
 			return errors.E(op, errors.Str("max concurrency number reached"))
 		}
 
 		go func(jj *Item) {
-			atomic.AddUint64(&j.goroutinesMaxNum, 1)
+			atomic.AddUint64(&j.goroutines, 1)
 			time.Sleep(jj.Options.DelayDuration())
 
 			// send the item after timeout expired
 			j.localPrefetch <- jj
 
-			atomic.AddUint64(&j.goroutinesMaxNum, ^uint64(0))
+			atomic.AddUint64(&j.goroutines, ^uint64(0))
 		}(msg)
 
 		return nil
@@ -149,7 +150,8 @@ func (j *JobConsumer) consume() {
 			}
 
 			// set requeue channel
-			item.Options.requeueCh = j.localPrefetch
+			item.Options.requeueFn = j.handleItem
+
 			j.pq.Insert(item)
 		case <-j.stopCh:
 			return
diff --git a/plugins/jobs/drivers/ephemeral/item.go b/plugins/jobs/drivers/ephemeral/item.go
index 9fab8d24..1a61d7e9 100644
--- a/plugins/jobs/drivers/ephemeral/item.go
+++ b/plugins/jobs/drivers/ephemeral/item.go
@@ -1,6 +1,7 @@
 package ephemeral
 
 import (
+	"context"
 	"time"
 
 	json "github.com/json-iterator/go"
@@ -37,11 +38,8 @@ type Options struct {
 	// Delay defines time duration to delay execution for. Defaults to none.
 	Delay int64 `json:"delay,omitempty"`
 
-	// Timeout defines for how broker should wait until treating job are failed. Defaults to 30 min.
-	Timeout int64 `json:"timeout,omitempty"`
-
 	// private
-	requeueCh chan *Item
+	requeueFn func(context.Context, *Item) error
 }
 
 // DelayDuration returns delay duration in a form of time.Duration.
@@ -49,15 +47,6 @@ func (o *Options) DelayDuration() time.Duration {
 	return time.Second * time.Duration(o.Delay)
 }
 
-// TimeoutDuration returns timeout duration in a form of time.Duration.
-func (o *Options) TimeoutDuration() time.Duration {
-	if o.Timeout == 0 {
-		return 30 * time.Minute
-	}
-
-	return time.Second * time.Duration(o.Timeout)
-}
-
 func (i *Item) ID() string {
 	return i.Ident
 }
@@ -78,9 +67,8 @@ func (i *Item) Context() ([]byte, error) {
 			ID       string              `json:"id"`
 			Job      string              `json:"job"`
 			Headers  map[string][]string `json:"headers"`
-			Timeout  int64               `json:"timeout"`
 			Pipeline string              `json:"pipeline"`
-		}{ID: i.Ident, Job: i.Job, Headers: i.Headers, Timeout: i.Options.Timeout, Pipeline: i.Options.Pipeline},
+		}{ID: i.Ident, Job: i.Job, Headers: i.Headers, Pipeline: i.Options.Pipeline},
 	)
 
 	if err != nil {
@@ -101,25 +89,16 @@ func (i *Item) Nack() error {
 }
 
 func (i *Item) Requeue(headers map[string][]string, delay int64) error {
-	go func() {
-		time.Sleep(time.Second * time.Duration(delay))
-		// overwrite the delay
-		i.Options.Delay = delay
-		i.Headers = headers
-		select {
-		case i.Options.requeueCh <- i:
-			return
-		default:
-			// TODO(rustatian): logs?
-			return
-		}
-	}()
+	// overwrite the delay
+	i.Options.Delay = delay
+	i.Headers = headers
 
-	return nil
-}
+	err := i.Options.requeueFn(context.Background(), i)
+	if err != nil {
+		return err
+	}
 
-func (i *Item) Recycle() {
-	i.Options = nil
+	return nil
 }
 
 func fromJob(job *job.Job) *Item {
@@ -131,7 +110,6 @@ func fromJob(job *job.Job) *Item {
 			Priority: job.Options.Priority,
 			Pipeline: job.Options.Pipeline,
 			Delay:    job.Options.Delay,
-			Timeout:  job.Options.Timeout,
 		},
 	}
 }