Add JOBS tests to the Makefile and GitHub CI.

Signed-off-by: Valery Piashchynski <[email protected]>

7 files changed, 610 insertions, 5 deletions

diff --git a/tests/plugins/jobs/configs/.rr-jobs-init-no-amqp-global.yaml b/tests/plugins/jobs/configs/.rr-jobs-init-no-amqp-global.yaml
new file mode 100644
index 00000000..0d141b2b
--- /dev/null
+++ b/tests/plugins/jobs/configs/.rr-jobs-init-no-amqp-global.yaml
@@ -0,0 +1,75 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+logs:
+  level: error
+  mode: development
+
+jobs:
+  # num logical cores by default
+  num_pollers: 10
+  # 1mi by default
+  pipeline_size: 100000
+  # worker pool configuration
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  # list of broker pipelines associated with endpoints
+  pipelines:
+    test-local:
+      driver: ephemeral
+      priority: 10
+      pipeline_size: 10000
+
+    test-local-2:
+      driver: ephemeral
+      priority: 1
+      pipeline_size: 10000
+
+    test-local-3:
+      driver: ephemeral
+      priority: 2
+      pipeline_size: 10000
+
+    test-1:
+      driver: amqp
+      priority: 1
+      pipeline_size: 1000000
+      queue: test-1-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test
+
+    test-2-amqp:
+      driver: amqp
+      priority: 2
+      pipeline_size: 100000
+      queue: test-2-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test-2
+
+    test-2:
+      driver: beanstalk
+      priority: 11
+      tube: default
+      pipeline_size: 1000000
+
+    test-3:
+      #      priority: 11 - not defined, 10 by default
+      # driver locality not specified, local by default
+      driver: sqs
+      pipeline_size: 1000000
+      queue: default
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-local", "test-local-2", "test-local-3", "test-1", "test-2-amqp" ]
+
diff --git a/tests/plugins/jobs/configs/.rr-jobs-init.yaml b/tests/plugins/jobs/configs/.rr-jobs-init.yaml
index 6efc760b..80826acc 100644
--- a/tests/plugins/jobs/configs/.rr-jobs-init.yaml
+++ b/tests/plugins/jobs/configs/.rr-jobs-init.yaml
@@ -1,5 +1,5 @@
 rpc:
-  listen: unix:///tmp/rr.sock
+  listen: tcp://127.0.0.1:6001
 
 server:
   command: "php ../../client.php echo pipes"
@@ -23,12 +23,13 @@ sqs:
     MessageRetentionPeriod: 86400
 
 logs:
-  level: debug
+  level: info
+  encoding: console
   mode: development
 
 jobs:
   # num logical cores by default
-  num_pollers: 64
+  num_pollers: 10
   # 1mi by default
   pipeline_size: 100000
   # worker pool configuration
diff --git a/tests/plugins/jobs/configs/.rr-jobs-list.yaml b/tests/plugins/jobs/configs/.rr-jobs-list.yaml
new file mode 100644
index 00000000..3d22a098
--- /dev/null
+++ b/tests/plugins/jobs/configs/.rr-jobs-list.yaml
@@ -0,0 +1,91 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+amqp:
+  addr: amqp://guest:guest@localhost:5672/
+
+  # beanstalk configuration
+beanstalk:
+  addr: tcp://localhost:11300
+
+  # amazon sqs configuration
+sqs:
+  key: api-key
+  secret: api-secret
+  region: us-west-1
+  endpoint: http://localhost:9324
+  declare:
+    MessageRetentionPeriod: 86400
+
+logs:
+  level: debug
+  mode: development
+
+jobs:
+  # num logical cores by default
+  num_pollers: 10
+  # 1mi by default
+  pipeline_size: 100000
+  # worker pool configuration
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  # list of broker pipelines associated with endpoints
+  pipelines:
+    test-local:
+      driver: ephemeral
+      priority: 10
+      pipeline_size: 10000
+
+    test-local-2:
+      driver: ephemeral
+      priority: 1
+      pipeline_size: 10000
+
+    test-local-3:
+      driver: ephemeral
+      priority: 2
+      pipeline_size: 10000
+
+    test-1:
+      driver: amqp
+      priority: 1
+      pipeline_size: 1000000
+      queue: test-1-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test
+
+    test-2-amqp:
+      driver: amqp
+      priority: 2
+      pipeline_size: 100000
+      queue: test-2-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test-2
+
+    test-2:
+      driver: beanstalk
+      priority: 11
+      tube: default
+      pipeline_size: 1000000
+
+    test-3:
+      #      priority: 11 - not defined, 10 by default
+      # driver locality not specified, local by default
+      driver: sqs
+      pipeline_size: 1000000
+      queue: default
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-local", "test-local-2", "test-local-3", "test-1", "test-2-amqp" ]
+
diff --git a/tests/plugins/jobs/configs/.rr-jobs-pause-resume-all.yaml b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-all.yaml
new file mode 100644
index 00000000..8789e872
--- /dev/null
+++ b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-all.yaml
@@ -0,0 +1,78 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+amqp:
+  addr: amqp://guest:guest@localhost:5672/
+
+logs:
+  level: debug
+  mode: development
+
+jobs:
+  # num logical cores by default
+  num_pollers: 10
+  # 1mi by default
+  pipeline_size: 100000
+  # worker pool configuration
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  # list of broker pipelines associated with endpoints
+  pipelines:
+    test-local:
+      driver: ephemeral
+      priority: 10
+      pipeline_size: 10000
+
+    test-local-2:
+      driver: ephemeral
+      priority: 1
+      pipeline_size: 10000
+
+    test-local-3:
+      driver: ephemeral
+      priority: 2
+      pipeline_size: 10000
+
+    test-1:
+      driver: amqp
+      priority: 1
+      pipeline_size: 1000000
+      queue: test-1-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test
+
+    test-2-amqp:
+      driver: amqp
+      priority: 2
+      pipeline_size: 100000
+      queue: test-2-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test-2
+
+    test-2:
+      driver: beanstalk
+      priority: 11
+      tube: default
+      pipeline_size: 1000000
+
+    test-3:
+      #      priority: 11 - not defined, 10 by default
+      # driver locality not specified, local by default
+      driver: sqs
+      pipeline_size: 1000000
+      queue: default
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-local", "test-local-2", "test-local-3", "test-1", "test-2-amqp" ]
+
diff --git a/tests/plugins/jobs/configs/.rr-jobs-pause-resume-amqp.yaml b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-amqp.yaml
new file mode 100644
index 00000000..8789e872
--- /dev/null
+++ b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-amqp.yaml
@@ -0,0 +1,78 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+amqp:
+  addr: amqp://guest:guest@localhost:5672/
+
+logs:
+  level: debug
+  mode: development
+
+jobs:
+  # num logical cores by default
+  num_pollers: 10
+  # 1mi by default
+  pipeline_size: 100000
+  # worker pool configuration
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  # list of broker pipelines associated with endpoints
+  pipelines:
+    test-local:
+      driver: ephemeral
+      priority: 10
+      pipeline_size: 10000
+
+    test-local-2:
+      driver: ephemeral
+      priority: 1
+      pipeline_size: 10000
+
+    test-local-3:
+      driver: ephemeral
+      priority: 2
+      pipeline_size: 10000
+
+    test-1:
+      driver: amqp
+      priority: 1
+      pipeline_size: 1000000
+      queue: test-1-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test
+
+    test-2-amqp:
+      driver: amqp
+      priority: 2
+      pipeline_size: 100000
+      queue: test-2-queue
+      exchange: default
+      exchange_type: direct
+      routing_key: test-2
+
+    test-2:
+      driver: beanstalk
+      priority: 11
+      tube: default
+      pipeline_size: 1000000
+
+    test-3:
+      #      priority: 11 - not defined, 10 by default
+      # driver locality not specified, local by default
+      driver: sqs
+      pipeline_size: 1000000
+      queue: default
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-local", "test-local-2", "test-local-3", "test-1", "test-2-amqp" ]
+
diff --git a/tests/plugins/jobs/configs/.rr-jobs-pause-resume-ephemeral.yaml b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-ephemeral.yaml
new file mode 100644
index 00000000..dc5bc3a1
--- /dev/null
+++ b/tests/plugins/jobs/configs/.rr-jobs-pause-resume-ephemeral.yaml
@@ -0,0 +1,44 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+logs:
+  level: info
+  mode: development
+
+jobs:
+  # num logical cores by default
+  num_pollers: 10
+  # 1mi by default
+  pipeline_size: 100000
+  # worker pool configuration
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  # list of broker pipelines associated with endpoints
+  pipelines:
+    test-local:
+      driver: ephemeral
+      priority: 10
+      pipeline_size: 10000
+
+    test-local-2:
+      driver: ephemeral
+      priority: 1
+      pipeline_size: 10000
+
+    test-local-3:
+      driver: ephemeral
+      priority: 2
+      pipeline_size: 10000
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-local", "test-local-2" ]
+
diff --git a/tests/plugins/jobs/jobs_plugin_test.go b/tests/plugins/jobs/jobs_plugin_test.go
index 76b7b879..034ffc45 100644
--- a/tests/plugins/jobs/jobs_plugin_test.go
+++ b/tests/plugins/jobs/jobs_plugin_test.go
@@ -1,6 +1,8 @@
 package jobs
 
 import (
+	"net"
+	"net/rpc"
 	"os"
 	"os/signal"
 	"sync"
@@ -8,15 +10,22 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang/mock/gomock"
 	endure "github.com/spiral/endure/pkg/container"
+	goridgeRpc "github.com/spiral/goridge/v3/pkg/rpc"
 	"github.com/spiral/roadrunner/v2/plugins/config"
+	"github.com/spiral/roadrunner/v2/plugins/informer"
 	"github.com/spiral/roadrunner/v2/plugins/jobs"
 	"github.com/spiral/roadrunner/v2/plugins/jobs/brokers/amqp"
 	"github.com/spiral/roadrunner/v2/plugins/jobs/brokers/ephemeral"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
+	"github.com/spiral/roadrunner/v2/plugins/resetter"
 	rpcPlugin "github.com/spiral/roadrunner/v2/plugins/rpc"
 	"github.com/spiral/roadrunner/v2/plugins/server"
+	jobsv1beta "github.com/spiral/roadrunner/v2/proto/jobs/v1beta"
+	"github.com/spiral/roadrunner/v2/tests/mocks"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestJobsInit(t *testing.T) {
@@ -28,12 +37,114 @@ func TestJobsInit(t *testing.T) {
 		Prefix: "rr",
 	}
 
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	// general
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "services", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Info("driver ready", "pipeline", "test-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("driver ready", "pipeline", "test-3", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-local-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-2-amqp", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-local-3", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("driver initialized", "driver", "amqp", "start", gomock.Any()).Times(2)
+
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local-3", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-2-amqp", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("delivery channel closed, leaving the rabbit listener").Times(2)
+
+	err = cont.RegisterAll(
+		cfg,
+		&server.Plugin{},
+		&rpcPlugin.Plugin{},
+		mockLogger,
+		&jobs.Plugin{},
+		&resetter.Plugin{},
+		&informer.Plugin{},
+		&ephemeral.Plugin{},
+		&amqp.Plugin{},
+	)
+	assert.NoError(t, err)
+
+	err = cont.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch, err := cont.Serve()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	stopCh := make(chan struct{}, 1)
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case e := <-ch:
+				assert.Fail(t, "error", e.Error.Error())
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+			case <-sig:
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			case <-stopCh:
+				// timeout
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			}
+		}
+	}()
+
+	time.Sleep(time.Second * 3)
+	stopCh <- struct{}{}
+	wg.Wait()
+}
+
+func TestJobsNoAMQPGlobal(t *testing.T) {
+	cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel))
+	assert.NoError(t, err)
+
+	cfg := &config.Viper{
+		Path:   "configs/.rr-jobs-init-no-amqp-global.yaml",
+		Prefix: "rr",
+	}
+
 	err = cont.RegisterAll(
 		cfg,
 		&server.Plugin{},
 		&rpcPlugin.Plugin{},
 		&logger.ZapLogger{},
 		&jobs.Plugin{},
+		&resetter.Plugin{},
+		&informer.Plugin{},
 		&ephemeral.Plugin{},
 		&amqp.Plugin{},
 	)
@@ -44,6 +155,55 @@ func TestJobsInit(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	_, err = cont.Serve()
+	require.Error(t, err)
+}
+
+func TestJobsPauseResume(t *testing.T) {
+	cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel))
+	assert.NoError(t, err)
+
+	cfg := &config.Viper{
+		Path:   "configs/.rr-jobs-pause-resume-ephemeral.yaml",
+		Prefix: "rr",
+	}
+
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	// general
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "services", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-local-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Info("pipeline started", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local-3", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2)
+
+	mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	err = cont.RegisterAll(
+		cfg,
+		&server.Plugin{},
+		&rpcPlugin.Plugin{},
+		mockLogger,
+		&jobs.Plugin{},
+		&resetter.Plugin{},
+		&informer.Plugin{},
+		&ephemeral.Plugin{},
+	)
+
+	assert.NoError(t, err)
+
+	err = cont.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	ch, err := cont.Serve()
 	if err != nil {
 		t.Fatal(err)
@@ -84,9 +244,87 @@ func TestJobsInit(t *testing.T) {
 		}
 	}()
 
-	time.Sleep(time.Second * 1200)
+	time.Sleep(time.Second * 3)
 
-	stopCh <- struct{}{}
+	t.Run("ephemeralPause", ephemeralPause)
+	t.Run("pushToDisabledPipe", pushToDisabledPipe("test-local"))
+	t.Run("ephemeralResume", ephemeralResume)
+	t.Run("pushToEnabledPipe", pushToPipe("test-local"))
+
+	time.Sleep(time.Second * 1)
 
+	stopCh <- struct{}{}
 	wg.Wait()
 }
+
+func ephemeralPause(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6001")
+	assert.NoError(t, err)
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	pipe := &jobsv1beta.MaintenanceRequest{Pipelines: make([]string, 1)}
+	pipe.GetPipelines()[0] = "test-local"
+
+	er := &jobsv1beta.Empty{}
+	err = client.Call("jobs.Pause", pipe, er)
+	assert.NoError(t, err)
+}
+
+func ephemeralResume(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6001")
+	assert.NoError(t, err)
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	pipe := &jobsv1beta.MaintenanceRequest{Pipelines: make([]string, 1)}
+	pipe.GetPipelines()[0] = "test-local"
+
+	er := &jobsv1beta.Empty{}
+	err = client.Call("jobs.Resume", pipe, er)
+	assert.NoError(t, err)
+}
+
+func pushToDisabledPipe(pipeline string) func(t *testing.T) {
+	return func(t *testing.T) {
+		conn, err := net.Dial("tcp", "127.0.0.1:6001")
+		assert.NoError(t, err)
+		client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+		req := &jobsv1beta.PushRequest{Job: &jobsv1beta.Job{
+			Job:     "some/php/namespace",
+			Id:      "1",
+			Payload: `{"hello":"world"}`,
+			Headers: nil,
+			Options: &jobsv1beta.Options{
+				Priority: 1,
+				Pipeline: pipeline,
+			},
+		}}
+
+		er := &jobsv1beta.Empty{}
+		err = client.Call("jobs.Push", req, er)
+		assert.Error(t, err)
+	}
+}
+
+func pushToPipe(pipeline string) func(t *testing.T) {
+	return func(t *testing.T) {
+		conn, err := net.Dial("tcp", "127.0.0.1:6001")
+		assert.NoError(t, err)
+		client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+		req := &jobsv1beta.PushRequest{Job: &jobsv1beta.Job{
+			Job:     "some/php/namespace",
+			Id:      "1",
+			Payload: `{"hello":"world"}`,
+			Headers: nil,
+			Options: &jobsv1beta.Options{
+				Priority: 1,
+				Pipeline: pipeline,
+			},
+		}}
+
+		er := &jobsv1beta.Empty{}
+		err = client.Call("jobs.Push", req, er)
+		assert.NoError(t, err)
+	}
+}