Beanstalk initial durability test

Signed-off-by: Valery Piashchynski <[email protected]>

3 files changed, 169 insertions, 2 deletions

diff --git a/tests/plugins/jobs/beanstalk/.rr-beanstalk-init.yaml b/tests/plugins/jobs/beanstalk/.rr-beanstalk-init.yaml
index c73ed961..66cfd52e 100644
--- a/tests/plugins/jobs/beanstalk/.rr-beanstalk-init.yaml
+++ b/tests/plugins/jobs/beanstalk/.rr-beanstalk-init.yaml
@@ -7,9 +7,7 @@ server:
   relay_timeout: "20s"
 
 beanstalk:
-  # beanstalk address
   addr: tcp://localhost:11300
-  # connect timeout
   timeout: 10s
 
 logs:
diff --git a/tests/plugins/jobs/durability/.rr-beanstalk-durability-redial.yaml b/tests/plugins/jobs/durability/.rr-beanstalk-durability-redial.yaml
new file mode 100644
index 00000000..de794b24
--- /dev/null
+++ b/tests/plugins/jobs/durability/.rr-beanstalk-durability-redial.yaml
@@ -0,0 +1,45 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+beanstalk:
+  addr: tcp://localhost:11400
+  timeout: 10s
+
+logs:
+  level: debug
+  encoding: console
+  mode: development
+
+jobs:
+  num_pollers: 10
+  pipeline_size: 100000
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  pipelines:
+    test-1:
+      driver: beanstalk
+      priority: 11
+      tube_priority: 1
+      tube: default-1
+      reserve_timeout: 10s
+
+    test-2:
+      driver: beanstalk
+      priority: 11
+      tube_priority: 3
+      tube: default-2
+      reserve_timeout: 10s
+
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-1", "test-2" ]
+
diff --git a/tests/plugins/jobs/jobs_with_toxics_test.go b/tests/plugins/jobs/jobs_with_toxics_test.go
index e31ca57f..4e33ba5c 100644
--- a/tests/plugins/jobs/jobs_with_toxics_test.go
+++ b/tests/plugins/jobs/jobs_with_toxics_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/spiral/roadrunner/v2/plugins/informer"
 	"github.com/spiral/roadrunner/v2/plugins/jobs"
 	"github.com/spiral/roadrunner/v2/plugins/jobs/drivers/amqp"
+	"github.com/spiral/roadrunner/v2/plugins/jobs/drivers/beanstalk"
 	"github.com/spiral/roadrunner/v2/plugins/jobs/drivers/sqs"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
 	"github.com/spiral/roadrunner/v2/plugins/resetter"
@@ -256,3 +257,126 @@ func TestDurabilitySQS(t *testing.T) {
 	stopCh <- struct{}{}
 	wg.Wait()
 }
+
+func TestDurabilityBeanstalk(t *testing.T) {
+	client := toxiproxy.NewClient("localhost:8474")
+
+	_, err := client.CreateProxy("redial", "localhost:11400", "localhost:11300")
+	require.NoError(t, err)
+	defer deleteProxy("redial", t)
+
+	cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel))
+	require.NoError(t, err)
+
+	cfg := &config.Viper{
+		Path:   "durability/.rr-beanstalk-durability-redial.yaml",
+		Prefix: "rr",
+	}
+
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	// general
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "services", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("driver initialized", "driver", "amqp", "start", gomock.Any()).Times(4)
+
+	mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2)
+	mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2)
+
+	mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("delivery channel closed, leaving the rabbit listener").Times(4)
+
+	// redial errors
+	mockLogger.EXPECT().Warn("rabbitmq reconnecting, caused by", "error", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Error("pipeline error", "pipeline", "test-1", "error", gomock.Any(), "start", gomock.Any(), "elapsed", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Error("pipeline error", "pipeline", "test-2", "error", gomock.Any(), "start", gomock.Any(), "elapsed", gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Info("rabbitmq dial succeed. trying to redeclare queues and subscribers").AnyTimes()
+	mockLogger.EXPECT().Info("queues and subscribers redeclared successfully").AnyTimes()
+
+	err = cont.RegisterAll(
+		cfg,
+		&server.Plugin{},
+		&rpcPlugin.Plugin{},
+		// mockLogger,
+		&logger.ZapLogger{},
+		&jobs.Plugin{},
+		&resetter.Plugin{},
+		&informer.Plugin{},
+		&beanstalk.Plugin{},
+	)
+	require.NoError(t, err)
+
+	err = cont.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch, err := cont.Serve()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	stopCh := make(chan struct{}, 1)
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case e := <-ch:
+				assert.Fail(t, "error", e.Error.Error())
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+			case <-sig:
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			case <-stopCh:
+				// timeout
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			}
+		}
+	}()
+
+	time.Sleep(time.Second * 3)
+	disableProxy("redial", t)
+	time.Sleep(time.Second * 3)
+
+	go func() {
+		time.Sleep(time.Second * 2)
+		t.Run("PushPipelineWhileRedialing-1", pushToPipe("test-1"))
+		t.Run("PushPipelineWhileRedialing-2", pushToPipe("test-2"))
+	}()
+
+	time.Sleep(time.Second * 5)
+	enableProxy("redial", t)
+
+	t.Run("PushPipelineWhileRedialing-1", pushToPipe("test-1"))
+	t.Run("PushPipelineWhileRedialing-2", pushToPipe("test-2"))
+
+	time.Sleep(time.Second * 10)
+
+	stopCh <- struct{}{}
+	wg.Wait()
+}