Add AMQP initial durability test

Signed-off-by: Valery Piashchynski <[email protected]>

7 files changed, 243 insertions, 1 deletions

diff --git a/go.mod b/go.mod
index b518c14a..33958617 100644
--- a/go.mod
+++ b/go.mod
@@ -3,13 +3,16 @@ module github.com/spiral/roadrunner/v2
 go 1.16
 
 require (
+	github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
 	github.com/alicebob/miniredis/v2 v2.14.5
+	// ========= AWS SDK v2
 	github.com/aws/aws-sdk-go-v2 v1.7.0
 	github.com/aws/aws-sdk-go-v2/config v1.4.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.3.0
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.6.0
 	github.com/aws/smithy-go v1.5.0
+	// =====================
 	github.com/beanstalkd/go-beanstalk v0.1.0
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/cenkalti/backoff/v4 v4.1.1
diff --git a/go.sum b/go.sum
index 0eca78cc..25783fcc 100644
--- a/go.sum
+++ b/go.sum
@@ -17,6 +17,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=
 github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
diff --git a/plugins/jobs/drivers/amqp/redial.go b/plugins/jobs/drivers/amqp/redial.go
index 532aadb4..47afdcb3 100644
--- a/plugins/jobs/drivers/amqp/redial.go
+++ b/plugins/jobs/drivers/amqp/redial.go
@@ -26,6 +26,7 @@ func (j *JobsConsumer) redialer() { //nolint:gocognit
 
 				t := time.Now()
 				pipe := j.pipeline.Load().(*pipeline.Pipeline)
+
 				j.eh.Push(events.JobEvent{
 					Event:    events.EventPipeError,
 					Pipeline: pipe.Name(),
@@ -34,7 +35,6 @@ func (j *JobsConsumer) redialer() { //nolint:gocognit
 					Start:    time.Now(),
 				})
 
-				j.log.Error("connection closed, reconnecting", "error", err)
 				expb := backoff.NewExponentialBackOff()
 				// set the retry timeout (minutes)
 				expb.MaxElapsedTime = j.retryTimeout
diff --git a/tests/env/docker-compose.yaml b/tests/env/docker-compose.yaml
index 67a927c4..2573fe7a 100644
--- a/tests/env/docker-compose.yaml
+++ b/tests/env/docker-compose.yaml
@@ -14,6 +14,13 @@ services:
     ports:
       - "6378:6379"
 
+  toxicproxy:
+    image: shopify/toxiproxy
+#    ports:
+#      - "8474:8474"
+#      - "5673:5673"
+    network_mode: "host"
+
   beanstalk:
     build:
       context: .
diff --git a/tests/plugins/jobs/durability/.rr-amqp-durability-redial.yaml b/tests/plugins/jobs/durability/.rr-amqp-durability-redial.yaml
new file mode 100644
index 00000000..e530dbcd
--- /dev/null
+++ b/tests/plugins/jobs/durability/.rr-amqp-durability-redial.yaml
@@ -0,0 +1,54 @@
+rpc:
+  listen: tcp://127.0.0.1:6001
+
+server:
+  command: "php ../../client.php echo pipes"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+amqp:
+  addr: amqp://guest:guest@localhost:23679/
+
+logs:
+  level: info
+  encoding: console
+  mode: development
+
+jobs:
+  num_pollers: 10
+  pipeline_size: 100000
+  pool:
+    num_workers: 10
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+  pipelines:
+    test-1:
+      driver: amqp
+      prefetch: 100
+      queue: test-1-queue
+      priority: 1
+      exchange: default
+      exchange_type: direct
+      routing_key: test-1
+      exclusive: false
+      multiple_ack: false
+      requeue_on_fail: false
+
+    test-2:
+      driver: amqp
+      prefetch: 100
+      queue: test-2-queue
+      priority: 2
+      exchange: default
+      exchange_type: direct
+      routing_key: test-2
+      exclusive: false
+      multiple_ack: false
+      requeue_on_fail: false
+
+
+  # list of pipelines to be consumed by the server, keep empty if you want to start consuming manually
+  consume: [ "test-1", "test-2" ]
+
diff --git a/tests/plugins/jobs/helpers.go b/tests/plugins/jobs/helpers.go
index 5ef5f022..27b2d1e0 100644
--- a/tests/plugins/jobs/helpers.go
+++ b/tests/plugins/jobs/helpers.go
@@ -1,13 +1,16 @@
 package jobs
 
 import (
+	"bytes"
 	"net"
+	"net/http"
 	"net/rpc"
 	"testing"
 
 	goridgeRpc "github.com/spiral/goridge/v3/pkg/rpc"
 	jobsv1beta "github.com/spiral/roadrunner/v2/proto/jobs/v1beta"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func resumePipes(pipes ...string) func(t *testing.T) {
@@ -113,3 +116,43 @@ func destroyPipelines(pipes ...string) func(t *testing.T) {
 		assert.NoError(t, err)
 	}
 }
+
+func enableProxy(name string, t *testing.T) {
+	buf := new(bytes.Buffer)
+	buf.WriteString(`{"enabled":true}`)
+
+	resp, err := http.Post("http://localhost:8474/proxies/"+name, "application/json", buf) //nolint:noctx
+	require.NoError(t, err)
+	require.Equal(t, 200, resp.StatusCode)
+	if resp.Body != nil {
+		_ = resp.Body.Close()
+	}
+}
+
+func disableProxy(name string, t *testing.T) {
+	buf := new(bytes.Buffer)
+	buf.WriteString(`{"enabled":false}`)
+
+	resp, err := http.Post("http://localhost:8474/proxies/"+name, "application/json", buf) //nolint:noctx
+	require.NoError(t, err)
+	require.Equal(t, 200, resp.StatusCode)
+	if resp.Body != nil {
+		_ = resp.Body.Close()
+	}
+}
+
+func deleteProxy(name string, t *testing.T) {
+	client := &http.Client{}
+
+	req, err := http.NewRequest(http.MethodDelete, "http://localhost:8474/proxies/"+name, nil) //nolint:noctx
+	require.NoError(t, err)
+
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+
+	require.NoError(t, err)
+	require.Equal(t, 204, resp.StatusCode)
+	if resp.Body != nil {
+		_ = resp.Body.Close()
+	}
+}
diff --git a/tests/plugins/jobs/jobs_with_toxics_test.go b/tests/plugins/jobs/jobs_with_toxics_test.go
new file mode 100644
index 00000000..93452cc4
--- /dev/null
+++ b/tests/plugins/jobs/jobs_with_toxics_test.go
@@ -0,0 +1,134 @@
+package jobs
+
+import (
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	toxiproxy "github.com/Shopify/toxiproxy/client"
+	"github.com/golang/mock/gomock"
+	endure "github.com/spiral/endure/pkg/container"
+	"github.com/spiral/roadrunner/v2/plugins/config"
+	"github.com/spiral/roadrunner/v2/plugins/informer"
+	"github.com/spiral/roadrunner/v2/plugins/jobs"
+	"github.com/spiral/roadrunner/v2/plugins/jobs/drivers/amqp"
+	"github.com/spiral/roadrunner/v2/plugins/resetter"
+	rpcPlugin "github.com/spiral/roadrunner/v2/plugins/rpc"
+	"github.com/spiral/roadrunner/v2/plugins/server"
+	"github.com/spiral/roadrunner/v2/tests/mocks"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDurabilityAMQP(t *testing.T) {
+	client := toxiproxy.NewClient("localhost:8474")
+
+	_, err := client.CreateProxy("redial", "localhost:23679", "localhost:5672")
+	require.NoError(t, err)
+	defer deleteProxy("redial", t)
+
+	cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel))
+	require.NoError(t, err)
+
+	cfg := &config.Viper{
+		Path:   "durability/.rr-amqp-durability-redial.yaml",
+		Prefix: "rr",
+	}
+
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	// general
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "services", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("driver initialized", "driver", "amqp", "start", gomock.Any()).Times(4)
+
+	mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2)
+	mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2)
+
+	mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-1", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+	mockLogger.EXPECT().Warn("pipeline stopped", "pipeline", "test-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1)
+
+	mockLogger.EXPECT().Info("delivery channel closed, leaving the rabbit listener").Times(4)
+
+	// redial errors
+	mockLogger.EXPECT().Warn("rabbitmq reconnecting, caused by", "error", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Error("pipeline error", "pipeline", "test-1", "error", gomock.Any(), "start", gomock.Any(), "elapsed", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Error("pipeline error", "pipeline", "test-2", "error", gomock.Any(), "start", gomock.Any(), "elapsed", gomock.Any()).AnyTimes()
+
+	mockLogger.EXPECT().Info("rabbitmq dial succeed. trying to redeclare queues and subscribers").AnyTimes()
+	mockLogger.EXPECT().Info("queues and subscribers redeclared successfully").AnyTimes()
+
+	err = cont.RegisterAll(
+		cfg,
+		&server.Plugin{},
+		&rpcPlugin.Plugin{},
+		mockLogger,
+		&jobs.Plugin{},
+		&resetter.Plugin{},
+		&informer.Plugin{},
+		&amqp.Plugin{},
+	)
+	require.NoError(t, err)
+
+	err = cont.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch, err := cont.Serve()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	stopCh := make(chan struct{}, 1)
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case e := <-ch:
+				assert.Fail(t, "error", e.Error.Error())
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+			case <-sig:
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			case <-stopCh:
+				// timeout
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			}
+		}
+	}()
+
+	time.Sleep(time.Second * 3)
+	disableProxy("redial", t)
+	time.Sleep(time.Second * 5)
+	enableProxy("redial", t)
+	time.Sleep(time.Second * 3)
+
+	stopCh <- struct{}{}
+	wg.Wait()
+}