- Update redis/memory pubsubs

- Rework internal message bus
- Add new tests for the broadcast plugin and include them into the GA

Signed-off-by: Valery Piashchynski <[email protected]>

11 files changed, 634 insertions, 3 deletions

diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml
index 67d5476b..b6ba0f66 100644
--- a/tests/docker-compose.yaml
+++ b/tests/docker-compose.yaml
@@ -9,3 +9,7 @@ services:
     image: redis:6
     ports:
       - "6379:6379"
+  redis2:
+    image: redis:6
+    ports:
+      - "6378:6379"
diff --git a/tests/plugins/broadcast/broadcast_plugin_test.go b/tests/plugins/broadcast/broadcast_plugin_test.go
index ce1aed45..d6510058 100644
--- a/tests/plugins/broadcast/broadcast_plugin_test.go
+++ b/tests/plugins/broadcast/broadcast_plugin_test.go
@@ -1,13 +1,18 @@
 package broadcast
 
 import (
+	"net"
+	"net/rpc"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
 	"testing"
+	"time"
 
+	"github.com/golang/mock/gomock"
 	endure "github.com/spiral/endure/pkg/container"
+	goridgeRpc "github.com/spiral/goridge/v3/pkg/rpc"
 	"github.com/spiral/roadrunner/v2/plugins/broadcast"
 	"github.com/spiral/roadrunner/v2/plugins/config"
 	httpPlugin "github.com/spiral/roadrunner/v2/plugins/http"
@@ -17,6 +22,9 @@ import (
 	rpcPlugin "github.com/spiral/roadrunner/v2/plugins/rpc"
 	"github.com/spiral/roadrunner/v2/plugins/server"
 	"github.com/spiral/roadrunner/v2/plugins/websockets"
+	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
+	"github.com/spiral/roadrunner/v2/tests/mocks"
+	"github.com/spiral/roadrunner/v2/tests/plugins/broadcast/plugins"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -112,6 +120,8 @@ func TestBroadcastConfigError(t *testing.T) {
 		&websockets.Plugin{},
 		&httpPlugin.Plugin{},
 		&memory.Plugin{},
+
+		&plugins.Plugin1{},
 	)
 
 	assert.NoError(t, err)
@@ -134,11 +144,18 @@ func TestBroadcastNoConfig(t *testing.T) {
 		Prefix: "rr",
 	}
 
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "services", []string{}).MinTimes(1)
+
 	err = cont.RegisterAll(
 		cfg,
 		&broadcast.Plugin{},
 		&rpcPlugin.Plugin{},
-		&logger.ZapLogger{},
+		mockLogger,
 		&server.Plugin{},
 		&redis.Plugin{},
 		&websockets.Plugin{},
@@ -157,3 +174,183 @@ func TestBroadcastNoConfig(t *testing.T) {
 	_, err = cont.Serve()
 	assert.NoError(t, err)
 }
+
+func TestBroadcastSameSubscriber(t *testing.T) {
+	cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel))
+	assert.NoError(t, err)
+
+	cfg := &config.Viper{
+		Path:   "configs/.rr-broadcast-same-section.yaml",
+		Prefix: "rr",
+	}
+
+	controller := gomock.NewController(t)
+	mockLogger := mocks.NewMockLogger(controller)
+
+	mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes()
+	mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6002", "services", []string{"broadcast"}).MinTimes(1)
+	mockLogger.EXPECT().Debug("message published", "msg", gomock.Any()).MinTimes(1)
+
+	mockLogger.EXPECT().Info(`plugin1: {foo hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin1: {foo2 hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin1: {foo3 hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin2: {foo hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin3: {foo hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin4: {foo hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin5: {foo hello}`).Times(3)
+	mockLogger.EXPECT().Info(`plugin6: {foo hello}`).Times(3)
+
+	err = cont.RegisterAll(
+		cfg,
+		&broadcast.Plugin{},
+		&rpcPlugin.Plugin{},
+		mockLogger,
+		&server.Plugin{},
+		&redis.Plugin{},
+		&websockets.Plugin{},
+		&httpPlugin.Plugin{},
+		&memory.Plugin{},
+
+		// test - redis
+		// test2 - redis (port 6378)
+		// test3 - memory
+		// test4 - memory
+		&plugins.Plugin1{}, // foo, foo2, foo3 test
+		&plugins.Plugin2{}, // foo, test
+		&plugins.Plugin3{}, // foo, test2
+		&plugins.Plugin4{}, // foo, test3
+		&plugins.Plugin5{}, // foo, test4
+		&plugins.Plugin6{}, // foo, test3
+	)
+
+	assert.NoError(t, err)
+
+	err = cont.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch, err := cont.Serve()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	stopCh := make(chan struct{}, 1)
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case e := <-ch:
+				assert.Fail(t, "error", e.Error.Error())
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+			case <-sig:
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			case <-stopCh:
+				// timeout
+				err = cont.Stop()
+				if err != nil {
+					assert.FailNow(t, "error", err.Error())
+				}
+				return
+			}
+		}
+	}()
+
+	time.Sleep(time.Second * 2)
+
+	t.Run("PublishHelloFooFoo2Foo3", BroadcastPublishFooFoo2Foo3)
+	t.Run("PublishHelloFoo2", BroadcastPublishFoo2)
+	t.Run("PublishHelloFoo3", BroadcastPublishFoo3)
+	t.Run("PublishAsyncHelloFooFoo2Foo3", BroadcastPublishAsyncFooFoo2Foo3)
+
+	time.Sleep(time.Second * 4)
+	stopCh <- struct{}{}
+
+	wg.Wait()
+}
+
+func BroadcastPublishFooFoo2Foo3(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6002")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	ret := &websocketsv1.Response{}
+	err = client.Call("broadcast.Publish", makeMessage([]byte("hello"), "foo", "foo2", "foo3"), ret)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func BroadcastPublishFoo2(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6002")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	ret := &websocketsv1.Response{}
+	err = client.Call("broadcast.Publish", makeMessage([]byte("hello"), "foo"), ret)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+func BroadcastPublishFoo3(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6002")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	ret := &websocketsv1.Response{}
+	err = client.Call("broadcast.Publish", makeMessage([]byte("hello"), "foo3"), ret)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+func BroadcastPublishAsyncFooFoo2Foo3(t *testing.T) {
+	conn, err := net.Dial("tcp", "127.0.0.1:6002")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := rpc.NewClientWithCodec(goridgeRpc.NewClientCodec(conn))
+
+	ret := &websocketsv1.Response{}
+	err = client.Call("broadcast.PublishAsync", makeMessage([]byte("hello"), "foo", "foo2", "foo3"), ret)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func makeMessage(payload []byte, topics ...string) *websocketsv1.Request {
+	m := &websocketsv1.Request{
+		Messages: []*websocketsv1.Message{
+			{
+				Topics:  topics,
+				Payload: payload,
+			},
+		},
+	}
+
+	return m
+}
diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml
index b01dad1e..d8daa251 100644
--- a/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml
+++ b/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml
@@ -25,7 +25,7 @@ broadcast:
 
 logs:
   mode: development
-  level: error
+  level: debug
 
 endure:
   grace_period: 120s
diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-no-config.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-no-config.yaml
index d324284d..90790869 100644
--- a/tests/plugins/broadcast/configs/.rr-broadcast-no-config.yaml
+++ b/tests/plugins/broadcast/configs/.rr-broadcast-no-config.yaml
@@ -21,7 +21,7 @@ http:
 
 logs:
   mode: development
-  level: error
+  level: debug
 
 endure:
   grace_period: 120s
diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml
new file mode 100644
index 00000000..360e05e5
--- /dev/null
+++ b/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml
@@ -0,0 +1,43 @@
+rpc:
+  listen: tcp://127.0.0.1:6002
+
+server:
+  command: "php ../../psr-worker-bench.php"
+  user: ""
+  group: ""
+  relay: "pipes"
+  relay_timeout: "20s"
+
+http:
+  address: 127.0.0.1:21345
+  max_request_size: 1024
+  middleware: [ "websockets" ]
+  trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ]
+  pool:
+    num_workers: 2
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s
+
+broadcast:
+  test:
+    driver: redis
+    addrs:
+      - "localhost:6379"
+  test2:
+    driver: redis
+    addrs:
+      - "localhost:6378"
+  test3:
+    driver: memory
+  test4:
+    driver: memory
+
+logs:
+  mode: development
+  level: debug
+
+endure:
+  grace_period: 120s
+  print_graph: false
+  log_level: error
diff --git a/tests/plugins/broadcast/plugins/plugin1.go b/tests/plugins/broadcast/plugins/plugin1.go
new file mode 100644
index 00000000..d3b16256
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin1.go
@@ -0,0 +1,67 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin1Name = "plugin1"
+
+type Plugin1 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin1) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin1) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test")
+	if err != nil {
+		errCh <- err
+		return errCh
+	}
+
+	err = p.driver.Subscribe("1", "foo", "foo2", "foo3")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin1Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin1) Stop() error {
+	_ = p.driver.Unsubscribe("1", "foo")
+	_ = p.driver.Unsubscribe("1", "foo2")
+	_ = p.driver.Unsubscribe("1", "foo3")
+	return nil
+}
+
+func (p *Plugin1) Name() string {
+	return Plugin1Name
+}
diff --git a/tests/plugins/broadcast/plugins/plugin2.go b/tests/plugins/broadcast/plugins/plugin2.go
new file mode 100644
index 00000000..2bd819d2
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin2.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin2Name = "plugin2"
+
+type Plugin2 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin2) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin2) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test")
+	if err != nil {
+		panic(err)
+	}
+
+	err = p.driver.Subscribe("2", "foo")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin2Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin2) Stop() error {
+	_ = p.driver.Unsubscribe("2", "foo")
+	return nil
+}
+
+func (p *Plugin2) Name() string {
+	return Plugin2Name
+}
diff --git a/tests/plugins/broadcast/plugins/plugin3.go b/tests/plugins/broadcast/plugins/plugin3.go
new file mode 100644
index 00000000..ef926222
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin3.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin3Name = "plugin3"
+
+type Plugin3 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin3) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin3) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test2")
+	if err != nil {
+		panic(err)
+	}
+
+	err = p.driver.Subscribe("3", "foo")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin3Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin3) Stop() error {
+	_ = p.driver.Unsubscribe("3", "foo")
+	return nil
+}
+
+func (p *Plugin3) Name() string {
+	return Plugin3Name
+}
diff --git a/tests/plugins/broadcast/plugins/plugin4.go b/tests/plugins/broadcast/plugins/plugin4.go
new file mode 100644
index 00000000..c9b94777
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin4.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin4Name = "plugin4"
+
+type Plugin4 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin4) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin4) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test3")
+	if err != nil {
+		panic(err)
+	}
+
+	err = p.driver.Subscribe("4", "foo")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin4Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin4) Stop() error {
+	_ = p.driver.Unsubscribe("4", "foo")
+	return nil
+}
+
+func (p *Plugin4) Name() string {
+	return Plugin4Name
+}
diff --git a/tests/plugins/broadcast/plugins/plugin5.go b/tests/plugins/broadcast/plugins/plugin5.go
new file mode 100644
index 00000000..01562a8f
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin5.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin5Name = "plugin5"
+
+type Plugin5 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin5) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin5) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test4")
+	if err != nil {
+		panic(err)
+	}
+
+	err = p.driver.Subscribe("5", "foo")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin5Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin5) Stop() error {
+	_ = p.driver.Unsubscribe("5", "foo")
+	return nil
+}
+
+func (p *Plugin5) Name() string {
+	return Plugin5Name
+}
diff --git a/tests/plugins/broadcast/plugins/plugin6.go b/tests/plugins/broadcast/plugins/plugin6.go
new file mode 100644
index 00000000..76f2d6e8
--- /dev/null
+++ b/tests/plugins/broadcast/plugins/plugin6.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"fmt"
+
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/broadcast"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
+)
+
+const Plugin6Name = "plugin6"
+
+type Plugin6 struct {
+	log    logger.Logger
+	b      broadcast.Broadcaster
+	driver pubsub.SubReader
+}
+
+func (p *Plugin6) Init(log logger.Logger, b broadcast.Broadcaster) error {
+	p.log = log
+	p.b = b
+	return nil
+}
+
+func (p *Plugin6) Serve() chan error {
+	errCh := make(chan error, 1)
+
+	var err error
+	p.driver, err = p.b.GetDriver("test")
+	if err != nil {
+		panic(err)
+	}
+
+	err = p.driver.Subscribe("6", "foo")
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			msg, err := p.driver.Next()
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == nil {
+				continue
+			}
+
+			p.log.Info(fmt.Sprintf("%s: %s", Plugin6Name, *msg))
+		}
+	}()
+
+	return errCh
+}
+
+func (p *Plugin6) Stop() error {
+	_ = p.driver.Unsubscribe("6", "foo")
+	return nil
+}
+
+func (p *Plugin6) Name() string {
+	return Plugin6Name
+}