- Update redis/memory pubsubs

- Rework internal message bus
- Add new tests for the broadcast plugin and include them into the GA

Signed-off-by: Valery Piashchynski <[email protected]>

6 files changed, 181 insertions, 236 deletions

diff --git a/plugins/broadcast/plugin.go b/plugins/broadcast/plugin.go
index 3b420a4b..04a4fb80 100644
--- a/plugins/broadcast/plugin.go
+++ b/plugins/broadcast/plugin.go
@@ -4,13 +4,12 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/google/uuid"
 	endure "github.com/spiral/endure/pkg/container"
 	"github.com/spiral/errors"
 	"github.com/spiral/roadrunner/v2/pkg/pubsub"
 	"github.com/spiral/roadrunner/v2/plugins/config"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
-	websocketsv1beta "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
-	"google.golang.org/protobuf/proto"
 )
 
 const (
@@ -55,78 +54,7 @@ func (p *Plugin) Init(cfg config.Configurer, log logger.Logger) error {
 }
 
 func (p *Plugin) Serve() chan error {
-	const op = errors.Op("broadcast_plugin_serve")
-	errCh := make(chan error, 1)
-
-	// iterate over config
-	for k, v := range p.cfg.Data {
-		if v == nil {
-			continue
-		}
-
-		// check type of the v
-		// should be a map[string]interface{}
-		switch t := v.(type) {
-		// correct type
-		case map[string]interface{}:
-			if _, ok := t[driver]; !ok {
-				errCh <- errors.E(op, errors.Errorf("could not find mandatory driver field in the %s storage", k))
-				return errCh
-			}
-		default:
-			errCh <- errors.E(op, errors.Str("wrong type detected in the configuration, please, check yaml indentation"))
-			return errCh
-		}
-
-		// config key for the particular sub-driver kv.memcached
-		configKey := fmt.Sprintf("%s.%s", PluginName, k)
-
-		switch v.(map[string]interface{})[driver] {
-		case memory:
-			if _, ok := p.constructors[memory]; !ok {
-				p.log.Warn("no memory drivers registered", "registered", p.publishers)
-				continue
-			}
-			ps, err := p.constructors[memory].PSConstruct(configKey)
-			if err != nil {
-				errCh <- errors.E(op, err)
-				return errCh
-			}
-
-			// save the pubsub
-			p.publishers[k] = ps
-		case redis:
-			if _, ok := p.constructors[redis]; !ok {
-				p.log.Warn("no redis drivers registered", "registered", p.publishers)
-				continue
-			}
-
-			// first - try local configuration
-			switch {
-			case p.cfgPlugin.Has(configKey):
-				ps, err := p.constructors[redis].PSConstruct(configKey)
-				if err != nil {
-					errCh <- errors.E(op, err)
-					return errCh
-				}
-
-				// save the pubsub
-				p.publishers[k] = ps
-			case p.cfgPlugin.Has(redis):
-				ps, err := p.constructors[redis].PSConstruct(configKey)
-				if err != nil {
-					errCh <- errors.E(op, err)
-					return errCh
-				}
-
-				// save the pubsub
-				p.publishers[k] = ps
-				continue
-			}
-		}
-	}
-
-	return errCh
+	return make(chan error)
 }
 
 func (p *Plugin) Stop() error {
@@ -140,61 +68,49 @@ func (p *Plugin) Collects() []interface{} {
 }
 
 // CollectPublishers collect all plugins who implement pubsub.Publisher interface
-func (p *Plugin) CollectPublishers(name endure.Named, subscriber pubsub.Constructor) {
+func (p *Plugin) CollectPublishers(name endure.Named, constructor pubsub.Constructor) {
 	// key redis, value - interface
-	p.constructors[name.Name()] = subscriber
+	p.constructors[name.Name()] = constructor
 }
 
 // Publish is an entry point to the websocket PUBSUB
-func (p *Plugin) Publish(m []byte) error {
+func (p *Plugin) Publish(m *pubsub.Message) error {
 	p.Lock()
 	defer p.Unlock()
 
 	const op = errors.Op("broadcast_plugin_publish")
 
-	msg := &websocketsv1beta.Message{}
-	err := proto.Unmarshal(m, msg)
-	if err != nil {
-		return errors.E(op, err)
-	}
-
-	// Get payload
-	for i := 0; i < len(msg.GetTopics()); i++ {
-		if len(p.publishers) > 0 {
-			for j := range p.publishers {
-				err = p.publishers[j].Publish(m)
-				if err != nil {
-					return errors.E(op, err)
-				}
+	// check if any publisher registered
+	if len(p.publishers) > 0 {
+		for j := range p.publishers {
+			err := p.publishers[j].Publish(m)
+			if err != nil {
+				return errors.E(op, err)
 			}
-
-			return nil
 		}
-
+		return nil
+	} else {
 		p.log.Warn("no publishers registered")
 	}
 
 	return nil
 }
 
-func (p *Plugin) PublishAsync(m []byte) {
+func (p *Plugin) PublishAsync(m *pubsub.Message) {
 	go func() {
 		p.Lock()
 		defer p.Unlock()
-		msg := &websocketsv1beta.Message{}
-		err := proto.Unmarshal(m, msg)
-		if err != nil {
-			p.log.Error("message unmarshal")
-		}
-
-		// Get payload
-		for i := 0; i < len(msg.GetTopics()); i++ {
-			if len(p.publishers) > 0 {
-				for j := range p.publishers {
-					p.publishers[j].PublishAsync(m)
+		// check if any publisher registered
+		if len(p.publishers) > 0 {
+			for j := range p.publishers {
+				err := p.publishers[j].Publish(m)
+				if err != nil {
+					p.log.Error("publishAsync", "error", err)
+					// continue publish to other registered publishers
+					continue
 				}
-				return
 			}
+		} else {
 			p.log.Warn("no publishers registered")
 		}
 	}()
@@ -202,10 +118,67 @@ func (p *Plugin) PublishAsync(m []byte) {
 
 func (p *Plugin) GetDriver(key string) (pubsub.SubReader, error) {
 	const op = errors.Op("broadcast_plugin_get_driver")
-	// key - driver, default for example
-	// we should find `default` in the collected pubsubs constructors
-	if pub, ok := p.publishers[key]; ok {
-		return pub, nil
+
+	// choose a driver
+	if val, ok := p.cfg.Data[key]; ok {
+		// check type of the v
+		// should be a map[string]interface{}
+		switch t := val.(type) {
+		// correct type
+		case map[string]interface{}:
+			if _, ok := t[driver]; !ok {
+				panic(errors.E(op, errors.Errorf("could not find mandatory driver field in the %s storage", val)))
+			}
+		default:
+			return nil, errors.E(op, errors.Str("wrong type detected in the configuration, please, check yaml indentation"))
+		}
+
+		// config key for the particular sub-driver kv.memcached
+		configKey := fmt.Sprintf("%s.%s", PluginName, key)
+
+		switch val.(map[string]interface{})[driver] {
+		case memory:
+			if _, ok := p.constructors[memory]; !ok {
+				return nil, errors.E(op, errors.Errorf("no memory drivers registered, registered: %s", p.publishers))
+			}
+			ps, err := p.constructors[memory].PSConstruct(configKey)
+			if err != nil {
+				return nil, errors.E(op, err)
+			}
+
+			// save the initialized publisher channel
+			// for the in-memory, register new publishers
+			p.publishers[uuid.NewString()] = ps
+
+			return ps, nil
+		case redis:
+			if _, ok := p.constructors[redis]; !ok {
+				return nil, errors.E(op, errors.Errorf("no redis drivers registered, registered: %s", p.publishers))
+			}
+
+			// first - try local configuration
+			switch {
+			case p.cfgPlugin.Has(configKey):
+				ps, err := p.constructors[redis].PSConstruct(configKey)
+				if err != nil {
+					return nil, errors.E(op, err)
+				}
+
+				// save the pubsub under a config key
+				//
+				p.publishers[configKey] = ps
+				return ps, nil
+			case p.cfgPlugin.Has(redis):
+				ps, err := p.constructors[redis].PSConstruct(configKey)
+				if err != nil {
+					return nil, errors.E(op, err)
+				}
+
+				// save the pubsub
+				p.publishers[configKey] = ps
+				return ps, nil
+			}
+		}
 	}
 	return nil, errors.E(op, errors.Str("could not find driver by provided key"))
 }
diff --git a/plugins/broadcast/rpc.go b/plugins/broadcast/rpc.go
index 4c27cdc3..2ee211f8 100644
--- a/plugins/broadcast/rpc.go
+++ b/plugins/broadcast/rpc.go
@@ -2,9 +2,9 @@ package broadcast
 
 import (
 	"github.com/spiral/errors"
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
 	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
-	"google.golang.org/protobuf/proto"
 )
 
 // rpc collectors struct
@@ -14,7 +14,7 @@ type rpc struct {
 }
 
 // Publish ... msg is a proto decoded payload
-// see: pkg/pubsub/message.fbs
+// see: root/proto
 func (r *rpc) Publish(in *websocketsv1.Request, out *websocketsv1.Response) error {
 	const op = errors.Op("broadcast_publish")
 
@@ -28,15 +28,23 @@ func (r *rpc) Publish(in *websocketsv1.Request, out *websocketsv1.Response) erro
 	msgLen := len(in.GetMessages())
 
 	for i := 0; i < msgLen; i++ {
-		bb, err := proto.Marshal(in.GetMessages()[i])
-		if err != nil {
-			return errors.E(op, err)
-		}
+		for j := 0; j < len(in.GetMessages()[i].GetTopics()); j++ {
+			if in.GetMessages()[i].GetTopics()[j] == "" {
+				r.log.Warn("message with empty topic, skipping")
+				// skip empty topics
+				continue
+			}
+
+			tmp := &pubsub.Message{
+				Topic:   in.GetMessages()[i].GetTopics()[j],
+				Payload: in.GetMessages()[i].GetPayload(),
+			}
 
-		err = r.plugin.Publish(bb)
-		if err != nil {
-			out.Ok = false
-			return errors.E(op, err)
+			err := r.plugin.Publish(tmp)
+			if err != nil {
+				out.Ok = false
+				return errors.E(op, err)
+			}
 		}
 	}
 
@@ -45,10 +53,8 @@ func (r *rpc) Publish(in *websocketsv1.Request, out *websocketsv1.Response) erro
 }
 
 // PublishAsync ...
-// see: pkg/pubsub/message.fbs
+// see: root/proto
 func (r *rpc) PublishAsync(in *websocketsv1.Request, out *websocketsv1.Response) error {
-	const op = errors.Op("publish_async")
-
 	// just return in case of nil message
 	if in == nil {
 		out.Ok = false
@@ -60,13 +66,20 @@ func (r *rpc) PublishAsync(in *websocketsv1.Request, out *websocketsv1.Response)
 	msgLen := len(in.GetMessages())
 
 	for i := 0; i < msgLen; i++ {
-		bb, err := proto.Marshal(in.GetMessages()[i])
-		if err != nil {
-			out.Ok = false
-			return errors.E(op, err)
-		}
+		for j := 0; j < len(in.GetMessages()[i].GetTopics()); j++ {
+			if in.GetMessages()[i].GetTopics()[j] == "" {
+				r.log.Warn("message with empty topic, skipping")
+				// skip empty topics
+				continue
+			}
+
+			tmp := &pubsub.Message{
+				Topic:   in.GetMessages()[i].GetTopics()[j],
+				Payload: in.GetMessages()[i].GetPayload(),
+			}
 
-		r.plugin.PublishAsync(bb)
+			r.plugin.PublishAsync(tmp)
+		}
 	}
 
 	out.Ok = true
diff --git a/plugins/memory/pubsub.go b/plugins/memory/pubsub.go
index 87638bd8..d027a8a5 100644
--- a/plugins/memory/pubsub.go
+++ b/plugins/memory/pubsub.go
@@ -6,14 +6,12 @@ import (
 	"github.com/spiral/roadrunner/v2/pkg/bst"
 	"github.com/spiral/roadrunner/v2/pkg/pubsub"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
-	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
-	"google.golang.org/protobuf/proto"
 )
 
 type PubSubDriver struct {
 	sync.RWMutex
 	// channel with the messages from the RPC
-	pushCh chan []byte
+	pushCh chan *pubsub.Message
 	// user-subscribed topics
 	storage bst.Storage
 	log     logger.Logger
@@ -21,21 +19,21 @@ type PubSubDriver struct {
 
 func NewPubSubDriver(log logger.Logger, _ string) (pubsub.PubSub, error) {
 	ps := &PubSubDriver{
-		pushCh:  make(chan []byte, 10),
+		pushCh:  make(chan *pubsub.Message, 10),
 		storage: bst.NewBST(),
 		log:     log,
 	}
 	return ps, nil
 }
 
-func (p *PubSubDriver) Publish(message []byte) error {
-	p.pushCh <- message
+func (p *PubSubDriver) Publish(msg *pubsub.Message) error {
+	p.pushCh <- msg
 	return nil
 }
 
-func (p *PubSubDriver) PublishAsync(message []byte) {
+func (p *PubSubDriver) PublishAsync(msg *pubsub.Message) {
 	go func() {
-		p.pushCh <- message
+		p.pushCh <- msg
 	}()
 }
 
@@ -67,7 +65,7 @@ func (p *PubSubDriver) Connections(topic string, res map[string]struct{}) {
 	}
 }
 
-func (p *PubSubDriver) Next() (*websocketsv1.Message, error) {
+func (p *PubSubDriver) Next() (*pubsub.Message, error) {
 	msg := <-p.pushCh
 	if msg == nil {
 		return nil, nil
@@ -76,20 +74,13 @@ func (p *PubSubDriver) Next() (*websocketsv1.Message, error) {
 	p.RLock()
 	defer p.RUnlock()
 
-	m := &websocketsv1.Message{}
-	err := proto.Unmarshal(msg, m)
-	if err != nil {
-		return nil, err
-	}
-
-	// push only messages, which are subscribed
+	// push only messages, which topics are subscibed
 	// TODO better???
-	for i := 0; i < len(m.GetTopics()); i++ {
-		// if we have active subscribers - send a message to a topic
-		// or send nil instead
-		if ok := p.storage.Contains(m.GetTopics()[i]); ok {
-			return m, nil
-		}
+	// if we have active subscribers - send a message to a topic
+	// or send nil instead
+	if ok := p.storage.Contains(msg.Topic); ok {
+		return msg, nil
 	}
+
 	return nil, nil
 }
diff --git a/plugins/redis/fanin.go b/plugins/redis/fanin.go
index 0bdd4cf5..40a99d20 100644
--- a/plugins/redis/fanin.go
+++ b/plugins/redis/fanin.go
@@ -4,12 +4,10 @@ import (
 	"context"
 	"sync"
 
-	"github.com/spiral/roadrunner/v2/plugins/logger"
-	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
-	"google.golang.org/protobuf/proto"
-
 	"github.com/go-redis/redis/v8"
 	"github.com/spiral/errors"
+	"github.com/spiral/roadrunner/v2/pkg/pubsub"
+	"github.com/spiral/roadrunner/v2/plugins/logger"
 	"github.com/spiral/roadrunner/v2/utils"
 )
 
@@ -23,13 +21,13 @@ type FanIn struct {
 	log logger.Logger
 
 	// out channel with all subs
-	out chan *websocketsv1.Message
+	out chan *pubsub.Message
 
 	exit chan struct{}
 }
 
 func newFanIn(redisClient redis.UniversalClient, log logger.Logger) *FanIn {
-	out := make(chan *websocketsv1.Message, 100)
+	out := make(chan *pubsub.Message, 100)
 	fi := &FanIn{
 		out:    out,
 		client: redisClient,
@@ -67,14 +65,11 @@ func (fi *FanIn) read() {
 				return
 			}
 
-			m := &websocketsv1.Message{}
-			err := proto.Unmarshal(utils.AsBytes(msg.Payload), m)
-			if err != nil {
-				fi.log.Error("message unmarshal")
-				continue
+			fi.out <- &pubsub.Message{
+				Topic:   msg.Channel,
+				Payload: utils.AsBytes(msg.Payload),
 			}
 
-			fi.out <- m
 		case <-fi.exit:
 			return
 		}
@@ -97,6 +92,6 @@ func (fi *FanIn) stop() error {
 	return nil
 }
 
-func (fi *FanIn) consume() <-chan *websocketsv1.Message {
+func (fi *FanIn) consume() <-chan *pubsub.Message {
 	return fi.out
 }
diff --git a/plugins/redis/pubsub.go b/plugins/redis/pubsub.go
index 9c3d0134..6ab281f3 100644
--- a/plugins/redis/pubsub.go
+++ b/plugins/redis/pubsub.go
@@ -9,8 +9,6 @@ import (
 	"github.com/spiral/roadrunner/v2/pkg/pubsub"
 	"github.com/spiral/roadrunner/v2/plugins/config"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
-	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
-	"google.golang.org/protobuf/proto"
 )
 
 type PubSubDriver struct {
@@ -83,41 +81,26 @@ func (p *PubSubDriver) stop() {
 	}()
 }
 
-func (p *PubSubDriver) Publish(msg []byte) error {
+func (p *PubSubDriver) Publish(msg *pubsub.Message) error {
 	p.Lock()
 	defer p.Unlock()
 
-	m := &websocketsv1.Message{}
-	err := proto.Unmarshal(msg, m)
-	if err != nil {
-		return errors.E(err)
+	f := p.universalClient.Publish(context.Background(), msg.Topic, msg.Payload)
+	if f.Err() != nil {
+		return f.Err()
 	}
 
-	for j := 0; j < len(m.GetTopics()); j++ {
-		f := p.universalClient.Publish(context.Background(), m.GetTopics()[j], msg)
-		if f.Err() != nil {
-			return f.Err()
-		}
-	}
 	return nil
 }
 
-func (p *PubSubDriver) PublishAsync(msg []byte) {
+func (p *PubSubDriver) PublishAsync(msg *pubsub.Message) {
 	go func() {
 		p.Lock()
 		defer p.Unlock()
-		m := &websocketsv1.Message{}
-		err := proto.Unmarshal(msg, m)
-		if err != nil {
-			p.log.Error("message unmarshal error")
-			return
-		}
 
-		for j := 0; j < len(m.GetTopics()); j++ {
-			f := p.universalClient.Publish(context.Background(), m.GetTopics()[j], msg)
-			if f.Err() != nil {
-				p.log.Error("redis publish", "error", f.Err())
-			}
+		f := p.universalClient.Publish(context.Background(), msg.Topic, msg.Payload)
+		if f.Err() != nil {
+			p.log.Error("redis publish", "error", f.Err())
 		}
 	}()
 }
@@ -189,6 +172,6 @@ func (p *PubSubDriver) Connections(topic string, res map[string]struct{}) {
 }
 
 // Next return next message
-func (p *PubSubDriver) Next() (*websocketsv1.Message, error) {
+func (p *PubSubDriver) Next() (*pubsub.Message, error) {
 	return <-p.fanin.consume(), nil
 }
diff --git a/plugins/websockets/pool/workers_pool.go b/plugins/websockets/pool/workers_pool.go
index 3d95ede0..00e053ec 100644
--- a/plugins/websockets/pool/workers_pool.go
+++ b/plugins/websockets/pool/workers_pool.go
@@ -7,7 +7,6 @@ import (
 	"github.com/spiral/roadrunner/v2/pkg/pubsub"
 	"github.com/spiral/roadrunner/v2/plugins/logger"
 	"github.com/spiral/roadrunner/v2/plugins/websockets/connection"
-	websocketsv1 "github.com/spiral/roadrunner/v2/proto/websockets/v1beta"
 	"github.com/spiral/roadrunner/v2/utils"
 )
 
@@ -17,7 +16,7 @@ type WorkersPool struct {
 	resPool     sync.Pool
 	log         logger.Logger
 
-	queue chan *websocketsv1.Message
+	queue chan *pubsub.Message
 	exit  chan struct{}
 }
 
@@ -25,7 +24,7 @@ type WorkersPool struct {
 func NewWorkersPool(subscriber pubsub.Subscriber, connections *sync.Map, log logger.Logger) *WorkersPool {
 	wp := &WorkersPool{
 		connections: connections,
-		queue:       make(chan *websocketsv1.Message, 100),
+		queue:       make(chan *pubsub.Message, 100),
 		subscriber:  subscriber,
 		log:         log,
 		exit:        make(chan struct{}),
@@ -43,7 +42,7 @@ func NewWorkersPool(subscriber pubsub.Subscriber, connections *sync.Map, log log
 	return wp
 }
 
-func (wp *WorkersPool) Queue(msg *websocketsv1.Message) {
+func (wp *WorkersPool) Queue(msg *pubsub.Message) {
 	wp.queue <- msg
 }
 
@@ -83,57 +82,48 @@ func (wp *WorkersPool) do() { //nolint:gocognit
 					return
 				}
 				_ = msg
-				if msg == nil {
+				if msg == nil || msg.Topic == "" {
 					continue
 				}
-				if len(msg.GetTopics()) == 0 {
+
+				// get free map
+				res := wp.get()
+
+				// get connections for the particular topic
+				wp.subscriber.Connections(msg.Topic, res)
+
+				if len(res) == 0 {
+					wp.log.Info("no such topic", "topic", msg.Topic)
+					wp.put(res)
 					continue
 				}
 
-				// send a message to every topic
-				for i := 0; i < len(msg.GetTopics()); i++ {
-					// get free map
-					res := wp.get()
+				// res is a map with a connectionsID
+				for connID := range res {
+					c, ok := wp.connections.Load(connID)
+					if !ok {
+						wp.log.Warn("the websocket disconnected before the message being written to it", "topics", msg.Topic)
+						wp.put(res)
+						continue
+					}
 
-					// get connections for the particular topic
-					wp.subscriber.Connections(msg.GetTopics()[i], res)
+					d, err := json.Marshal(&Response{
+						Topic:   msg.Topic,
+						Payload: utils.AsString(msg.Payload),
+					})
 
-					if len(res) == 0 {
-						wp.log.Info("no such topic", "topic", msg.GetTopics()[i])
+					if err != nil {
+						wp.log.Error("error marshaling response", "error", err)
 						wp.put(res)
-						continue
+						break
 					}
 
-					// res is a map with a connectionsID
-					for connID := range res {
-						c, ok := wp.connections.Load(connID)
-						if !ok {
-							wp.log.Warn("the websocket disconnected before the message being written to it", "topics", msg.GetTopics()[i])
-							wp.put(res)
-							continue
-						}
-
-						response := &Response{
-							Topic:   msg.GetTopics()[i],
-							Payload: utils.AsString(msg.GetPayload()),
-						}
-
-						d, err := json.Marshal(response)
-						if err != nil {
-							wp.log.Error("error marshaling response", "error", err)
-							wp.put(res)
-							break
-						}
-
-						// put data into the bytes buffer
-						err = c.(*connection.Connection).Write(d)
-						if err != nil {
-							for i := 0; i < len(msg.GetTopics()); i++ {
-								wp.log.Error("error sending payload over the connection", "error", err, "topics", msg.GetTopics()[i])
-							}
-							wp.put(res)
-							continue
-						}
+					// put data into the bytes buffer
+					err = c.(*connection.Connection).Write(d)
+					if err != nil {
+						wp.log.Error("error sending payload over the connection", "error", err, "topic", msg.Topic)
+						wp.put(res)
+						continue
 					}
 				}
 			case <-wp.exit: