diff options
| author | Valery Piashchynski <[email protected]> | 2021-09-14 14:37:08 +0300 |
|---|---|---|
| committer | Valery Piashchynski <[email protected]> | 2021-09-14 14:37:08 +0300 |
| commit | 60c1be36e71315983f8aaed046bac3e22b7a6208 (patch) | |
| tree | d48c2f53f249bf3ef8d7458c1bce997363d9f7b6 | |
| parent | d9c463468985e0b5d814c54d7d1a1880ac18a177 (diff) | |
| parent | 5d2cd55ab522d4f1e65a833f91146444465a32ac (diff) | |
Merge remote-tracking branch 'origin' into plugin/grpc
45 files changed, 930 insertions, 709 deletions
diff --git a/.vscode/settings.json b/.vscode/settings.json index 03fa9b74..44b782ad 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,21 +1,28 @@ { "cSpell.words": [ "addrs", - "AMQP", - "extendee", - "Goridge", - "Idxs", - "Itestdata", - "jobsv", - "mdwr", - "Nyholm", + "amqp", + "amqpjobs", + "boltdb", + "codecov", + "golangci", + "gomemcache", + "goridge", + "hget", + "hset", + "INMEMORY", + "memcachedkv", + "memorykv", + "mexpire", + "mget", "prefetch", - "protobuf", - "protoc", - "protoimpl", + "proto", + "SETEX", + "shivammathur", + "srem", "stretchr", - "tcreds", - "tmpdir", + "unsub", + "Upgrader", "websockets", "websocketsv" ], diff --git a/CHANGELOG.md b/CHANGELOG.md index 897877d3..d820b981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,270 +1,320 @@ -CHANGELOG -========= +# CHANGELOG -v2.4.0 (02.09.2021) -------------------- +## v2.5.0 (-.-.2021) + +## ๐ BC: + +- ๐จ All drivers now uses new `config` key to handle local configuration. Involved plugins and drivers: +- `plugins`: broadcast, kv +- `drivers`: memory, redis, memcached, boltdb. + +## ATTENTION!!!, this is configuration BC release, please, update your configuration: + +### Old style: + +```yaml +broadcast: + default: + driver: memory + interval: 1 +``` + +### New style: + +```yaml +broadcast: + default: + driver: memory + config: {} <--------------- NEW +``` + +```yaml +kv: + memory-rr: + driver: memory + config: <--------------- NEW + interval: 1 + +kv: + memcached-rr: + driver: memcached + config: <--------------- NEW + addr: + - "127.0.0.1:11211" +``` + +## ๐ New: + +- โ๏ธ + +## ๐ฉน Fixes: + +- ๐ Fix: local and global configuration parsing + +## ๐ฆ Packages: + +- ๐ฆ + +## ๐ Summary: + +- RR Milestone [2.5.0]() +- RR-Binary Milestone [2.5.0]() + +## v2.4.1 (13.09.2021) + +## ๐ฉน Fixes: + +- ๐ Fix: bug with not-idempotent call to the `attributes.Init`. +- ๐ Fix: memory jobs driver behavior. Now memory driver starts consuming automatically if the user consumes the pipeline in the configuration. + +## v2.4.0 (02.09.2021) ## ๐ Internal BC: -- ๐จ Pool, worker interfaces: payload now passed and returned by the pointer. +- ๐จ Pool, worker interfaces: payload now passed and returned by the pointer. ## ๐ New: -- โ๏ธ Long-awaited, reworked `Jobs` plugin with pluggable drivers. Now you can allocate/destroy pipelines in the runtime. Drivers included in the initial release: `RabbitMQ (0-9-1)`, `SQS v2`, `beanstalk`, `memory` and local queue powered by the `boltdb`. [PR](https://github.com/spiral/roadrunner/pull/726) -- โ๏ธ Support for the IPv6 (`tcp|http(s)|empty [::]:port`, `tcp|http(s)|empty [::1]:port`, `tcp|http(s)|empty :// [0:0:0:0:0:0:0:1]:port`) for RPC, HTTP and other plugins. [RFC](https://datatracker.ietf.org/doc/html/rfc2732#section-2) -- โ๏ธ Support for the Docker images via GitHub packages. -- โ๏ธ Go 1.17 support for the all spiral packages. +- โ๏ธ Long-awaited, reworked `Jobs` plugin with pluggable drivers. Now you can allocate/destroy pipelines in the runtime. Drivers included in the initial release: `RabbitMQ (0-9-1)`, `SQS v2`, `beanstalk`, `memory` and local queue powered by the `boltdb`. [PR](https://github.com/spiral/roadrunner/pull/726) +- โ๏ธ Support for the IPv6 (`tcp|http(s)|empty [::]:port`, `tcp|http(s)|empty [::1]:port`, `tcp|http(s)|empty :// [0:0:0:0:0:0:0:1]:port`) for RPC, HTTP and other plugins. [RFC](https://datatracker.ietf.org/doc/html/rfc2732#section-2) +- โ๏ธ Support for the Docker images via GitHub packages. +- โ๏ธ Go 1.17 support for the all spiral packages. ## ๐ฉน Fixes: -- ๐ Fix: fixed bug with goroutines waiting on the internal worker's container channel, [issue](https://github.com/spiral/roadrunner/issues/750). -- ๐ Fix: RR become unresponsive when new workers failed to re-allocate, [issue](https://github.com/spiral/roadrunner/issues/772). -- ๐ Fix: add `debug` pool config key to the `.rr.yaml` configuration [reference](https://github.com/spiral/roadrunner-binary/issues/79). +- ๐ Fix: fixed bug with goroutines waiting on the internal worker's container channel, [issue](https://github.com/spiral/roadrunner/issues/750). +- ๐ Fix: RR become unresponsive when new workers failed to re-allocate, [issue](https://github.com/spiral/roadrunner/issues/772). +- ๐ Fix: add `debug` pool config key to the `.rr.yaml` configuration [reference](https://github.com/spiral/roadrunner-binary/issues/79). ## ๐ฆ Packages: -- ๐ฆ Update goridge to `v3.2.1` -- ๐ฆ Update temporal to `v1.0.9` -- ๐ฆ Update endure to `v1.0.4` +- ๐ฆ Update goridge to `v3.2.1` +- ๐ฆ Update temporal to `v1.0.9` +- ๐ฆ Update endure to `v1.0.4` ## ๐ Summary: -- RR Milestone [2.4.0](https://github.com/spiral/roadrunner/milestone/29?closed=1) -- RR-Binary Milestone [2.4.0](https://github.com/spiral/roadrunner-binary/milestone/10?closed=1) +- RR Milestone [2.4.0](https://github.com/spiral/roadrunner/milestone/29?closed=1) +- RR-Binary Milestone [2.4.0](https://github.com/spiral/roadrunner-binary/milestone/10?closed=1) --- -v2.3.2 (14.07.2021) -------------------- +## v2.3.2 (14.07.2021) ## ๐ฉน Fixes: -- ๐ Fix: Do not call the container's Stop method after the container stopped by an error. -- ๐ Fix: Bug with ttl incorrectly handled by the worker [PR](https://github.com/spiral/roadrunner/pull/749) -- ๐ Fix: Add `RR_BROADCAST_PATH` to the `websockets` plugin [PR](https://github.com/spiral/roadrunner/pull/749) +- ๐ Fix: Do not call the container's Stop method after the container stopped by an error. +- ๐ Fix: Bug with ttl incorrectly handled by the worker [PR](https://github.com/spiral/roadrunner/pull/749) +- ๐ Fix: Add `RR_BROADCAST_PATH` to the `websockets` plugin [PR](https://github.com/spiral/roadrunner/pull/749) ## ๐ Summary: -- RR Milestone [2.3.2](https://github.com/spiral/roadrunner/milestone/31?closed=1) +- RR Milestone [2.3.2](https://github.com/spiral/roadrunner/milestone/31?closed=1) --- -v2.3.1 (30.06.2021) -------------------- +## v2.3.1 (30.06.2021) ## ๐ New: -- โ๏ธ Rework `broadcast` plugin. Add architecture diagrams to the `doc` - folder. [PR](https://github.com/spiral/roadrunner/pull/732) -- โ๏ธ Add `Clear` method to the KV plugin RPC. [PR](https://github.com/spiral/roadrunner/pull/736) +- โ๏ธ Rework `broadcast` plugin. Add architecture diagrams to the `doc` + folder. [PR](https://github.com/spiral/roadrunner/pull/732) +- โ๏ธ Add `Clear` method to the KV plugin RPC. [PR](https://github.com/spiral/roadrunner/pull/736) ## ๐ฉน Fixes: -- ๐ Fix: Bug with channel deadlock when `exec_ttl` was used and TTL limit - reached [PR](https://github.com/spiral/roadrunner/pull/738) -- ๐ Fix: Bug with healthcheck endpoint when workers were marked as invalid and stay is that state until next - request [PR](https://github.com/spiral/roadrunner/pull/738) -- ๐ Fix: Bugs with `boltdb` storage: [Boom](https://github.com/spiral/roadrunner/issues/717) - , [Boom](https://github.com/spiral/roadrunner/issues/718), [Boom](https://github.com/spiral/roadrunner/issues/719) -- ๐ Fix: Bug with incorrect redis initialization and usage [Bug](https://github.com/spiral/roadrunner/issues/720) -- ๐ Fix: Bug, Goridge duplicate error messages [Bug](https://github.com/spiral/goridge/issues/128) -- ๐ Fix: Bug, incorrect request `origin` check [Bug](https://github.com/spiral/roadrunner/issues/727) +- ๐ Fix: Bug with channel deadlock when `exec_ttl` was used and TTL limit + reached [PR](https://github.com/spiral/roadrunner/pull/738) +- ๐ Fix: Bug with healthcheck endpoint when workers were marked as invalid and stay is that state until next + request [PR](https://github.com/spiral/roadrunner/pull/738) +- ๐ Fix: Bugs with `boltdb` storage: [Boom](https://github.com/spiral/roadrunner/issues/717) + , [Boom](https://github.com/spiral/roadrunner/issues/718), [Boom](https://github.com/spiral/roadrunner/issues/719) +- ๐ Fix: Bug with incorrect redis initialization and usage [Bug](https://github.com/spiral/roadrunner/issues/720) +- ๐ Fix: Bug, Goridge duplicate error messages [Bug](https://github.com/spiral/goridge/issues/128) +- ๐ Fix: Bug, incorrect request `origin` check [Bug](https://github.com/spiral/roadrunner/issues/727) ## ๐ฆ Packages: -- ๐ฆ Update goridge to `v3.1.4` -- ๐ฆ Update temporal to `v1.0.8` +- ๐ฆ Update goridge to `v3.1.4` +- ๐ฆ Update temporal to `v1.0.8` ## ๐ Summary: -- RR Milestone [2.3.1](https://github.com/spiral/roadrunner/milestone/30?closed=1) -- Temporal Milestone [1.0.8](https://github.com/temporalio/roadrunner-temporal/milestone/11?closed=1) -- Goridge Milestone [3.1.4](https://github.com/spiral/goridge/milestone/11?closed=1) +- RR Milestone [2.3.1](https://github.com/spiral/roadrunner/milestone/30?closed=1) +- Temporal Milestone [1.0.8](https://github.com/temporalio/roadrunner-temporal/milestone/11?closed=1) +- Goridge Milestone [3.1.4](https://github.com/spiral/goridge/milestone/11?closed=1) --- -v2.3.0 (08.06.2021) -------------------- +## v2.3.0 (08.06.2021) ## ๐ New: -- โ๏ธ Brand new `broadcast` plugin now has the name - `websockets` with broadcast capabilities. It can handle hundreds of - thousands websocket connections very efficiently (~300k messages per second with 1k connected clients, in-memory bus - on 2CPU cores and 1GB of RAM) [Issue](https://github.com/spiral/roadrunner/issues/513) -- โ๏ธ Protobuf binary messages for the `websockets` and `kv` RPC calls under the - hood. [Issue](https://github.com/spiral/roadrunner/issues/711) -- โ๏ธ Json-schemas for the config file v1.0 (it also registered - in [schemastore.org](https://github.com/SchemaStore/schemastore/pull/1614)) -- โ๏ธ `latest` docker image tag supported now (but we strongly recommend using a versioned tag (like `0.2.3`) instead) -- โ๏ธ Add new option to the `http` config section: `internal_error_code` to override default (500) internal error - code. [Issue](https://github.com/spiral/roadrunner/issues/659) -- โ๏ธ Expose HTTP plugin metrics (workers memory, requests count, requests duration) - . [Issue](https://github.com/spiral/roadrunner/issues/489) -- โ๏ธ Scan `server.command` and find errors related to the wrong path to a `PHP` file, or `.ph`, `.sh` - scripts. [Issue](https://github.com/spiral/roadrunner/issues/658) -- โ๏ธ Support file logger with log rotation [Wiki](https://en.wikipedia.org/wiki/Log_rotation) - , [Issue](https://github.com/spiral/roadrunner/issues/545) +- โ๏ธ Brand new `broadcast` plugin now has the name - `websockets` with broadcast capabilities. It can handle hundreds of + thousands websocket connections very efficiently (~300k messages per second with 1k connected clients, in-memory bus + on 2CPU cores and 1GB of RAM) [Issue](https://github.com/spiral/roadrunner/issues/513) +- โ๏ธ Protobuf binary messages for the `websockets` and `kv` RPC calls under the + hood. [Issue](https://github.com/spiral/roadrunner/issues/711) +- โ๏ธ Json-schemas for the config file v1.0 (it also registered + in [schemastore.org](https://github.com/SchemaStore/schemastore/pull/1614)) +- โ๏ธ `latest` docker image tag supported now (but we strongly recommend using a versioned tag (like `0.2.3`) instead) +- โ๏ธ Add new option to the `http` config section: `internal_error_code` to override default (500) internal error + code. [Issue](https://github.com/spiral/roadrunner/issues/659) +- โ๏ธ Expose HTTP plugin metrics (workers memory, requests count, requests duration) + . [Issue](https://github.com/spiral/roadrunner/issues/489) +- โ๏ธ Scan `server.command` and find errors related to the wrong path to a `PHP` file, or `.ph`, `.sh` + scripts. [Issue](https://github.com/spiral/roadrunner/issues/658) +- โ๏ธ Support file logger with log rotation [Wiki](https://en.wikipedia.org/wiki/Log_rotation) + , [Issue](https://github.com/spiral/roadrunner/issues/545) ## ๐ฉน Fixes: -- ๐ Fix: Bug with `informer.Workers` worked incorrectly: [Bug](https://github.com/spiral/roadrunner/issues/686) -- ๐ Fix: Internal error messages will not be shown to the user (except HTTP status code). Error message will be in - logs: [Bug](https://github.com/spiral/roadrunner/issues/659) -- ๐ Fix: Error message will be properly shown in the log in case of `SoftJob` - error: [Bug](https://github.com/spiral/roadrunner/issues/691) -- ๐ Fix: Wrong applied middlewares for the `fcgi` server leads to the - NPE: [Bug](https://github.com/spiral/roadrunner/issues/701) +- ๐ Fix: Bug with `informer.Workers` worked incorrectly: [Bug](https://github.com/spiral/roadrunner/issues/686) +- ๐ Fix: Internal error messages will not be shown to the user (except HTTP status code). Error message will be in + logs: [Bug](https://github.com/spiral/roadrunner/issues/659) +- ๐ Fix: Error message will be properly shown in the log in case of `SoftJob` + error: [Bug](https://github.com/spiral/roadrunner/issues/691) +- ๐ Fix: Wrong applied middlewares for the `fcgi` server leads to the + NPE: [Bug](https://github.com/spiral/roadrunner/issues/701) ## ๐ฆ Packages: -- ๐ฆ Update goridge to `v3.1.0` +- ๐ฆ Update goridge to `v3.1.0` --- -v2.2.1 (13.05.2021) -------------------- +## v2.2.1 (13.05.2021) ## ๐ฉน Fixes: -- ๐ Fix: revert static plugin. It stays as a separate plugin on the main route (`/`) and supports all the previously - announced features. -- ๐ Fix: remove `build` and other old targets from the Makefile. +- ๐ Fix: revert static plugin. It stays as a separate plugin on the main route (`/`) and supports all the previously + announced features. +- ๐ Fix: remove `build` and other old targets from the Makefile. --- -v2.2.0 (11.05.2021) -------------------- +## v2.2.0 (11.05.2021) ## ๐ New: -- โ๏ธ Reworked `static` plugin. Now, it does not affect the performance of the main route and persist on the separate - file server (within the `http` plugin). Looong awaited feature: `Etag` (+ weak Etags) as well with the `If-Mach` - , `If-None-Match`, `If-Range`, `Last-Modified` - and `If-Modified-Since` tags supported. Static plugin has a bunch of new options such as: `allow`, `calculate_etag` - , `weak` and `pattern`. - ### Option `always` was deleted from the plugin. +- โ๏ธ Reworked `static` plugin. Now, it does not affect the performance of the main route and persist on the separate + file server (within the `http` plugin). Looong awaited feature: `Etag` (+ weak Etags) as well with the `If-Mach` + , `If-None-Match`, `If-Range`, `Last-Modified` + and `If-Modified-Since` tags supported. Static plugin has a bunch of new options such as: `allow`, `calculate_etag` + , `weak` and `pattern`. + ### Option `always` was deleted from the plugin. -- โ๏ธ Update `informer.List` implementation. Now it returns a list with the all available plugins in the runtime. +- โ๏ธ Update `informer.List` implementation. Now it returns a list with the all available plugins in the runtime. ## ๐ฉน Fixes: -- ๐ Fix: issue with wrong ordered middlewares (reverse). Now the order is correct. -- ๐ Fix: issue when RR fails if a user sets `debug` mode with the `exec_ttl` supervisor option. -- ๐ Fix: uniform log levels. Use everywhere the same levels (warn, error, debug, info, panic). +- ๐ Fix: issue with wrong ordered middlewares (reverse). Now the order is correct. +- ๐ Fix: issue when RR fails if a user sets `debug` mode with the `exec_ttl` supervisor option. +- ๐ Fix: uniform log levels. Use everywhere the same levels (warn, error, debug, info, panic). --- -v2.1.1 (29.04.2021) -------------------- +## v2.1.1 (29.04.2021) ## ๐ฉน Fixes: -- ๐ Fix: issue with endure provided wrong logger interface implementation. +- ๐ Fix: issue with endure provided wrong logger interface implementation. -v2.1.0 (27.04.2021) -------------------- +## v2.1.0 (27.04.2021) ## ๐ New: -- โ๏ธ New `service` plugin. Docs: [link](https://roadrunner.dev/docs/beep-beep-service) -- โ๏ธ Stabilize `kv` plugin with `boltdb`, `in-memory`, `memcached` and `redis` drivers. +- โ๏ธ New `service` plugin. Docs: [link](https://roadrunner.dev/docs/beep-beep-service) +- โ๏ธ Stabilize `kv` plugin with `boltdb`, `in-memory`, `memcached` and `redis` drivers. ## ๐ฉน Fixes: -- ๐ Fix: Logger didn't provide an anonymous log instance to a plugins w/o `Named` interface implemented. -- ๐ Fix: http handler was without log listener after `rr reset`. +- ๐ Fix: Logger didn't provide an anonymous log instance to a plugins w/o `Named` interface implemented. +- ๐ Fix: http handler was without log listener after `rr reset`. -v2.0.4 (06.04.2021) -------------------- +## v2.0.4 (06.04.2021) ## ๐ New: -- โ๏ธ Add support for `linux/arm64` platform for docker image (thanks @tarampampam). -- โ๏ธ Add dotenv file support (`.env` in working directory by default; file location can be changed using CLI - flag `--dotenv` or `DOTENV_PATH` environment variable) (thanks @tarampampam). -- ๐ Add a new `raw` mode for the `logger` plugin to keep the stderr log message of the worker unmodified (logger - severity level should be at least `INFO`). -- ๐ Add Readiness probe check. The `status` plugin provides `/ready` endpoint which return the `204` HTTP code if there - are no workers in the `Ready` state and `200 OK` status if there are at least 1 worker in the `Ready` state. +- โ๏ธ Add support for `linux/arm64` platform for docker image (thanks @tarampampam). +- โ๏ธ Add dotenv file support (`.env` in working directory by default; file location can be changed using CLI + flag `--dotenv` or `DOTENV_PATH` environment variable) (thanks @tarampampam). +- ๐ Add a new `raw` mode for the `logger` plugin to keep the stderr log message of the worker unmodified (logger + severity level should be at least `INFO`). +- ๐ Add Readiness probe check. The `status` plugin provides `/ready` endpoint which return the `204` HTTP code if there + are no workers in the `Ready` state and `200 OK` status if there are at least 1 worker in the `Ready` state. ## ๐ฉน Fixes: -- ๐ Fix: bug with the temporal worker which does not follow general graceful shutdown period. +- ๐ Fix: bug with the temporal worker which does not follow general graceful shutdown period. -v2.0.3 (29.03.2021) -------------------- +## v2.0.3 (29.03.2021) ## ๐ฉน Fixes: -- ๐ Fix: slow last response when reached `max_jobs` limit. - -v2.0.2 (06.04.2021) -------------------- - -- ๐ Fix: Bug with required Root CA certificate for the SSL, now it's optional. -- ๐ Fix: Bug with incorrectly consuming metrics collector from the RPC calls (thanks @dstrop). -- ๐ New: HTTP/FCGI/HTTPS internal logs instead of going to the raw stdout will be displayed in the RR logger at - the `Info` log level. -- โก New: Builds for the Mac with the M1 processor (arm64). -- ๐ท Rework ServeHTTP handler logic. Use http.Error instead of writing code directly to the response writer. Other small - improvements. - -v2.0.1 (09.03.2021) -------------------- - -- ๐ Fix: incorrect PHP command validation -- ๐ Fix: ldflags properly inject RR version -- โฌ๏ธ Update: README, links to the go.pkg from v1 to v2 -- ๐ฆ Bump golang version in the Dockerfile and in the `go.mod` to 1.16 -- ๐ฆ Bump Endure container to v1.0.0. - -v2.0.0 (02.03.2021) -------------------- - -- โ๏ธ Add a shared server to create PHP worker pools instead of isolated worker pool in each individual plugin. -- ๐ New plugin system with auto-recovery, easier plugin API. -- ๐ New `logger` plugin to configure logging for each plugin individually. -- ๐ Up to 50% performance increase in HTTP workloads. -- โ๏ธ Add **[Temporal Workflow](https://temporal.io)** plugin to run distributed computations on scale. -- โ๏ธ Add `debug` flag to reload PHP worker ahead of a request (emulates PHP-FPM behavior). -- โ Eliminate `limit` service, now each worker pool includes `supervisor` configuration. -- ๐ New resetter, informer plugins to perform hot reloads and observe loggers in a system. -- ๐ซ Expose more HTTP plugin configuration options. -- ๐ Headers, static and gzip services now located in HTTP config. -- ๐ Ability to configure the middleware sequence. -- ๐ฃ Faster Goridge protocol (eliminated 50% of syscalls). -- ๐พ Add support for binary payloads for RPC (`msgpack`). -- ๐ Server no longer stops when a PHP worker dies (attempts to restart). -- ๐พ New RR binary server downloader. -- ๐ฃ Echoing no longer breaks execution (yay!). -- ๐ Migration to ZapLogger instead of Logrus. -- ๐ฅ RR can no longer stuck when studding down with broken tasks in a pipeline. -- ๐งช More tests, more static analysis. -- ๐ฅ Create a new foundation for new KV, WebSocket, GRPC and Queue plugins. - -v2.0.0-RC.4 (20.02.2021) -------------------- - -- PHP tests use latest signatures (https://github.com/spiral/roadrunner/pull/550). -- Endure container update to v1.0.0-RC.2 version. -- Remove unneeded mutex from the `http.Workers` method. -- Rename `checker` plugin package to `status`, remove `/v1` endpoint prefix (#557). -- Add static, headers, status, gzip plugins to the `main.go`. -- Fix workers pool behavior -> idle_ttl, ttl, max_memory are soft errors and exec_ttl is hard error. - -v2.0.0-RC.3 (17.02.2021) -------------------- - -- Add support for the overwriting `.rr.yaml` keys with values (ref: https://roadrunner.dev/docs/intro-config) -- Make logger plugin optional to define in the config. Default values: level -> `debug`, mode -> `development` -- Add the ability to read env variables from the `.rr.yaml` in the form of: `rpc.listen: {RPC_ADDR}`. Reference: - ref: https://roadrunner.dev/docs/intro-config (Environment Variables paragraph) - -v2.0.0-RC.2 (11.02.2021) -------------------- - -- Update RR to version v2.0.0-RC.2 -- Update Temporal plugin to version v2.0.0-RC.1 -- Update Goridge to version v3.0.1 -- Update Endure to version v1.0.0-RC.1 +- ๐ Fix: slow last response when reached `max_jobs` limit. + +## v2.0.2 (06.04.2021) + +- ๐ Fix: Bug with required Root CA certificate for the SSL, now it's optional. +- ๐ Fix: Bug with incorrectly consuming metrics collector from the RPC calls (thanks @dstrop). +- ๐ New: HTTP/FCGI/HTTPS internal logs instead of going to the raw stdout will be displayed in the RR logger at + the `Info` log level. +- โก New: Builds for the Mac with the M1 processor (arm64). +- ๐ท Rework ServeHTTP handler logic. Use http.Error instead of writing code directly to the response writer. Other small + improvements. + +## v2.0.1 (09.03.2021) + +- ๐ Fix: incorrect PHP command validation +- ๐ Fix: ldflags properly inject RR version +- โฌ๏ธ Update: README, links to the go.pkg from v1 to v2 +- ๐ฆ Bump golang version in the Dockerfile and in the `go.mod` to 1.16 +- ๐ฆ Bump Endure container to v1.0.0. + +## v2.0.0 (02.03.2021) + +- โ๏ธ Add a shared server to create PHP worker pools instead of isolated worker pool in each individual plugin. +- ๐ New plugin system with auto-recovery, easier plugin API. +- ๐ New `logger` plugin to configure logging for each plugin individually. +- ๐ Up to 50% performance increase in HTTP workloads. +- โ๏ธ Add **[Temporal Workflow](https://temporal.io)** plugin to run distributed computations on scale. +- โ๏ธ Add `debug` flag to reload PHP worker ahead of a request (emulates PHP-FPM behavior). +- โ Eliminate `limit` service, now each worker pool includes `supervisor` configuration. +- ๐ New resetter, informer plugins to perform hot reloads and observe loggers in a system. +- ๐ซ Expose more HTTP plugin configuration options. +- ๐ Headers, static and gzip services now located in HTTP config. +- ๐ Ability to configure the middleware sequence. +- ๐ฃ Faster Goridge protocol (eliminated 50% of syscalls). +- ๐พ Add support for binary payloads for RPC (`msgpack`). +- ๐ Server no longer stops when a PHP worker dies (attempts to restart). +- ๐พ New RR binary server downloader. +- ๐ฃ Echoing no longer breaks execution (yay!). +- ๐ Migration to ZapLogger instead of Logrus. +- ๐ฅ RR can no longer stuck when studding down with broken tasks in a pipeline. +- ๐งช More tests, more static analysis. +- ๐ฅ Create a new foundation for new KV, WebSocket, GRPC and Queue plugins. + +## v2.0.0-RC.4 (20.02.2021) + +- PHP tests use latest signatures (https://github.com/spiral/roadrunner/pull/550). +- Endure container update to v1.0.0-RC.2 version. +- Remove unneeded mutex from the `http.Workers` method. +- Rename `checker` plugin package to `status`, remove `/v1` endpoint prefix (#557). +- Add static, headers, status, gzip plugins to the `main.go`. +- Fix workers pool behavior -> idle_ttl, ttl, max_memory are soft errors and exec_ttl is hard error. + +## v2.0.0-RC.3 (17.02.2021) + +- Add support for the overwriting `.rr.yaml` keys with values (ref: https://roadrunner.dev/docs/intro-config) +- Make logger plugin optional to define in the config. Default values: level -> `debug`, mode -> `development` +- Add the ability to read env variables from the `.rr.yaml` in the form of: `rpc.listen: {RPC_ADDR}`. Reference: + ref: https://roadrunner.dev/docs/intro-config (Environment Variables paragraph) + +## v2.0.0-RC.2 (11.02.2021) + +- Update RR to version v2.0.0-RC.2 +- Update Temporal plugin to version v2.0.0-RC.1 +- Update Goridge to version v3.0.1 +- Update Endure to version v1.0.0-RC.1 @@ -8,38 +8,37 @@ test_coverage: docker-compose -f tests/env/docker-compose.yaml up -d --remove-orphans rm -rf coverage-ci mkdir ./coverage-ci - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipe.txt -covermode=atomic ./pkg/transport/pipe - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/socket.txt -covermode=atomic ./pkg/transport/socket - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pool.txt -covermode=atomic ./pkg/pool - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker.txt -covermode=atomic ./pkg/worker - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/bst.txt -covermode=atomic ./pkg/bst - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pq.txt -covermode=atomic ./pkg/priority_queue - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker_stack.txt -covermode=atomic ./pkg/worker_watcher - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/ws_origin.txt -covermode=atomic ./plugins/websockets - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/struct_jobs.txt -covermode=atomic ./plugins/jobs/job - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http_config.txt -covermode=atomic ./plugins/http/config - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server_cmd.txt -covermode=atomic ./plugins/server - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipeline_jobs.txt -covermode=atomic ./plugins/jobs/pipeline - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/grpc_plugin.txt -covermode=atomic ./plugins/grpc - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/kv_plugin.txt -covermode=atomic ./tests/plugins/kv - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/grpc_plugin.txt -covermode=atomic ./tests/plugins/grpc - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/broadcast_plugin.txt -covermode=atomic ./tests/plugins/broadcast - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/websockets.txt -covermode=atomic ./tests/plugins/websockets - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/jobs_core.txt -covermode=atomic ./tests/plugins/jobs - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http.txt -covermode=atomic ./tests/plugins/http - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/informer.txt -covermode=atomic ./tests/plugins/informer - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/reload.txt -covermode=atomic ./tests/plugins/reload - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server.txt -covermode=atomic ./tests/plugins/server - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/service.txt -covermode=atomic ./tests/plugins/service - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/status.txt -covermode=atomic ./tests/plugins/status - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/config.txt -covermode=atomic ./tests/plugins/config - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/gzip.txt -covermode=atomic ./tests/plugins/gzip - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/headers.txt -covermode=atomic ./tests/plugins/headers - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/logger.txt -covermode=atomic ./tests/plugins/logger - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/metrics.txt -covermode=atomic ./tests/plugins/metrics - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/resetter.txt -covermode=atomic ./tests/plugins/resetter - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/rpc.txt -covermode=atomic ./tests/plugins/rpc - cat ./coverage-ci/*.txt > ./coverage-ci/summary.txt + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipe.out -covermode=atomic ./pkg/transport/pipe + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/socket.out -covermode=atomic ./pkg/transport/socket + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pool.out -covermode=atomic ./pkg/pool + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker.out -covermode=atomic ./pkg/worker + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/bst.out -covermode=atomic ./pkg/bst + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pq.out -covermode=atomic ./pkg/priority_queue + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker_stack.out -covermode=atomic ./pkg/worker_watcher + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/ws_origin.out -covermode=atomic ./plugins/websockets + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http_config.out -covermode=atomic ./plugins/http/config + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server_cmd.out -covermode=atomic ./plugins/server + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/struct_jobs.out -covermode=atomic ./plugins/jobs/job + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipeline_jobs.out -covermode=atomic ./plugins/jobs/pipeline + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/jobs_core.out -covermode=atomic ./tests/plugins/jobs + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/kv_plugin.out -covermode=atomic ./tests/plugins/kv + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/broadcast_plugin.out -covermode=atomic ./tests/plugins/broadcast + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/websockets.out -covermode=atomic ./tests/plugins/websockets + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http.out -covermode=atomic ./tests/plugins/http + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/informer.out -covermode=atomic ./tests/plugins/informer + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/reload.out -covermode=atomic ./tests/plugins/reload + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server.out -covermode=atomic ./tests/plugins/server + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/service.out -covermode=atomic ./tests/plugins/service + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/status.out -covermode=atomic ./tests/plugins/status + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/config.out -covermode=atomic ./tests/plugins/config + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/gzip.out -covermode=atomic ./tests/plugins/gzip + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/headers.out -covermode=atomic ./tests/plugins/headers + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/logger.out -covermode=atomic ./tests/plugins/logger + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/metrics.out -covermode=atomic ./tests/plugins/metrics + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/resetter.out -covermode=atomic ./tests/plugins/resetter + go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/rpc.out -covermode=atomic ./tests/plugins/rpc + echo 'mode: atomic' > ./coverage-ci/summary.txt + tail -q -n +2 ./coverage-ci/*.out >> ./coverage-ci/summary.txt docker-compose -f tests/env/docker-compose.yaml down test: ## Run application tests diff --git a/codecov.yml b/codecov.yml index d895d264..e61cda06 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,25 +3,33 @@ coverage: project: default: target: auto - threshold: 0% + threshold: 50% informational: true patch: default: target: auto - threshold: 0% + threshold: 50% informational: true # do not include tests folders ignore: + - "common" + - "internal" + - "proto" - "tests" + - "systemd" + - "utils/to_ptr.go" - "plugins/metrics/config_test.go" - "plugins/websockets/storage/storage_test.go" - "plugins/websockets/config.go" - - "pkg/bst/bst_test.go" + - "plugins/amqp/amqpjobs/config.go" + - "plugins/beanstalk/config.go" + - "plugins/redis/config.go" + - "plugins/redis/kv/config.go" - "pkg/doc" + - "pkg/bst/bst_test.go" - "pkg/pool/static_pool_test.go" - "pkg/pool/supervisor_test.go" - - "pkg/pubsub" - "pkg/transport/pipe/pipe_factory_spawn_test.go" - "pkg/transport/pipe/pipe_factory_test.go" - "pkg/transport/socket/socket_factory_spawn_test.go" @@ -32,5 +40,4 @@ ignore: - "pkg/worker/worker_test.go" - "pkg/events/pool_events.go" - "pkg/events/worker_events.go" - - "systemd" - - "utils/to_ptr.go" + - "pkg/events/jobs_events.go" diff --git a/pkg/priority_queue/binary_heap_test.go b/pkg/priority_queue/binary_heap_test.go index fb5b83de..ab0f9266 100644 --- a/pkg/priority_queue/binary_heap_test.go +++ b/pkg/priority_queue/binary_heap_test.go @@ -61,6 +61,32 @@ func TestBinHeap_Init(t *testing.T) { require.Equal(t, expected, res) } +func TestBinHeap_MaxLen(t *testing.T) { + a := []Item{Test(2), Test(23), Test(33), Test(44), Test(1), Test(2), Test(2), Test(2), Test(4), Test(6), Test(99)} + + bh := NewBinHeap(1) + + go func() { + expected := []Item{Test(2), Test(23), Test(33), Test(44), Test(1), Test(2), Test(2), Test(2), Test(4), Test(6), Test(99)} + + res := make([]Item, 0, 12) + + for i := 0; i < 11; i++ { + item := bh.ExtractMin() + res = append(res, item) + } + require.Equal(t, expected, res) + return + }() + + time.Sleep(time.Second) + for i := 0; i < len(a); i++ { + bh.Insert(a[i]) + } + + time.Sleep(time.Second) +} + func TestNewPriorityQueue(t *testing.T) { insertsPerSec := uint64(0) getPerSec := uint64(0) diff --git a/pkg/transport/pipe/pipe_factory_test.go b/pkg/transport/pipe/pipe_factory_test.go index d243a93f..e396fe57 100755 --- a/pkg/transport/pipe/pipe_factory_test.go +++ b/pkg/transport/pipe/pipe_factory_test.go @@ -16,6 +16,7 @@ import ( ) func Test_GetState(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -36,6 +37,7 @@ func Test_GetState(t *testing.T) { } func Test_Kill(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -60,6 +62,7 @@ func Test_Kill(t *testing.T) { } func Test_Pipe_Start(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -75,6 +78,7 @@ func Test_Pipe_Start(t *testing.T) { } func Test_Pipe_StartError(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") err := cmd.Start() if err != nil { @@ -88,6 +92,7 @@ func Test_Pipe_StartError(t *testing.T) { } func Test_Pipe_PipeError(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") _, err := cmd.StdinPipe() if err != nil { @@ -101,6 +106,7 @@ func Test_Pipe_PipeError(t *testing.T) { } func Test_Pipe_PipeError2(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") // error cause _, err := cmd.StdinPipe() @@ -115,6 +121,7 @@ func Test_Pipe_PipeError2(t *testing.T) { } func Test_Pipe_Failboot(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/failboot.php") ctx := context.Background() @@ -137,6 +144,7 @@ func Test_Pipe_Failboot(t *testing.T) { } func Test_Pipe_Invalid(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/invalid.php") ctx := context.Background() w, err := NewPipeFactory().SpawnWorkerWithTimeout(ctx, cmd) @@ -145,6 +153,7 @@ func Test_Pipe_Invalid(t *testing.T) { } func Test_Pipe_Echo(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") ctx := context.Background() w, err := NewPipeFactory().SpawnWorkerWithTimeout(ctx, cmd) @@ -171,6 +180,7 @@ func Test_Pipe_Echo(t *testing.T) { } func Test_Pipe_Broken(t *testing.T) { + t.Parallel() cmd := exec.Command("php", "../../../tests/client.php", "broken", "pipes") ctx := context.Background() w, err := NewPipeFactory().SpawnWorkerWithTimeout(ctx, cmd) @@ -286,6 +296,7 @@ func Benchmark_Pipe_Worker_ExecEchoWithoutContext(b *testing.B) { } func Test_Echo(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -316,6 +327,7 @@ func Test_Echo(t *testing.T) { } func Test_BadPayload(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -342,6 +354,7 @@ func Test_BadPayload(t *testing.T) { } func Test_String(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") @@ -362,6 +375,7 @@ func Test_String(t *testing.T) { } func Test_Echo_Slow(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/slow-client.php", "echo", "pipes", "10", "10") @@ -389,6 +403,7 @@ func Test_Echo_Slow(t *testing.T) { } func Test_Broken(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "broken", "pipes") data := "" @@ -422,6 +437,7 @@ func Test_Broken(t *testing.T) { } func Test_Error(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "error", "pipes") @@ -450,6 +466,7 @@ func Test_Error(t *testing.T) { } func Test_NumExecs(t *testing.T) { + t.Parallel() ctx := context.Background() cmd := exec.Command("php", "../../../tests/client.php", "echo", "pipes") diff --git a/plugins/broadcast/config.go b/plugins/broadcast/config.go index 4f1e5213..9531025b 100644 --- a/plugins/broadcast/config.go +++ b/plugins/broadcast/config.go @@ -3,6 +3,8 @@ package broadcast /* # Global redis config (priority - 2) +default: + # redis configuration here websockets: # <----- one of possible subscribers path: /ws diff --git a/plugins/broadcast/plugin.go b/plugins/broadcast/plugin.go index a2390df5..40263eaa 100644 --- a/plugins/broadcast/plugin.go +++ b/plugins/broadcast/plugin.go @@ -15,6 +15,9 @@ const ( PluginName string = "broadcast" // driver is the mandatory field which should present in every storage driver string = "driver" + + // every driver should have config section for the local configuration + conf string = "config" ) type Plugin struct { @@ -50,7 +53,7 @@ func (p *Plugin) Init(cfg config.Configurer, log logger.Logger) error { } func (p *Plugin) Serve() chan error { - return make(chan error) + return make(chan error, 1) } func (p *Plugin) Stop() error { @@ -130,8 +133,8 @@ func (p *Plugin) GetDriver(key string) (pubsub.SubReader, error) { return nil, errors.E(op, errors.Str("wrong type detected in the configuration, please, check yaml indentation")) } - // config key for the particular sub-driver kv.memcached - configKey := fmt.Sprintf("%s.%s", PluginName, key) + // config key for the particular sub-driver broadcast.memcached.config + configKey := fmt.Sprintf("%s.%s.%s", PluginName, key, conf) drName := val.(map[string]interface{})[driver] @@ -141,8 +144,10 @@ func (p *Plugin) GetDriver(key string) (pubsub.SubReader, error) { return nil, errors.E(op, errors.Errorf("no drivers with the requested name registered, registered: %s, requested: %s", p.publishers, drStr)) } + switch { // try local config first - if p.cfgPlugin.Has(configKey) { + case p.cfgPlugin.Has(configKey): + // we found a local configuration ps, err := p.constructors[drStr].PSConstruct(configKey) if err != nil { return nil, errors.E(op, err) @@ -153,9 +158,9 @@ func (p *Plugin) GetDriver(key string) (pubsub.SubReader, error) { p.publishers[configKey] = ps return ps, nil - } else { - // try global driver section - ps, err := p.constructors[drStr].PSConstruct(drStr) + case p.cfgPlugin.Has(key): + // try global driver section after local + ps, err := p.constructors[drStr].PSConstruct(key) if err != nil { return nil, errors.E(op, err) } @@ -165,6 +170,8 @@ func (p *Plugin) GetDriver(key string) (pubsub.SubReader, error) { p.publishers[configKey] = ps return ps, nil + default: + p.log.Error("can't find local or global configuration, this section will be skipped", "local: ", configKey, "global: ", key) } } } diff --git a/plugins/http/attributes/attributes.go b/plugins/http/attributes/attributes.go index 5df80da4..201c2d5e 100644 --- a/plugins/http/attributes/attributes.go +++ b/plugins/http/attributes/attributes.go @@ -39,11 +39,10 @@ func (v attrs) del(key string) { // Init is idempotent returns request with new context and attribute bag. func Init(r *http.Request) *http.Request { - // do not overwrite the PsrContextKey payload + // do not overwrite psr attributes if val := r.Context().Value(PsrContextKey); val == nil { return r.WithContext(context.WithValue(r.Context(), PsrContextKey, attrs{})) } - return r } diff --git a/plugins/kv/plugin.go b/plugins/kv/plugin.go index a1144b85..86bd982f 100644 --- a/plugins/kv/plugin.go +++ b/plugins/kv/plugin.go @@ -10,12 +10,13 @@ import ( "github.com/spiral/roadrunner/v2/plugins/logger" ) -// PluginName linked to the memory, boltdb, memcached, redis plugins. DO NOT change w/o sync. -const PluginName string = "kv" - const ( + // PluginName linked to the memory, boltdb, memcached, redis plugins. DO NOT change w/o sync. + PluginName string = "kv" // driver is the mandatory field which should present in every storage driver string = "driver" + // config key used to detect local configuration for the driver + cfg string = "config" ) // Plugin for the unified storage @@ -75,28 +76,50 @@ func (p *Plugin) Serve() chan error { continue } - // config key for the particular sub-driver kv.memcached - configKey := fmt.Sprintf("%s.%s", PluginName, k) + // config key for the particular sub-driver kv.memcached.config + configKey := fmt.Sprintf("%s.%s.%s", PluginName, k, cfg) // at this point we know, that driver field present in the configuration drName := v.(map[string]interface{})[driver] // driver name should be a string if drStr, ok := drName.(string); ok { - if _, ok := p.constructors[drStr]; !ok { - p.log.Warn("no constructors registered", "requested constructor", drStr, "registered", p.constructors) + switch { + // local configuration section key + case p.cfgPlugin.Has(configKey): + if _, ok := p.constructors[drStr]; !ok { + p.log.Warn("no constructors registered", "requested constructor", drStr, "registered", p.constructors) + continue + } + + storage, err := p.constructors[drStr].KVConstruct(configKey) + if err != nil { + errCh <- errors.E(op, err) + return errCh + } + + // save the storage + p.storages[k] = storage + // try global then + case p.cfgPlugin.Has(k): + if _, ok := p.constructors[drStr]; !ok { + p.log.Warn("no constructors registered", "requested constructor", drStr, "registered", p.constructors) + continue + } + + // use only key for the driver registration, for example rr-boltdb should be globally available + storage, err := p.constructors[drStr].KVConstruct(k) + if err != nil { + errCh <- errors.E(op, err) + return errCh + } + + // save the storage + p.storages[k] = storage + default: + p.log.Error("can't find local or global configuration, this section will be skipped", "local: ", configKey, "global: ", k) continue } - - storage, err := p.constructors[drStr].KVConstruct(configKey) - if err != nil { - errCh <- errors.E(op, err) - return errCh - } - - // save the storage - p.storages[k] = storage } - continue } diff --git a/plugins/memcached/memcachedkv/driver.go b/plugins/memcached/memcachedkv/driver.go index 6d5e1802..dcb071b4 100644 --- a/plugins/memcached/memcachedkv/driver.go +++ b/plugins/memcached/memcachedkv/driver.go @@ -32,6 +32,10 @@ func NewMemcachedDriver(log logger.Logger, key string, cfgPlugin config.Configur return nil, errors.E(op, err) } + if s.cfg == nil { + return nil, errors.E(op, errors.Errorf("config not found by provided key: %s", key)) + } + s.cfg.InitDefaults() m := memcache.New(s.cfg.Addr...) diff --git a/plugins/memory/memoryjobs/consumer.go b/plugins/memory/memoryjobs/consumer.go index fbdedefe..79246063 100644 --- a/plugins/memory/memoryjobs/consumer.go +++ b/plugins/memory/memoryjobs/consumer.go @@ -61,6 +61,10 @@ func NewJobBroker(configKey string, log logger.Logger, cfg config.Configurer, eh return nil, errors.E(op, err) } + if jb.cfg == nil { + return nil, errors.E(op, errors.Errorf("config not found by provided key: %s", configKey)) + } + if jb.cfg.Prefetch == 0 { jb.cfg.Prefetch = 100_000 } @@ -118,6 +122,28 @@ func (c *consumer) Register(_ context.Context, pipeline *pipeline.Pipeline) erro return nil } +func (c *consumer) Run(_ context.Context, pipe *pipeline.Pipeline) error { + const op = errors.Op("memory_jobs_run") + c.eh.Push(events.JobEvent{ + Event: events.EventPipeActive, + Driver: pipe.Driver(), + Pipeline: pipe.Name(), + Start: time.Now(), + }) + + l := atomic.LoadUint32(&c.listeners) + // listener already active + if l == 1 { + c.log.Warn("listener already in the active state") + return errors.E(op, errors.Str("listener already in the active state")) + } + + c.consume() + atomic.StoreUint32(&c.listeners, 1) + + return nil +} + func (c *consumer) Pause(_ context.Context, p string) { start := time.Now() pipe := c.pipeline.Load().(*pipeline.Pipeline) @@ -173,17 +199,6 @@ func (c *consumer) Resume(_ context.Context, p string) { }) } -// Run is no-op for the ephemeral -func (c *consumer) Run(_ context.Context, pipe *pipeline.Pipeline) error { - c.eh.Push(events.JobEvent{ - Event: events.EventPipeActive, - Driver: pipe.Driver(), - Pipeline: pipe.Name(), - Start: time.Now(), - }) - return nil -} - func (c *consumer) Stop(_ context.Context) error { start := time.Now() pipe := c.pipeline.Load().(*pipeline.Pipeline) diff --git a/plugins/memory/memorykv/kv.go b/plugins/memory/memorykv/kv.go index 9b3e176c..5383275c 100644 --- a/plugins/memory/memorykv/kv.go +++ b/plugins/memory/memorykv/kv.go @@ -33,6 +33,10 @@ func NewInMemoryDriver(key string, log logger.Logger, cfgPlugin config.Configure return nil, errors.E(op, err) } + if d.cfg == nil { + return nil, errors.E(op, errors.Errorf("config not found by provided key: %s", key)) + } + d.cfg.InitDefaults() go d.gc() diff --git a/plugins/memory/memorypubsub/pubsub.go b/plugins/memory/memorypubsub/pubsub.go index 75122571..231da134 100644 --- a/plugins/memory/memorypubsub/pubsub.go +++ b/plugins/memory/memorypubsub/pubsub.go @@ -21,7 +21,7 @@ type PubSubDriver struct { func NewPubSubDriver(log logger.Logger, _ string) (*PubSubDriver, error) { ps := &PubSubDriver{ - pushCh: make(chan *pubsub.Message, 10), + pushCh: make(chan *pubsub.Message, 100), storage: bst.NewBST(), log: log, } diff --git a/plugins/memory/plugin.go b/plugins/memory/plugin.go index 515e469a..87e0f84b 100644 --- a/plugins/memory/plugin.go +++ b/plugins/memory/plugin.go @@ -49,7 +49,7 @@ func (p *Plugin) PSConstruct(key string) (pubsub.PubSub, error) { } func (p *Plugin) KVConstruct(key string) (kv.Storage, error) { - const op = errors.Op("inmemory_plugin_provide") + const op = errors.Op("memory_plugin_construct") st, err := memorykv.NewInMemoryDriver(key, p.log, p.cfg) if err != nil { return nil, errors.E(op, err) diff --git a/plugins/redis/jobs/config.go b/plugins/redis/jobs/config.go deleted file mode 100644 index 89d707af..00000000 --- a/plugins/redis/jobs/config.go +++ /dev/null @@ -1,34 +0,0 @@ -package jobs - -import "time" - -type Config struct { - Addrs []string `mapstructure:"addrs"` - DB int `mapstructure:"db"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - MasterName string `mapstructure:"master_name"` - SentinelPassword string `mapstructure:"sentinel_password"` - RouteByLatency bool `mapstructure:"route_by_latency"` - RouteRandomly bool `mapstructure:"route_randomly"` - MaxRetries int `mapstructure:"max_retries"` - DialTimeout time.Duration `mapstructure:"dial_timeout"` - MinRetryBackoff time.Duration `mapstructure:"min_retry_backoff"` - MaxRetryBackoff time.Duration `mapstructure:"max_retry_backoff"` - PoolSize int `mapstructure:"pool_size"` - MinIdleConns int `mapstructure:"min_idle_conns"` - MaxConnAge time.Duration `mapstructure:"max_conn_age"` - ReadTimeout time.Duration `mapstructure:"read_timeout"` - WriteTimeout time.Duration `mapstructure:"write_timeout"` - PoolTimeout time.Duration `mapstructure:"pool_timeout"` - IdleTimeout time.Duration `mapstructure:"idle_timeout"` - IdleCheckFreq time.Duration `mapstructure:"idle_check_freq"` - ReadOnly bool `mapstructure:"read_only"` -} - -// InitDefaults initializing fill config with default values -func (s *Config) InitDefaults() { - if s.Addrs == nil { - s.Addrs = []string{"127.0.0.1:6379"} // default addr is pointing to local storage - } -} diff --git a/plugins/redis/jobs/consumer.go b/plugins/redis/jobs/consumer.go deleted file mode 100644 index 415ac457..00000000 --- a/plugins/redis/jobs/consumer.go +++ /dev/null @@ -1 +0,0 @@ -package jobs diff --git a/plugins/redis/jobs/item.go b/plugins/redis/jobs/item.go deleted file mode 100644 index 415ac457..00000000 --- a/plugins/redis/jobs/item.go +++ /dev/null @@ -1 +0,0 @@ -package jobs diff --git a/plugins/redis/kv/config.go b/plugins/redis/kv/config.go index 5b760952..5bd772a9 100644 --- a/plugins/redis/kv/config.go +++ b/plugins/redis/kv/config.go @@ -1,6 +1,8 @@ package kv -import "time" +import ( + "time" +) type Config struct { Addrs []string `mapstructure:"addrs"` diff --git a/plugins/redis/kv/kv.go b/plugins/redis/kv/kv.go index 3d062fbb..ae55d332 100644 --- a/plugins/redis/kv/kv.go +++ b/plugins/redis/kv/kv.go @@ -20,7 +20,7 @@ type Driver struct { } func NewRedisDriver(log logger.Logger, key string, cfgPlugin config.Configurer) (*Driver, error) { - const op = errors.Op("new_boltdb_driver") + const op = errors.Op("new_redis_driver") d := &Driver{ log: log, @@ -32,8 +32,11 @@ func NewRedisDriver(log logger.Logger, key string, cfgPlugin config.Configurer) return nil, errors.E(op, err) } + if d.cfg == nil { + return nil, errors.E(op, errors.Errorf("config not found by provided key: %s", key)) + } + d.cfg.InitDefaults() - d.log = log d.universalClient = redis.NewUniversalClient(&redis.UniversalOptions{ Addrs: d.cfg.Addrs, diff --git a/plugins/redis/pubsub/pubsub.go b/plugins/redis/pubsub/pubsub.go index c9ad3d58..3561ef18 100644 --- a/plugins/redis/pubsub/pubsub.go +++ b/plugins/redis/pubsub/pubsub.go @@ -13,7 +13,7 @@ import ( type PubSubDriver struct { sync.RWMutex - cfg *Config `mapstructure:"redis"` + cfg *Config log logger.Logger channel *redisChannel @@ -34,6 +34,10 @@ func NewPubSubDriver(log logger.Logger, key string, cfgPlugin config.Configurer, return nil, errors.E(op, err) } + if ps.cfg == nil { + return nil, errors.E(op, errors.Errorf("config not found by provided key: %s", key)) + } + ps.cfg.InitDefaults() ps.universalClient = redis.NewUniversalClient(&redis.UniversalOptions{ diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml index 66114d64..1474feb7 100644 --- a/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml +++ b/tests/plugins/broadcast/configs/.rr-broadcast-config-error.yaml @@ -17,6 +17,7 @@ http: allocate_timeout: 60s destroy_timeout: 60s +# no global or local config broadcast: default: driver: redis diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-global.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-global.yaml index 5ae5a101..a7f9c35d 100644 --- a/tests/plugins/broadcast/configs/.rr-broadcast-global.yaml +++ b/tests/plugins/broadcast/configs/.rr-broadcast-global.yaml @@ -1,39 +1,50 @@ rpc: - listen: tcp://127.0.0.1:6003 + listen: tcp://127.0.0.1:6003 server: - command: "php ../../psr-worker-bench.php" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../psr-worker-bench.php" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:21543 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:21543 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s -redis: - addrs: - - "127.0.0.1:6379" - -broadcast: - test: - driver: redis - test2: - driver: redis +test: addrs: - - "127.0.0.1:6378" - test3: - driver: memory - test4: - driver: memory + - "127.0.0.1:6379" +broadcast: + test: + driver: redis + test2: + driver: redis + config: + addrs: + - "127.0.0.1:6378" + test3: + driver: memory + config: {} + test4: + driver: memory + config: {} logs: - mode: development - level: info + mode: development + level: info diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-init.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-init.yaml index d8457578..1cbebdd7 100644 --- a/tests/plugins/broadcast/configs/.rr-broadcast-init.yaml +++ b/tests/plugins/broadcast/configs/.rr-broadcast-init.yaml @@ -22,8 +22,9 @@ http: broadcast: default: driver: redis - addrs: - - "127.0.0.1:6379" + config: + addrs: + - "127.0.0.1:6379" logs: mode: development diff --git a/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml b/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml index 2337b8fe..85a767cb 100644 --- a/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml +++ b/tests/plugins/broadcast/configs/.rr-broadcast-same-section.yaml @@ -1,36 +1,48 @@ rpc: - listen: tcp://127.0.0.1:6002 + listen: tcp://127.0.0.1:6002 server: - command: "php ../../psr-worker-bench.php" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../psr-worker-bench.php" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:21345 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:21345 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - test: - driver: redis - addrs: - - "127.0.0.1:6379" - test2: - driver: redis - addrs: - - "127.0.0.1:6378" - test3: - driver: memory - test4: - driver: memory - + test: + driver: redis + config: + addrs: + - "127.0.0.1:6379" + test2: + driver: redis + config: + addrs: + - "127.0.0.1:6378" + test3: + driver: memory + config: {} + test4: + driver: memory + config: {} logs: - mode: development - level: info + mode: development + level: info diff --git a/tests/plugins/jobs/jobs_memory_test.go b/tests/plugins/jobs/jobs_memory_test.go index 20cbfb3f..7e39c556 100644 --- a/tests/plugins/jobs/jobs_memory_test.go +++ b/tests/plugins/jobs/jobs_memory_test.go @@ -107,7 +107,7 @@ func TestMemoryInit(t *testing.T) { } }() - time.Sleep(time.Second * 3) + time.Sleep(time.Second * 1) stopCh <- struct{}{} wg.Wait() } @@ -229,7 +229,7 @@ func TestMemoryPauseResume(t *testing.T) { mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-local-2", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1) - mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(3) + mockLogger.EXPECT().Info("pipeline active", "pipeline", "test-local", "start", gomock.Any(), "elapsed", gomock.Any()).Times(2) mockLogger.EXPECT().Info("pipeline paused", "pipeline", "test-local", "driver", "memory", "start", gomock.Any(), "elapsed", gomock.Any()).Times(1) @@ -301,7 +301,6 @@ func TestMemoryPauseResume(t *testing.T) { time.Sleep(time.Second * 3) - t.Run("Resume", resumePipes("test-local")) t.Run("Pause", pausePipelines("test-local")) t.Run("pushToDisabledPipe", pushToDisabledPipe("test-local")) t.Run("Resume", resumePipes("test-local")) diff --git a/tests/plugins/kv/configs/.rr-boltdb.yaml b/tests/plugins/kv/configs/.rr-boltdb.yaml index 81b47715..7a8aee4e 100644 --- a/tests/plugins/kv/configs/.rr-boltdb.yaml +++ b/tests/plugins/kv/configs/.rr-boltdb.yaml @@ -1,15 +1,16 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - boltdb-rr: - driver: boltdb - dir: "." - file: "rr.db" - bucket: "test" - permissions: 0666 - interval: 1 # seconds + boltdb-rr: + driver: boltdb + config: + dir: "." + file: "rr.db" + bucket: "test" + permissions: 0666 + interval: 1 # seconds diff --git a/tests/plugins/kv/configs/.rr-in-memory.yaml b/tests/plugins/kv/configs/.rr-in-memory.yaml index b3b01f46..0452d8bc 100644 --- a/tests/plugins/kv/configs/.rr-in-memory.yaml +++ b/tests/plugins/kv/configs/.rr-in-memory.yaml @@ -1,11 +1,12 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - memory-rr: - driver: memory - interval: 1 + memory-rr: + driver: memory + config: + interval: 1 diff --git a/tests/plugins/kv/configs/.rr-kv-bolt-no-interval.yaml b/tests/plugins/kv/configs/.rr-kv-bolt-no-interval.yaml index 471e5c77..476369c5 100644 --- a/tests/plugins/kv/configs/.rr-kv-bolt-no-interval.yaml +++ b/tests/plugins/kv/configs/.rr-kv-bolt-no-interval.yaml @@ -1,17 +1,19 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - boltdb-south: - driver: boltdb - file: "rr.db" - permissions: 755 + boltdb-south: + driver: boltdb + config: + file: "rr.db" + permissions: 755 - boltdb-africa: - driver: boltdb - file: "africa.db" - permissions: 755 + boltdb-africa: + driver: boltdb + config: + file: "africa.db" + permissions: 755 diff --git a/tests/plugins/kv/configs/.rr-kv-bolt-perms.yaml b/tests/plugins/kv/configs/.rr-kv-bolt-perms.yaml index b46bcb1c..e7728972 100644 --- a/tests/plugins/kv/configs/.rr-kv-bolt-perms.yaml +++ b/tests/plugins/kv/configs/.rr-kv-bolt-perms.yaml @@ -1,16 +1,18 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - boltdb-south: - driver: boltdb - file: "rr.db" + boltdb-south: + driver: boltdb + config: + file: "rr.db" - boltdb-africa: - driver: boltdb - file: "africa.db" - permissions: 0777 + boltdb-africa: + driver: boltdb + config: + file: "africa.db" + permissions: 0777 diff --git a/tests/plugins/kv/configs/.rr-kv-init.yaml b/tests/plugins/kv/configs/.rr-kv-init.yaml index 6407c7ad..10cf6491 100644 --- a/tests/plugins/kv/configs/.rr-kv-init.yaml +++ b/tests/plugins/kv/configs/.rr-kv-init.yaml @@ -1,30 +1,35 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - default: - driver: memory + default: + driver: memory + config: + interval: 60 - boltdb-south: - driver: boltdb - dir: "." - file: "rr.db" - bucket: "rr" - permissions: 0666 - interval: 1 + boltdb-south: + driver: boltdb + config: + dir: "." + file: "rr.db" + bucket: "rr" + permissions: 0666 + interval: 1 - boltdb-africa: - driver: boltdb - dir: "." - file: "africa.db" - bucket: "rr" - permissions: 0666 - interval: 1 + boltdb-africa: + driver: boltdb + config: + dir: "." + file: "africa.db" + bucket: "rr" + permissions: 0666 + interval: 1 - memcached: - driver: memcached - addr: [ "127.0.0.1:11211" ] + memcached: + driver: memcached + config: + addr: ["127.0.0.1:11211"] diff --git a/tests/plugins/kv/configs/.rr-memcached.yaml b/tests/plugins/kv/configs/.rr-memcached.yaml index da5d59c6..ef8de2ab 100644 --- a/tests/plugins/kv/configs/.rr-memcached.yaml +++ b/tests/plugins/kv/configs/.rr-memcached.yaml @@ -1,12 +1,13 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - memcached-rr: - driver: memcached - addr: - - "127.0.0.1:11211" + memcached-rr: + driver: memcached + config: + addr: + - "127.0.0.1:11211" diff --git a/tests/plugins/kv/configs/.rr-redis-global.yaml b/tests/plugins/kv/configs/.rr-redis-global.yaml index a4979879..27377835 100644 --- a/tests/plugins/kv/configs/.rr-redis-global.yaml +++ b/tests/plugins/kv/configs/.rr-redis-global.yaml @@ -1,14 +1,14 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error redis-rr: - addrs: - - '127.0.0.1:6379' + addrs: + - "127.0.0.1:6379" kv: - redis-rr: - driver: redis + redis-rr: + driver: redis diff --git a/tests/plugins/kv/configs/.rr-redis-no-config.yaml b/tests/plugins/kv/configs/.rr-redis-no-config.yaml index 9cf06374..56113f13 100644 --- a/tests/plugins/kv/configs/.rr-redis-no-config.yaml +++ b/tests/plugins/kv/configs/.rr-redis-no-config.yaml @@ -1,10 +1,10 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - redis-rr: - driver: redis + redis-rr: + driver: redis diff --git a/tests/plugins/kv/configs/.rr-redis.yaml b/tests/plugins/kv/configs/.rr-redis.yaml index 522e365a..f9b967d5 100644 --- a/tests/plugins/kv/configs/.rr-redis.yaml +++ b/tests/plugins/kv/configs/.rr-redis.yaml @@ -1,12 +1,13 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 logs: - mode: development - level: error + mode: development + level: error kv: - redis-rr: - driver: redis - addrs: - - '127.0.0.1:6379' + redis-rr: + driver: redis + config: + addrs: + - "127.0.0.1:6379" diff --git a/tests/plugins/kv/storage_plugin_test.go b/tests/plugins/kv/storage_plugin_test.go index e757a9e6..c10e4726 100644 --- a/tests/plugins/kv/storage_plugin_test.go +++ b/tests/plugins/kv/storage_plugin_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/golang/mock/gomock" endure "github.com/spiral/endure/pkg/container" goridgeRpc "github.com/spiral/goridge/v3/pkg/rpc" "github.com/spiral/roadrunner/v2/plugins/boltdb" @@ -21,6 +22,7 @@ import ( "github.com/spiral/roadrunner/v2/plugins/redis" rpcPlugin "github.com/spiral/roadrunner/v2/plugins/rpc" payload "github.com/spiral/roadrunner/v2/proto/kv/v1beta" + "github.com/spiral/roadrunner/v2/tests/mocks" "github.com/stretchr/testify/assert" ) @@ -1293,12 +1295,21 @@ func TestRedisNoConfig(t *testing.T) { Prefix: "rr", } + controller := gomock.NewController(t) + mockLogger := mocks.NewMockLogger(controller) + + mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("Started RPC service", "address", "tcp://127.0.0.1:6001", "plugins", []string{"kv"}).AnyTimes() + + mockLogger.EXPECT().Error(`can't find local or global configuration, this section will be skipped`, "local: ", "kv.redis-rr.config", "global: ", "redis-rr").Times(1) + err = cont.RegisterAll( cfg, &kv.Plugin{}, &redis.Plugin{}, &rpcPlugin.Plugin{}, - &logger.ZapLogger{}, + mockLogger, &memory.Plugin{}, ) assert.NoError(t, err) @@ -1308,48 +1319,8 @@ func TestRedisNoConfig(t *testing.T) { t.Fatal(err) } - ch, err := cont.Serve() + _, err = cont.Serve() assert.NoError(t, err) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - wg := &sync.WaitGroup{} - wg.Add(1) - - stopCh := make(chan struct{}, 1) - - go func() { - defer wg.Done() - for { - select { - case e := <-ch: - assert.Fail(t, "error", e.Error.Error()) - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - case <-sig: - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - case <-stopCh: - // timeout - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - } - } - }() - - time.Sleep(time.Second * 1) - t.Run("REDIS", testRPCMethodsRedis) - stopCh <- struct{}{} - wg.Wait() } func testRPCMethodsRedis(t *testing.T) { diff --git a/tests/plugins/websockets/configs/.rr-websockets-allow.yaml b/tests/plugins/websockets/configs/.rr-websockets-allow.yaml index 900094a4..3d0268d4 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-allow.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-allow.yaml @@ -1,42 +1,52 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../worker-ok.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../worker-ok.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:41278 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:41278 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s redis: - addrs: - - "127.0.0.1:6379" + addrs: + - "127.0.0.1:6379" broadcast: - test: - driver: memory + test: + driver: memory + config: {} websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-allow2.yaml b/tests/plugins/websockets/configs/.rr-websockets-allow2.yaml index 43f4b2ec..f8e36136 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-allow2.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-allow2.yaml @@ -1,44 +1,54 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../worker-ok.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../worker-ok.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:41270 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:41270 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s -redis: - addrs: - - "127.0.0.1:6379" +test: + addrs: + - "127.0.0.1:6379" broadcast: - test: - driver: redis - addrs: - - "127.0.0.1:6379" + test: + driver: redis + config: + addrs: + - "127.0.0.1:6379" websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-broker-no-section.yaml b/tests/plugins/websockets/configs/.rr-websockets-broker-no-section.yaml index ada23845..c72e1f15 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-broker-no-section.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-broker-no-section.yaml @@ -1,38 +1,48 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../psr-worker-bench.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../psr-worker-bench.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:13235 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:13235 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - test1: - driver: no + test1: + driver: no + config: websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-deny.yaml b/tests/plugins/websockets/configs/.rr-websockets-deny.yaml index 594a746d..61265c4b 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-deny.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-deny.yaml @@ -1,38 +1,48 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../worker-deny.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../worker-deny.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:15587 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:15587 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - test: - driver: memory + test: + driver: memory + config: {} websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-deny2.yaml b/tests/plugins/websockets/configs/.rr-websockets-deny2.yaml index e0bdf993..b99a3571 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-deny2.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-deny2.yaml @@ -1,40 +1,50 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../worker-deny.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../worker-deny.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:15588 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:15588 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - test: - driver: redis - addrs: - - "127.0.0.1:6379" + test: + driver: redis + config: + addrs: + - "127.0.0.1:6379" websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-init.yaml b/tests/plugins/websockets/configs/.rr-websockets-init.yaml index 115f9a71..3120f146 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-init.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-init.yaml @@ -1,39 +1,48 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../psr-worker-bench.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../psr-worker-bench.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:11111 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:11111 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - default: - driver: memory + default: + driver: memory + config: {} websockets: - broker: default - allowed_origin: "*" - path: "/ws" - + broker: default + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-redis.yaml b/tests/plugins/websockets/configs/.rr-websockets-redis.yaml index e3d5f0b8..fc01e0b1 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-redis.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-redis.yaml @@ -1,42 +1,51 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../psr-worker-bench.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../psr-worker-bench.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:13235 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:13235 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s -redis: - addrs: - - "127.0.0.1:6379" +test: + addrs: + - "127.0.0.1:6379" broadcast: - test: - driver: redis + test: + driver: redis websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/plugins/websockets/configs/.rr-websockets-stop.yaml b/tests/plugins/websockets/configs/.rr-websockets-stop.yaml index 5377aef2..35529e9e 100644 --- a/tests/plugins/websockets/configs/.rr-websockets-stop.yaml +++ b/tests/plugins/websockets/configs/.rr-websockets-stop.yaml @@ -1,38 +1,48 @@ rpc: - listen: tcp://127.0.0.1:6001 + listen: tcp://127.0.0.1:6001 server: - command: "php ../../worker-stop.php" - user: "" - group: "" - relay: "pipes" - relay_timeout: "20s" + command: "php ../../worker-stop.php" + user: "" + group: "" + relay: "pipes" + relay_timeout: "20s" http: - address: 127.0.0.1:11114 - max_request_size: 1024 - middleware: [ "websockets" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s + address: 127.0.0.1:11114 + max_request_size: 1024 + middleware: ["websockets"] + trusted_subnets: + [ + "10.0.0.0/8", + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "::1/128", + "fc00::/7", + "fe80::/10", + ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s broadcast: - test: - driver: memory + test: + driver: memory + config: {} websockets: - broker: test - allowed_origin: "*" - path: "/ws" + broker: test + allowed_origin: "*" + path: "/ws" logs: - mode: development - level: error + mode: development + level: error endure: - grace_period: 120s - print_graph: false - log_level: error + grace_period: 120s + print_graph: false + log_level: error diff --git a/tests/script.sh b/tests/script.sh new file mode 100755 index 00000000..746fb768 --- /dev/null +++ b/tests/script.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +php ../../../tests/client.php echo pipes
\ No newline at end of file |