From f6359114607f9daa41aa90d452ebdc970615c3ab Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Sun, 25 Apr 2021 20:18:35 +0300 Subject: - Initial commit of the updated static plugin Signed-off-by: Valery Piashchynski --- plugins/static/config.go | 36 ++----- plugins/static/plugin.go | 119 ++++++++++++++++----- tests/plugins/static/config_test.go | 49 --------- .../static/configs/.rr-http-static-files.yaml | 8 +- tests/plugins/static/static_plugin_test.go | 3 +- 5 files changed, 104 insertions(+), 111 deletions(-) delete mode 100644 tests/plugins/static/config_test.go diff --git a/plugins/static/config.go b/plugins/static/config.go index 90efea76..2519c04f 100644 --- a/plugins/static/config.go +++ b/plugins/static/config.go @@ -2,8 +2,6 @@ package static import ( "os" - "path" - "strings" "github.com/spiral/errors" ) @@ -14,10 +12,14 @@ type Config struct { // Dir contains name of directory to control access to. Dir string - // Forbid specifies list of file extensions which are forbidden for access. - // Example: .php, .exe, .bat, .htaccess and etc. + // forbid specifies list of file extensions which are forbidden for access. + // example: .php, .exe, .bat, .htaccess and etc. Forbid []string + // Allow specifies list of file extensions which are allowed for access. + // example: .php, .exe, .bat, .htaccess and etc. + Allow []string + // Always specifies list of extensions which must always be served by static // service, even if file not found. Always []string @@ -48,29 +50,3 @@ func (c *Config) Valid() error { return nil } - -// AlwaysForbid must return true if file extension is not allowed for the upload. -func (c *Config) AlwaysForbid(filename string) bool { - ext := strings.ToLower(path.Ext(filename)) - - for _, v := range c.Static.Forbid { - if ext == v { - return true - } - } - - return false -} - -// AlwaysServe must indicate that file is expected to be served by static service. -func (c *Config) AlwaysServe(filename string) bool { - ext := strings.ToLower(path.Ext(filename)) - - for _, v := range c.Static.Always { - if ext == v { - return true - } - } - - return false -} diff --git a/plugins/static/plugin.go b/plugins/static/plugin.go index 76cb9e68..b6c25f3d 100644 --- a/plugins/static/plugin.go +++ b/plugins/static/plugin.go @@ -1,8 +1,10 @@ package static import ( + "io/fs" "net/http" "path" + "strings" "github.com/spiral/errors" "github.com/spiral/roadrunner/v2/plugins/config" @@ -23,6 +25,14 @@ type Plugin struct { // root is initiated http directory root http.Dir + + // file extensions which are allowed to be served + allowedExtensions map[string]struct{} + + // file extensions which are forbidden to be served + forbiddenExtensions map[string]struct{} + + alwaysServe map[string]struct{} } // Init must return configure service and return true if service hasStatus enabled. Must return error in case of @@ -50,6 +60,33 @@ func (s *Plugin) Init(cfg config.Configurer, log logger.Logger) error { return errors.E(op, err) } + // create 2 hashmaps with the allowed and forbidden file extensions + s.allowedExtensions = make(map[string]struct{}, len(s.cfg.Static.Allow)) + s.forbiddenExtensions = make(map[string]struct{}, len(s.cfg.Static.Forbid)) + s.alwaysServe = make(map[string]struct{}, len(s.cfg.Static.Always)) + + for i := 0; i < len(s.cfg.Static.Forbid); i++ { + s.forbiddenExtensions[s.cfg.Static.Forbid[i]] = struct{}{} + } + + for i := 0; i < len(s.cfg.Static.Allow); i++ { + s.forbiddenExtensions[s.cfg.Static.Allow[i]] = struct{}{} + } + + // check if any forbidden items presented in the allowed + // if presented, delete such items from allowed + for k := range s.forbiddenExtensions { + if _, ok := s.allowedExtensions[k]; ok { + delete(s.allowedExtensions, k) + } + } + + for i := 0; i < len(s.cfg.Static.Always); i++ { + s.alwaysServe[s.cfg.Static.Always[i]] = struct{}{} + } + + // at this point we have distinct allowed and forbidden hashmaps, also with alwaysServed + return nil } @@ -73,45 +110,77 @@ func (s *Plugin) Middleware(next http.Handler) http.HandlerFunc { } } - if !s.handleStatic(w, r) { - next.ServeHTTP(w, r) + fPath := path.Clean(r.URL.Path) + ext := strings.ToLower(path.Ext(fPath)) + + // check that file is in forbidden list + if _, ok := s.forbiddenExtensions[ext]; ok { + http.Error(w, "file is forbidden", 404) + return + } + + f, err := s.root.Open(fPath) + if err != nil { + // if we should always serve files with some extensions + // show error to the user and invoke next middleware + if _, ok := s.alwaysServe[ext]; ok { + //http.Error(w, err.Error(), 404) + w.WriteHeader(404) + next.ServeHTTP(w, r) + return + } + // else, return with error + http.Error(w, err.Error(), 404) + return } - } -} -func (s *Plugin) handleStatic(w http.ResponseWriter, r *http.Request) bool { - fPath := path.Clean(r.URL.Path) + defer func() { + err = f.Close() + if err != nil { + s.log.Error("file close error", "error", err) + } + }() - if s.cfg.AlwaysForbid(fPath) { - return false - } + // here we know, that file extension is not in the AlwaysServe and file exists + // (or by some reason, there is no error from the http.Open method) - f, err := s.root.Open(fPath) - if err != nil { - if s.cfg.AlwaysServe(fPath) { - w.WriteHeader(404) - return true + // if we have some allowed extensions, we should check them + if len(s.allowedExtensions) > 0 { + if _, ok := s.allowedExtensions[ext]; ok { + d, err := s.check(f) + if err != nil { + http.Error(w, err.Error(), 404) + return + } + + http.ServeContent(w, r, d.Name(), d.ModTime(), f) + } + // otherwise we guess, that all file extensions are allowed + } else { + d, err := s.check(f) + if err != nil { + http.Error(w, err.Error(), 404) + return + } + + http.ServeContent(w, r, d.Name(), d.ModTime(), f) } - return false + next.ServeHTTP(w, r) } - defer func() { - err = f.Close() - if err != nil { - s.log.Error("file closing error", "error", err) - } - }() +} +func (s *Plugin) check(f http.File) (fs.FileInfo, error) { + const op = errors.Op("http_file_check") d, err := f.Stat() if err != nil { - return false + return nil, err } // do not serve directories if d.IsDir() { - return false + return nil, errors.E(op, errors.Str("directory path provided, should be path to the file")) } - http.ServeContent(w, r, d.Name(), d.ModTime(), f) - return true + return d, nil } diff --git a/tests/plugins/static/config_test.go b/tests/plugins/static/config_test.go deleted file mode 100644 index d73fd845..00000000 --- a/tests/plugins/static/config_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package static - -import ( - "testing" - - "github.com/spiral/roadrunner/v2/plugins/static" - "github.com/stretchr/testify/assert" -) - -func TestConfig_Forbids(t *testing.T) { - cfg := static.Config{Static: &struct { - Dir string - Forbid []string - Always []string - Request map[string]string - Response map[string]string - }{Dir: "", Forbid: []string{".php"}, Always: nil, Request: nil, Response: nil}} - - assert.True(t, cfg.AlwaysForbid("index.php")) - assert.True(t, cfg.AlwaysForbid("index.PHP")) - assert.True(t, cfg.AlwaysForbid("phpadmin/index.bak.php")) - assert.False(t, cfg.AlwaysForbid("index.html")) -} - -func TestConfig_Valid(t *testing.T) { - assert.NoError(t, (&static.Config{Static: &struct { - Dir string - Forbid []string - Always []string - Request map[string]string - Response map[string]string - }{Dir: "./"}}).Valid()) - - assert.Error(t, (&static.Config{Static: &struct { - Dir string - Forbid []string - Always []string - Request map[string]string - Response map[string]string - }{Dir: "./http.go"}}).Valid()) - - assert.Error(t, (&static.Config{Static: &struct { - Dir string - Forbid []string - Always []string - Request map[string]string - Response map[string]string - }{Dir: "./dir/"}}).Valid()) -} diff --git a/tests/plugins/static/configs/.rr-http-static-files.yaml b/tests/plugins/static/configs/.rr-http-static-files.yaml index d6b3032e..0e003dae 100644 --- a/tests/plugins/static/configs/.rr-http-static-files.yaml +++ b/tests/plugins/static/configs/.rr-http-static-files.yaml @@ -18,11 +18,7 @@ http: dir: "../../../tests" forbid: [ ".php", ".htaccess" ] always: [ ".ico" ] - request: - "Example-Request-Header": "Value" - # Automatically add headers to every response. - response: - "X-Powered-By": "RoadRunner" + pool: num_workers: 2 max_jobs: 0 @@ -30,4 +26,4 @@ http: destroy_timeout: 60s logs: mode: development - level: error \ No newline at end of file + level: error diff --git a/tests/plugins/static/static_plugin_test.go b/tests/plugins/static/static_plugin_test.go index 38562537..b58f1f6b 100644 --- a/tests/plugins/static/static_plugin_test.go +++ b/tests/plugins/static/static_plugin_test.go @@ -259,7 +259,8 @@ func TestStaticFilesForbid(t *testing.T) { err = cont.RegisterAll( cfg, - mockLogger, + //mockLogger, + &logger.ZapLogger{}, &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, -- cgit v1.2.3 From 9c07d12a0cc137de0dc79eb94057470985ee5e6c Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Mon, 26 Apr 2021 21:04:37 +0300 Subject: - Totally rework static plugin - Remove old one, now it is part of the HTTP plugin Signed-off-by: Valery Piashchynski --- .github/workflows/linux.yml | 1 - Makefile | 2 - plugins/http/config/http.go | 18 + plugins/http/config/static.go | 51 +++ plugins/http/plugin.go | 57 ++- plugins/http/static.go | 88 +++++ plugins/static/config.go | 52 --- plugins/static/plugin.go | 186 ---------- .../http/configs/.rr-http-static-disabled.yaml | 27 ++ .../configs/.rr-http-static-files-disable.yaml | 24 ++ .../http/configs/.rr-http-static-files.yaml | 30 ++ tests/plugins/http/configs/.rr-http-static.yaml | 33 ++ tests/plugins/http/http_plugin_test.go | 374 +++++++++++++++++++- .../static/configs/.rr-http-static-disabled.yaml | 31 -- .../configs/.rr-http-static-files-disable.yaml | 32 -- .../static/configs/.rr-http-static-files.yaml | 29 -- tests/plugins/static/configs/.rr-http-static.yaml | 31 -- tests/plugins/static/static_plugin_test.go | 387 --------------------- 18 files changed, 674 insertions(+), 779 deletions(-) create mode 100644 plugins/http/config/static.go create mode 100644 plugins/http/static.go delete mode 100644 plugins/static/config.go delete mode 100644 plugins/static/plugin.go create mode 100644 tests/plugins/http/configs/.rr-http-static-disabled.yaml create mode 100644 tests/plugins/http/configs/.rr-http-static-files-disable.yaml create mode 100644 tests/plugins/http/configs/.rr-http-static-files.yaml create mode 100644 tests/plugins/http/configs/.rr-http-static.yaml delete mode 100644 tests/plugins/static/configs/.rr-http-static-disabled.yaml delete mode 100644 tests/plugins/static/configs/.rr-http-static-files-disable.yaml delete mode 100644 tests/plugins/static/configs/.rr-http-static-files.yaml delete mode 100644 tests/plugins/static/configs/.rr-http-static.yaml delete mode 100644 tests/plugins/static/static_plugin_test.go diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8cc61f09..f2b203f4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -76,7 +76,6 @@ jobs: go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/redis.txt -covermode=atomic ./tests/plugins/redis go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/resetter.txt -covermode=atomic ./tests/plugins/resetter go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/rpc.txt -covermode=atomic ./tests/plugins/rpc - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/static.txt -covermode=atomic ./tests/plugins/static go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/kv_plugin.txt -covermode=atomic ./tests/plugins/kv docker-compose -f ./tests/docker-compose.yaml down cat ./coverage-ci/*.txt > ./coverage-ci/summary.txt diff --git a/Makefile b/Makefile index ca0159cd..8cce149a 100755 --- a/Makefile +++ b/Makefile @@ -47,7 +47,6 @@ test_coverage: go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/redis.out -covermode=atomic ./tests/plugins/redis go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/resetter.out -covermode=atomic ./tests/plugins/resetter go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/rpc.out -covermode=atomic ./tests/plugins/rpc - go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/static.out -covermode=atomic ./tests/plugins/static go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/boltdb.out -covermode=atomic ./tests/plugins/kv/boltdb go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/memory.out -covermode=atomic ./tests/plugins/kv/memory go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/memcached.out -covermode=atomic ./tests/plugins/kv/memcached @@ -77,6 +76,5 @@ test: ## Run application tests go test -v -race -tags=debug ./tests/plugins/redis go test -v -race -tags=debug ./tests/plugins/resetter go test -v -race -tags=debug ./tests/plugins/rpc - go test -v -race -tags=debug ./tests/plugins/static go test -v -race -tags=debug ./tests/plugins/kv docker-compose -f tests/docker-compose.yaml down diff --git a/plugins/http/config/http.go b/plugins/http/config/http.go index 8b63395f..31b10322 100644 --- a/plugins/http/config/http.go +++ b/plugins/http/config/http.go @@ -33,6 +33,9 @@ type HTTP struct { // Uploads configures uploads configuration. Uploads *Uploads `mapstructure:"uploads"` + // static configuration + Static *Static `mapstructure:"static"` + // Pool configures worker pool. Pool *poolImpl.Config `mapstructure:"pool"` @@ -100,6 +103,13 @@ func (c *HTTP) InitDefaults() error { c.SSLConfig.Address = "127.0.0.1:443" } + // static files + if c.Static != nil { + if c.Static.Pattern == "" { + c.Static.Pattern = "/static" + } + } + err := c.HTTP2Config.InitDefaults() if err != nil { return err @@ -176,5 +186,13 @@ func (c *HTTP) Valid() error { } } + // validate static + if c.Static != nil { + err := c.Static.Valid() + if err != nil { + return errors.E(op, err) + } + } + return nil } diff --git a/plugins/http/config/static.go b/plugins/http/config/static.go new file mode 100644 index 00000000..416169d2 --- /dev/null +++ b/plugins/http/config/static.go @@ -0,0 +1,51 @@ +package config + +import ( + "os" + + "github.com/spiral/errors" +) + +// Static describes file location and controls access to them. +type Static struct { + // Dir contains name of directory to control access to. + Dir string + + // HTTP pattern, where to serve static files + // for example - `/static`, `/my-files/static`, etc + // Default - /static + Pattern string + + // forbid specifies list of file extensions which are forbidden for access. + // example: .php, .exe, .bat, .htaccess and etc. + Forbid []string + + // Allow specifies list of file extensions which are allowed for access. + // example: .php, .exe, .bat, .htaccess and etc. + Allow []string + + // Request headers to add to every static. + Request map[string]string + + // Response headers to add to every static. + Response map[string]string +} + +// Valid returns nil if config is valid. +func (c *Static) Valid() error { + const op = errors.Op("static_plugin_valid") + st, err := os.Stat(c.Dir) + if err != nil { + if os.IsNotExist(err) { + return errors.E(op, errors.Errorf("root directory '%s' does not exists", c.Dir)) + } + + return err + } + + if !st.IsDir() { + return errors.E(op, errors.Errorf("invalid root directory '%s'", c.Dir)) + } + + return nil +} diff --git a/plugins/http/plugin.go b/plugins/http/plugin.go index 01bd243f..dcfb7ddb 100644 --- a/plugins/http/plugin.go +++ b/plugins/http/plugin.go @@ -59,7 +59,9 @@ type Plugin struct { // stdlog passed to the http/https/fcgi servers to log their internal messages stdLog *log.Logger + // http configuration cfg *httpConfig.HTTP `mapstructure:"http"` + // middlewares to chain mdwr middleware @@ -138,7 +140,7 @@ func (s *Plugin) Serve() chan error { return errCh } -func (s *Plugin) serve(errCh chan error) { +func (s *Plugin) serve(errCh chan error) { //nolint:gocognit var err error const op = errors.Op("http_plugin_serve") s.pool, err = s.server.NewWorkerPool(context.Background(), pool.Config{ @@ -167,11 +169,37 @@ func (s *Plugin) serve(errCh chan error) { s.handler.AddListener(s.logCallback) + // Create new HTTP Multiplexer + mux := http.NewServeMux() + + // if we have static, handler here, create a fileserver + if s.cfg.Static != nil { + h := http.FileServer(StaticFilesHandler(s.cfg.Static)) + // Static files handler + mux.HandleFunc(s.cfg.Static.Pattern, func(w http.ResponseWriter, r *http.Request) { + if s.cfg.Static.Request != nil { + for k, v := range s.cfg.Static.Request { + r.Header.Add(k, v) + } + } + + if s.cfg.Static.Response != nil { + for k, v := range s.cfg.Static.Response { + w.Header().Set(k, v) + } + } + + h.ServeHTTP(w, r) + }) + } + + mux.HandleFunc("/", s.ServeHTTP) + if s.cfg.EnableHTTP() { if s.cfg.EnableH2C() { - s.http = &http.Server{Handler: h2c.NewHandler(s, &http2.Server{}), ErrorLog: s.stdLog} + s.http = &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{}), ErrorLog: s.stdLog} } else { - s.http = &http.Server{Handler: s, ErrorLog: s.stdLog} + s.http = &http.Server{Handler: mux, ErrorLog: s.stdLog} } } @@ -195,7 +223,7 @@ func (s *Plugin) serve(errCh chan error) { } if s.cfg.EnableFCGI() { - s.fcgi = &http.Server{Handler: s, ErrorLog: s.stdLog} + s.fcgi = &http.Server{Handler: mux, ErrorLog: s.stdLog} } // start http, https and fcgi servers if requested in the config @@ -216,9 +244,11 @@ func (s *Plugin) serveHTTP(errCh chan error) { if s.http == nil { return } - const op = errors.Op("http_plugin_serve_http") - applyMiddlewares(s.http, s.mdwr, s.cfg.Middleware, s.log) + + if len(s.mdwr) > 0 { + applyMiddlewares(s.http, s.mdwr, s.cfg.Middleware, s.log) + } l, err := utils.CreateListener(s.cfg.Address) if err != nil { errCh <- errors.E(op, err) @@ -236,9 +266,10 @@ func (s *Plugin) serveHTTPS(errCh chan error) { if s.https == nil { return } - const op = errors.Op("http_plugin_serve_https") - applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) + if len(s.mdwr) > 0 { + applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) + } l, err := utils.CreateListener(s.cfg.SSLConfig.Address) if err != nil { errCh <- errors.E(op, err) @@ -262,9 +293,12 @@ func (s *Plugin) serveFCGI(errCh chan error) { if s.fcgi == nil { return } - const op = errors.Op("http_plugin_serve_fcgi") - applyMiddlewares(s.fcgi, s.mdwr, s.cfg.Middleware, s.log) + + if len(s.mdwr) > 0 { + applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) + } + l, err := utils.CreateListener(s.cfg.FCGIConfig.Address) if err != nil { errCh <- errors.E(op, err) @@ -607,9 +641,6 @@ func (s *Plugin) tlsAddr(host string, forcePort bool) string { } func applyMiddlewares(server *http.Server, middlewares map[string]Middleware, order []string, log logger.Logger) { - if len(middlewares) == 0 { - return - } for i := 0; i < len(order); i++ { if mdwr, ok := middlewares[order[i]]; ok { server.Handler = mdwr.Middleware(server.Handler) diff --git a/plugins/http/static.go b/plugins/http/static.go new file mode 100644 index 00000000..be977fb3 --- /dev/null +++ b/plugins/http/static.go @@ -0,0 +1,88 @@ +package http + +import ( + "io/fs" + "net/http" + "path/filepath" + "strings" + + httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" +) + +type ExtensionFilter struct { + allowed map[string]struct{} + forbidden map[string]struct{} +} + +func NewExtensionFilter(allow, forbid []string) *ExtensionFilter { + ef := &ExtensionFilter{ + allowed: make(map[string]struct{}, len(allow)), + forbidden: make(map[string]struct{}, len(forbid)), + } + + for i := 0; i < len(forbid); i++ { + // skip empty lines + if forbid[i] == "" { + continue + } + ef.forbidden[forbid[i]] = struct{}{} + } + + for i := 0; i < len(allow); i++ { + // skip empty lines + if allow[i] == "" { + continue + } + ef.allowed[allow[i]] = struct{}{} + } + + // check if any forbidden items presented in the allowed + // if presented, delete such items from allowed + for k := range ef.allowed { + if _, ok := ef.forbidden[k]; ok { + delete(ef.allowed, k) + } + } + + return ef +} + +type FileSystem struct { + ef *ExtensionFilter + // embedded + http.FileSystem +} + +// Open wrapper around http.FileSystem Open method, name here is the name of the +func (f FileSystem) Open(name string) (http.File, error) { + file, err := f.FileSystem.Open(name) + if err != nil { + return nil, err + } + + fstat, err := file.Stat() + if err != nil { + return nil, fs.ErrNotExist + } + + if fstat.IsDir() { + return nil, fs.ErrPermission + } + + ext := strings.ToLower(filepath.Ext(fstat.Name())) + if _, ok := f.ef.forbidden[ext]; ok { + return nil, fs.ErrPermission + } + + // if file extension is allowed, append it to the FileInfo slice + if _, ok := f.ef.allowed[ext]; ok { + return file, nil + } + + return nil, fs.ErrNotExist +} + +// StaticFilesHandler is a constructor for the http.FileSystem +func StaticFilesHandler(config *httpConfig.Static) http.FileSystem { + return FileSystem{NewExtensionFilter(config.Allow, config.Forbid), http.Dir(config.Dir)} +} diff --git a/plugins/static/config.go b/plugins/static/config.go deleted file mode 100644 index 2519c04f..00000000 --- a/plugins/static/config.go +++ /dev/null @@ -1,52 +0,0 @@ -package static - -import ( - "os" - - "github.com/spiral/errors" -) - -// Config describes file location and controls access to them. -type Config struct { - Static *struct { - // Dir contains name of directory to control access to. - Dir string - - // forbid specifies list of file extensions which are forbidden for access. - // example: .php, .exe, .bat, .htaccess and etc. - Forbid []string - - // Allow specifies list of file extensions which are allowed for access. - // example: .php, .exe, .bat, .htaccess and etc. - Allow []string - - // Always specifies list of extensions which must always be served by static - // service, even if file not found. - Always []string - - // Request headers to add to every static. - Request map[string]string - - // Response headers to add to every static. - Response map[string]string - } -} - -// Valid returns nil if config is valid. -func (c *Config) Valid() error { - const op = errors.Op("static_plugin_valid") - st, err := os.Stat(c.Static.Dir) - if err != nil { - if os.IsNotExist(err) { - return errors.E(op, errors.Errorf("root directory '%s' does not exists", c.Static.Dir)) - } - - return err - } - - if !st.IsDir() { - return errors.E(op, errors.Errorf("invalid root directory '%s'", c.Static.Dir)) - } - - return nil -} diff --git a/plugins/static/plugin.go b/plugins/static/plugin.go deleted file mode 100644 index b6c25f3d..00000000 --- a/plugins/static/plugin.go +++ /dev/null @@ -1,186 +0,0 @@ -package static - -import ( - "io/fs" - "net/http" - "path" - "strings" - - "github.com/spiral/errors" - "github.com/spiral/roadrunner/v2/plugins/config" - "github.com/spiral/roadrunner/v2/plugins/logger" -) - -// ID contains default service name. -const PluginName = "static" - -const RootPluginName = "http" - -// Plugin serves static files. Potentially convert into middleware? -type Plugin struct { - // server configuration (location, forbidden files and etc) - cfg *Config - - log logger.Logger - - // root is initiated http directory - root http.Dir - - // file extensions which are allowed to be served - allowedExtensions map[string]struct{} - - // file extensions which are forbidden to be served - forbiddenExtensions map[string]struct{} - - alwaysServe map[string]struct{} -} - -// Init must return configure service and return true if service hasStatus enabled. Must return error in case of -// misconfiguration. Services must not be used without proper configuration pushed first. -func (s *Plugin) Init(cfg config.Configurer, log logger.Logger) error { - const op = errors.Op("static_plugin_init") - if !cfg.Has(RootPluginName) { - return errors.E(op, errors.Disabled) - } - - err := cfg.UnmarshalKey(RootPluginName, &s.cfg) - if err != nil { - return errors.E(op, errors.Disabled, err) - } - - if s.cfg.Static == nil { - return errors.E(op, errors.Disabled) - } - - s.log = log - s.root = http.Dir(s.cfg.Static.Dir) - - err = s.cfg.Valid() - if err != nil { - return errors.E(op, err) - } - - // create 2 hashmaps with the allowed and forbidden file extensions - s.allowedExtensions = make(map[string]struct{}, len(s.cfg.Static.Allow)) - s.forbiddenExtensions = make(map[string]struct{}, len(s.cfg.Static.Forbid)) - s.alwaysServe = make(map[string]struct{}, len(s.cfg.Static.Always)) - - for i := 0; i < len(s.cfg.Static.Forbid); i++ { - s.forbiddenExtensions[s.cfg.Static.Forbid[i]] = struct{}{} - } - - for i := 0; i < len(s.cfg.Static.Allow); i++ { - s.forbiddenExtensions[s.cfg.Static.Allow[i]] = struct{}{} - } - - // check if any forbidden items presented in the allowed - // if presented, delete such items from allowed - for k := range s.forbiddenExtensions { - if _, ok := s.allowedExtensions[k]; ok { - delete(s.allowedExtensions, k) - } - } - - for i := 0; i < len(s.cfg.Static.Always); i++ { - s.alwaysServe[s.cfg.Static.Always[i]] = struct{}{} - } - - // at this point we have distinct allowed and forbidden hashmaps, also with alwaysServed - - return nil -} - -func (s *Plugin) Name() string { - return PluginName -} - -// Middleware must return true if request/response pair is handled within the middleware. -func (s *Plugin) Middleware(next http.Handler) http.HandlerFunc { - // Define the http.HandlerFunc - return func(w http.ResponseWriter, r *http.Request) { - if s.cfg.Static.Request != nil { - for k, v := range s.cfg.Static.Request { - r.Header.Add(k, v) - } - } - - if s.cfg.Static.Response != nil { - for k, v := range s.cfg.Static.Response { - w.Header().Set(k, v) - } - } - - fPath := path.Clean(r.URL.Path) - ext := strings.ToLower(path.Ext(fPath)) - - // check that file is in forbidden list - if _, ok := s.forbiddenExtensions[ext]; ok { - http.Error(w, "file is forbidden", 404) - return - } - - f, err := s.root.Open(fPath) - if err != nil { - // if we should always serve files with some extensions - // show error to the user and invoke next middleware - if _, ok := s.alwaysServe[ext]; ok { - //http.Error(w, err.Error(), 404) - w.WriteHeader(404) - next.ServeHTTP(w, r) - return - } - // else, return with error - http.Error(w, err.Error(), 404) - return - } - - defer func() { - err = f.Close() - if err != nil { - s.log.Error("file close error", "error", err) - } - }() - - // here we know, that file extension is not in the AlwaysServe and file exists - // (or by some reason, there is no error from the http.Open method) - - // if we have some allowed extensions, we should check them - if len(s.allowedExtensions) > 0 { - if _, ok := s.allowedExtensions[ext]; ok { - d, err := s.check(f) - if err != nil { - http.Error(w, err.Error(), 404) - return - } - - http.ServeContent(w, r, d.Name(), d.ModTime(), f) - } - // otherwise we guess, that all file extensions are allowed - } else { - d, err := s.check(f) - if err != nil { - http.Error(w, err.Error(), 404) - return - } - - http.ServeContent(w, r, d.Name(), d.ModTime(), f) - } - - next.ServeHTTP(w, r) - } -} - -func (s *Plugin) check(f http.File) (fs.FileInfo, error) { - const op = errors.Op("http_file_check") - d, err := f.Stat() - if err != nil { - return nil, err - } - - // do not serve directories - if d.IsDir() { - return nil, errors.E(op, errors.Str("directory path provided, should be path to the file")) - } - - return d, nil -} diff --git a/tests/plugins/http/configs/.rr-http-static-disabled.yaml b/tests/plugins/http/configs/.rr-http-static-disabled.yaml new file mode 100644 index 00000000..d248ce48 --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static-disabled.yaml @@ -0,0 +1,27 @@ +server: + command: "php ../../http/client.php pid pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:21234 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + static: + dir: "abc" #not exists + forbid: [ ".php", ".htaccess" ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/configs/.rr-http-static-files-disable.yaml b/tests/plugins/http/configs/.rr-http-static-files-disable.yaml new file mode 100644 index 00000000..9f91d75b --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static-files-disable.yaml @@ -0,0 +1,24 @@ +server: + command: "php ../../http/client.php echo pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:45877 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/configs/.rr-http-static-files.yaml b/tests/plugins/http/configs/.rr-http-static-files.yaml new file mode 100644 index 00000000..5d8b50e8 --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static-files.yaml @@ -0,0 +1,30 @@ +server: + command: "php ../../http/client.php echo pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:34653 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + static: + dir: "../../../" + pattern: "/tests/" + allow: [ ".ico" ] + forbid: [ ".php", ".htaccess" ] + + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/configs/.rr-http-static.yaml b/tests/plugins/http/configs/.rr-http-static.yaml new file mode 100644 index 00000000..9bb2abc0 --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static.yaml @@ -0,0 +1,33 @@ +server: + command: "php ../../http/client.php pid pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:21603 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + static: + dir: "../../../" + pattern: "/tests/" + forbid: [ "" ] + allow: [ ".txt", ".php" ] + request: + "input": "custom-header" + response: + "output": "output-header" + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index 0e43dac4..8bfaa2e8 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "crypto/tls" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -23,6 +24,7 @@ import ( "github.com/spiral/roadrunner/v2/pkg/events" "github.com/spiral/roadrunner/v2/pkg/process" "github.com/spiral/roadrunner/v2/plugins/config" + "github.com/spiral/roadrunner/v2/plugins/gzip" "github.com/spiral/roadrunner/v2/plugins/informer" "github.com/spiral/roadrunner/v2/plugins/logger" "github.com/spiral/roadrunner/v2/plugins/resetter" @@ -1397,21 +1399,6 @@ func informerTestAfter(t *testing.T) { assert.NotEqual(t, workerPid, list.Workers[0].Pid) } -func get(url string) (string, *http.Response, error) { - r, err := http.Get(url) //nolint:gosec - if err != nil { - return "", nil, err - } - b, err := ioutil.ReadAll(r.Body) - if err != nil { - return "", nil, err - } - defer func() { - _ = r.Body.Close() - }() - return string(b), r, err -} - // get request and return body func getHeader(url string, h map[string]string) (string, *http.Response, error) { req, err := http.NewRequest("GET", url, bytes.NewBuffer(nil)) @@ -1574,3 +1561,360 @@ func bigEchoHTTP(t *testing.T) { err = r.Body.Close() assert.NoError(t, err) } + +func TestStaticPlugin(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static.yaml", + Prefix: "rr", + } + + err = cont.RegisterAll( + cfg, + &logger.ZapLogger{}, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + + err = cont.Init() + if err != nil { + t.Fatal(err) + } + + ch, err := cont.Serve() + assert.NoError(t, err) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + wg := &sync.WaitGroup{} + wg.Add(1) + + stopCh := make(chan struct{}, 1) + + go func() { + defer wg.Done() + for { + select { + case e := <-ch: + assert.Fail(t, "error", e.Error.Error()) + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + case <-sig: + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + case <-stopCh: + // timeout + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + } + } + }() + + time.Sleep(time.Second) + t.Run("ServeSample", serveStaticSample) + t.Run("StaticNotForbid", staticNotForbid) + t.Run("StaticHeaders", staticHeaders) + + stopCh <- struct{}{} + wg.Wait() +} + +func staticHeaders(t *testing.T) { + req, err := http.NewRequest("GET", "http://localhost:21603/tests/client.php", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + + if resp.Header.Get("Output") != "output-header" { + t.Fatal("can't find output header in response") + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + defer func() { + _ = resp.Body.Close() + }() + + assert.Equal(t, all("../../../tests/client.php"), string(b)) + assert.Equal(t, all("../../../tests/client.php"), string(b)) +} + +func staticNotForbid(t *testing.T) { + b, r, err := get("http://localhost:21603/tests/client.php") + assert.NoError(t, err) + assert.Equal(t, all("../../../tests/client.php"), b) + assert.Equal(t, all("../../../tests/client.php"), b) + _ = r.Body.Close() +} + +func serveStaticSample(t *testing.T) { + b, r, err := get("http://localhost:21603/tests/sample.txt") + assert.NoError(t, err) + assert.Equal(t, "sample", b) + _ = r.Body.Close() +} + +func TestStaticDisabled_Error(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static-disabled.yaml", + Prefix: "rr", + } + + err = cont.RegisterAll( + cfg, + &logger.ZapLogger{}, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + assert.Error(t, cont.Init()) +} + +func TestStaticFilesDisabled(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static-files-disable.yaml", + Prefix: "rr", + } + + err = cont.RegisterAll( + cfg, + &logger.ZapLogger{}, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + + err = cont.Init() + if err != nil { + t.Fatal(err) + } + + ch, err := cont.Serve() + assert.NoError(t, err) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + wg := &sync.WaitGroup{} + wg.Add(1) + + stopCh := make(chan struct{}, 1) + + go func() { + defer wg.Done() + for { + select { + case e := <-ch: + assert.Fail(t, "error", e.Error.Error()) + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + case <-sig: + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + case <-stopCh: + // timeout + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + } + } + }() + + time.Sleep(time.Second) + t.Run("StaticFilesDisabled", staticFilesDisabled) + + stopCh <- struct{}{} + wg.Wait() +} + +func staticFilesDisabled(t *testing.T) { + b, r, err := get("http://localhost:45877/client.php?hello=world") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "WORLD", b) + _ = r.Body.Close() +} + +func TestStaticFilesForbid(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static-files.yaml", + Prefix: "rr", + } + + controller := gomock.NewController(t) + mockLogger := mocks.NewMockLogger(controller) + + mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/http?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/client.XXX?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/client.php?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Error("file open error", "error", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() // placeholder for the workerlogerror + + err = cont.RegisterAll( + cfg, + mockLogger, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + + err = cont.Init() + if err != nil { + t.Fatal(err) + } + + ch, err := cont.Serve() + assert.NoError(t, err) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + wg := &sync.WaitGroup{} + wg.Add(1) + + stopCh := make(chan struct{}, 1) + + go func() { + defer wg.Done() + for { + select { + case e := <-ch: + assert.Fail(t, "error", e.Error.Error()) + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + case <-sig: + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + case <-stopCh: + // timeout + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + } + } + }() + + time.Sleep(time.Second) + t.Run("StaticTestFilesDir", staticTestFilesDir) + t.Run("StaticNotFound", staticNotFound) + t.Run("StaticFilesForbid", staticFilesForbid) + t.Run("StaticFilesAlways", staticFilesAlways) + + stopCh <- struct{}{} + wg.Wait() +} + +func staticTestFilesDir(t *testing.T) { + b, r, err := get("http://localhost:34653/tests/http?hello=world") + assert.NoError(t, err) + assert.Equal(t, "403 Forbidden\n", b) + _ = r.Body.Close() +} + +func staticNotFound(t *testing.T) { + b, _, _ := get("http://localhost:34653/tests/client.XXX?hello=world") //nolint:bodyclose + assert.Equal(t, "404 page not found\n", b) +} + +func staticFilesAlways(t *testing.T) { + _, r, err := get("http://localhost:34653/tests/favicon.ico") + assert.NoError(t, err) + assert.Equal(t, 404, r.StatusCode) + _ = r.Body.Close() +} + +func staticFilesForbid(t *testing.T) { + b, r, err := get("http://localhost:34653/tests/client.php?hello=world") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "403 Forbidden\n", b) + _ = r.Body.Close() +} + +// HELPERS +func get(url string) (string, *http.Response, error) { + r, err := http.Get(url) //nolint:gosec + if err != nil { + return "", nil, err + } + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return "", nil, err + } + + err = r.Body.Close() + if err != nil { + return "", nil, err + } + + return string(b), r, err +} + +func all(fn string) string { + f, _ := os.Open(fn) + + b := new(bytes.Buffer) + _, err := io.Copy(b, f) + if err != nil { + return "" + } + + err = f.Close() + if err != nil { + return "" + } + + return b.String() +} diff --git a/tests/plugins/static/configs/.rr-http-static-disabled.yaml b/tests/plugins/static/configs/.rr-http-static-disabled.yaml deleted file mode 100644 index a85bc408..00000000 --- a/tests/plugins/static/configs/.rr-http-static-disabled.yaml +++ /dev/null @@ -1,31 +0,0 @@ -server: - command: "php ../../http/client.php pid pipes" - user: "" - group: "" - env: - "RR_HTTP": "true" - relay: "pipes" - relay_timeout: "20s" - -http: - address: 127.0.0.1:21234 - max_request_size: 1024 - middleware: [ "gzip" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - uploads: - forbid: [ ".php", ".exe", ".bat" ] - static: - dir: "abc" #not exists - forbid: [ ".php", ".htaccess" ] - request: - Example-Request-Header: "Value" - response: - X-Powered-By: "RoadRunner" - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s -logs: - mode: development - level: error \ No newline at end of file diff --git a/tests/plugins/static/configs/.rr-http-static-files-disable.yaml b/tests/plugins/static/configs/.rr-http-static-files-disable.yaml deleted file mode 100644 index 6ba47c91..00000000 --- a/tests/plugins/static/configs/.rr-http-static-files-disable.yaml +++ /dev/null @@ -1,32 +0,0 @@ -server: - command: "php ../../http/client.php echo pipes" - user: "" - group: "" - env: - "RR_HTTP": "true" - relay: "pipes" - relay_timeout: "20s" - -http: - address: 127.0.0.1:45877 - max_request_size: 1024 - middleware: [ "gzip" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - uploads: - forbid: [ ".php", ".exe", ".bat" ] - static: - dir: "../../../tests" - forbid: [ ".php" ] - request: - Example-Request-Header: "Value" - # Automatically add headers to every response. - response: - X-Powered-By: "RoadRunner" - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s -logs: - mode: development - level: error \ No newline at end of file diff --git a/tests/plugins/static/configs/.rr-http-static-files.yaml b/tests/plugins/static/configs/.rr-http-static-files.yaml deleted file mode 100644 index 0e003dae..00000000 --- a/tests/plugins/static/configs/.rr-http-static-files.yaml +++ /dev/null @@ -1,29 +0,0 @@ -server: - command: "php ../../http/client.php echo pipes" - user: "" - group: "" - env: - "RR_HTTP": "true" - relay: "pipes" - relay_timeout: "20s" - -http: - address: 127.0.0.1:34653 - max_request_size: 1024 - middleware: [ "gzip", "static" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - uploads: - forbid: [ ".php", ".exe", ".bat" ] - static: - dir: "../../../tests" - forbid: [ ".php", ".htaccess" ] - always: [ ".ico" ] - - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s -logs: - mode: development - level: error diff --git a/tests/plugins/static/configs/.rr-http-static.yaml b/tests/plugins/static/configs/.rr-http-static.yaml deleted file mode 100644 index e5af9043..00000000 --- a/tests/plugins/static/configs/.rr-http-static.yaml +++ /dev/null @@ -1,31 +0,0 @@ -server: - command: "php ../../http/client.php pid pipes" - user: "" - group: "" - env: - "RR_HTTP": "true" - relay: "pipes" - relay_timeout: "20s" - -http: - address: 127.0.0.1:21603 - max_request_size: 1024 - middleware: [ "gzip", "static" ] - trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] - uploads: - forbid: [ ".php", ".exe", ".bat" ] - static: - dir: "../../../tests" - forbid: [ "" ] - request: - "input": "custom-header" - response: - "output": "output-header" - pool: - num_workers: 2 - max_jobs: 0 - allocate_timeout: 60s - destroy_timeout: 60s -logs: - mode: development - level: error \ No newline at end of file diff --git a/tests/plugins/static/static_plugin_test.go b/tests/plugins/static/static_plugin_test.go deleted file mode 100644 index b58f1f6b..00000000 --- a/tests/plugins/static/static_plugin_test.go +++ /dev/null @@ -1,387 +0,0 @@ -package static - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "os" - "os/signal" - "sync" - "syscall" - "testing" - "time" - - "github.com/golang/mock/gomock" - endure "github.com/spiral/endure/pkg/container" - "github.com/spiral/roadrunner/v2/plugins/config" - "github.com/spiral/roadrunner/v2/plugins/gzip" - httpPlugin "github.com/spiral/roadrunner/v2/plugins/http" - "github.com/spiral/roadrunner/v2/plugins/logger" - "github.com/spiral/roadrunner/v2/plugins/server" - "github.com/spiral/roadrunner/v2/plugins/static" - "github.com/spiral/roadrunner/v2/tests/mocks" - "github.com/stretchr/testify/assert" -) - -func TestStaticPlugin(t *testing.T) { - cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) - assert.NoError(t, err) - - cfg := &config.Viper{ - Path: "configs/.rr-http-static.yaml", - Prefix: "rr", - } - - err = cont.RegisterAll( - cfg, - &logger.ZapLogger{}, - &server.Plugin{}, - &httpPlugin.Plugin{}, - &gzip.Plugin{}, - &static.Plugin{}, - ) - assert.NoError(t, err) - - err = cont.Init() - if err != nil { - t.Fatal(err) - } - - ch, err := cont.Serve() - assert.NoError(t, err) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - wg := &sync.WaitGroup{} - wg.Add(1) - - stopCh := make(chan struct{}, 1) - - go func() { - defer wg.Done() - for { - select { - case e := <-ch: - assert.Fail(t, "error", e.Error.Error()) - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - case <-sig: - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - case <-stopCh: - // timeout - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - } - } - }() - - time.Sleep(time.Second) - t.Run("ServeSample", serveStaticSample) - t.Run("StaticNotForbid", staticNotForbid) - t.Run("StaticHeaders", staticHeaders) - - stopCh <- struct{}{} - wg.Wait() -} - -func staticHeaders(t *testing.T) { - req, err := http.NewRequest("GET", "http://localhost:21603/client.php", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - - if resp.Header.Get("Output") != "output-header" { - t.Fatal("can't find output header in response") - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - - defer func() { - _ = resp.Body.Close() - }() - - assert.Equal(t, all("../../../tests/client.php"), string(b)) - assert.Equal(t, all("../../../tests/client.php"), string(b)) -} - -func staticNotForbid(t *testing.T) { - b, r, err := get("http://localhost:21603/client.php") - assert.NoError(t, err) - assert.Equal(t, all("../../../tests/client.php"), b) - assert.Equal(t, all("../../../tests/client.php"), b) - _ = r.Body.Close() -} - -func serveStaticSample(t *testing.T) { - b, r, err := get("http://localhost:21603/sample.txt") - assert.NoError(t, err) - assert.Equal(t, "sample", b) - _ = r.Body.Close() -} - -func TestStaticDisabled_Error(t *testing.T) { - cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) - assert.NoError(t, err) - - cfg := &config.Viper{ - Path: "configs/.rr-http-static-disabled.yaml", - Prefix: "rr", - } - - err = cont.RegisterAll( - cfg, - &logger.ZapLogger{}, - &server.Plugin{}, - &httpPlugin.Plugin{}, - &gzip.Plugin{}, - &static.Plugin{}, - ) - assert.NoError(t, err) - assert.Error(t, cont.Init()) -} - -func TestStaticFilesDisabled(t *testing.T) { - cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) - assert.NoError(t, err) - - cfg := &config.Viper{ - Path: "configs/.rr-http-static-files-disable.yaml", - Prefix: "rr", - } - - err = cont.RegisterAll( - cfg, - &logger.ZapLogger{}, - &server.Plugin{}, - &httpPlugin.Plugin{}, - &gzip.Plugin{}, - &static.Plugin{}, - ) - assert.NoError(t, err) - - err = cont.Init() - if err != nil { - t.Fatal(err) - } - - ch, err := cont.Serve() - assert.NoError(t, err) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - wg := &sync.WaitGroup{} - wg.Add(1) - - stopCh := make(chan struct{}, 1) - - go func() { - defer wg.Done() - for { - select { - case e := <-ch: - assert.Fail(t, "error", e.Error.Error()) - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - case <-sig: - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - case <-stopCh: - // timeout - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - } - } - }() - - time.Sleep(time.Second) - t.Run("StaticFilesDisabled", staticFilesDisabled) - - stopCh <- struct{}{} - wg.Wait() -} - -func staticFilesDisabled(t *testing.T) { - b, r, err := get("http://localhost:45877/client.php?hello=world") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "WORLD", b) - _ = r.Body.Close() -} - -func TestStaticFilesForbid(t *testing.T) { - cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) - assert.NoError(t, err) - - cfg := &config.Viper{ - Path: "configs/.rr-http-static-files.yaml", - Prefix: "rr", - } - - controller := gomock.NewController(t) - mockLogger := mocks.NewMockLogger(controller) - - mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes() - mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes() - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/http?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/client.XXX?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/client.php?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) - mockLogger.EXPECT().Error("file open error", "error", gomock.Any()).AnyTimes() - mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() // placeholder for the workerlogerror - - err = cont.RegisterAll( - cfg, - //mockLogger, - &logger.ZapLogger{}, - &server.Plugin{}, - &httpPlugin.Plugin{}, - &gzip.Plugin{}, - &static.Plugin{}, - ) - assert.NoError(t, err) - - err = cont.Init() - if err != nil { - t.Fatal(err) - } - - ch, err := cont.Serve() - assert.NoError(t, err) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - wg := &sync.WaitGroup{} - wg.Add(1) - - stopCh := make(chan struct{}, 1) - - go func() { - defer wg.Done() - for { - select { - case e := <-ch: - assert.Fail(t, "error", e.Error.Error()) - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - case <-sig: - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - case <-stopCh: - // timeout - err = cont.Stop() - if err != nil { - assert.FailNow(t, "error", err.Error()) - } - return - } - } - }() - - time.Sleep(time.Second) - t.Run("StaticTestFilesDir", staticTestFilesDir) - t.Run("StaticNotFound", staticNotFound) - t.Run("StaticFilesForbid", staticFilesForbid) - t.Run("StaticFilesAlways", staticFilesAlways) - - stopCh <- struct{}{} - wg.Wait() -} - -func staticTestFilesDir(t *testing.T) { - b, r, err := get("http://localhost:34653/http?hello=world") - assert.NoError(t, err) - assert.Equal(t, "WORLD", b) - _ = r.Body.Close() -} - -func staticNotFound(t *testing.T) { - b, _, _ := get("http://localhost:34653/client.XXX?hello=world") //nolint:bodyclose - assert.Equal(t, "WORLD", b) -} - -func staticFilesAlways(t *testing.T) { - _, r, err := get("http://localhost:34653/favicon.ico") - assert.NoError(t, err) - assert.Equal(t, 404, r.StatusCode) - _ = r.Body.Close() -} - -func staticFilesForbid(t *testing.T) { - b, r, err := get("http://localhost:34653/client.php?hello=world") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "WORLD", b) - _ = r.Body.Close() -} - -// HELPERS -func get(url string) (string, *http.Response, error) { - r, err := http.Get(url) //nolint:gosec - if err != nil { - return "", nil, err - } - - b, err := ioutil.ReadAll(r.Body) - if err != nil { - return "", nil, err - } - - err = r.Body.Close() - if err != nil { - return "", nil, err - } - - return string(b), r, err -} - -func all(fn string) string { - f, _ := os.Open(fn) - - b := new(bytes.Buffer) - _, err := io.Copy(b, f) - if err != nil { - return "" - } - - err = f.Close() - if err != nil { - return "" - } - - return b.String() -} -- cgit v1.2.3 From b1a7f0cafa9eb10fbb6ac7dc79ec7da8257051ea Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Mon, 26 Apr 2021 21:33:35 +0300 Subject: - Bump golang version in the go.mod Signed-off-by: Valery Piashchynski --- .github/workflows/linux.yml | 2 +- tests/plugins/http/http_plugin_test.go | 2 +- tests/sample.txt | 1 - tests/static/sample.txt | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 tests/sample.txt create mode 100644 tests/static/sample.txt diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f2b203f4..bb7d646b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -11,7 +11,7 @@ jobs: fail-fast: true matrix: php: [ "7.4", "8.0" ] - go: [ "1.15", "1.16" ] + go: [ "1.16" ] os: [ ubuntu-20.04 ] steps: - name: Set up Go ${{ matrix.go }} diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index 8bfaa2e8..000fd25d 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -1669,7 +1669,7 @@ func staticNotForbid(t *testing.T) { } func serveStaticSample(t *testing.T) { - b, r, err := get("http://localhost:21603/tests/sample.txt") + b, r, err := get("http://localhost:21603/tests/static/sample.txt") assert.NoError(t, err) assert.Equal(t, "sample", b) _ = r.Body.Close() diff --git a/tests/sample.txt b/tests/sample.txt deleted file mode 100644 index eed7e79a..00000000 --- a/tests/sample.txt +++ /dev/null @@ -1 +0,0 @@ -sample \ No newline at end of file diff --git a/tests/static/sample.txt b/tests/static/sample.txt new file mode 100644 index 00000000..eed7e79a --- /dev/null +++ b/tests/static/sample.txt @@ -0,0 +1 @@ +sample \ No newline at end of file -- cgit v1.2.3 From d3912947d8c5500649b86e886623c434e24cacc8 Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Mon, 26 Apr 2021 21:41:37 +0300 Subject: - Update default pattern --- plugins/http/config/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/http/config/http.go b/plugins/http/config/http.go index 31b10322..53a22f3c 100644 --- a/plugins/http/config/http.go +++ b/plugins/http/config/http.go @@ -106,7 +106,7 @@ func (c *HTTP) InitDefaults() error { // static files if c.Static != nil { if c.Static.Pattern == "" { - c.Static.Pattern = "/static" + c.Static.Pattern = "/static/" } } -- cgit v1.2.3 From 4cb2247f909d02c922edb6f8e3d3741cc5a2c077 Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Mon, 26 Apr 2021 21:49:52 +0300 Subject: - Update defaults Signed-off-by: Valery Piashchynski --- plugins/http/config/http.go | 3 +++ plugins/http/config/static.go | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/http/config/http.go b/plugins/http/config/http.go index 53a22f3c..59735e2e 100644 --- a/plugins/http/config/http.go +++ b/plugins/http/config/http.go @@ -108,6 +108,9 @@ func (c *HTTP) InitDefaults() error { if c.Static.Pattern == "" { c.Static.Pattern = "/static/" } + if c.Static.Dir == "" { + c.Static.Dir = "." + } } err := c.HTTP2Config.InitDefaults() diff --git a/plugins/http/config/static.go b/plugins/http/config/static.go index 416169d2..e9acc3e4 100644 --- a/plugins/http/config/static.go +++ b/plugins/http/config/static.go @@ -9,11 +9,12 @@ import ( // Static describes file location and controls access to them. type Static struct { // Dir contains name of directory to control access to. + // Default - "." Dir string // HTTP pattern, where to serve static files - // for example - `/static`, `/my-files/static`, etc - // Default - /static + // for example - `/static/`, `/my-files/static/`, etc + // Default - /static/ Pattern string // forbid specifies list of file extensions which are forbidden for access. -- cgit v1.2.3 From 30c25f17fa7d6386e33a4894c812f7ca5db990ad Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Wed, 28 Apr 2021 14:10:27 +0300 Subject: - Fix middleware order - Update tests - Move worker handler into a separate folder with separate package name Signed-off-by: Valery Piashchynski --- plugins/gzip/plugin.go | 6 +- plugins/headers/plugin.go | 6 +- plugins/http/constants.go | 8 - plugins/http/errors.go | 25 --- plugins/http/errors_windows.go | 27 --- plugins/http/handler.go | 217 ------------------------ plugins/http/parse.go | 149 ---------------- plugins/http/plugin.go | 27 +-- plugins/http/request.go | 187 -------------------- plugins/http/response.go | 105 ------------ plugins/http/static.go | 88 ---------- plugins/http/static/static.go | 88 ++++++++++ plugins/http/uploads.go | 159 ----------------- plugins/http/worker_handler/constants.go | 8 + plugins/http/worker_handler/errors.go | 25 +++ plugins/http/worker_handler/errors_windows.go | 27 +++ plugins/http/worker_handler/handler.go | 217 ++++++++++++++++++++++++ plugins/http/worker_handler/parse.go | 149 ++++++++++++++++ plugins/http/worker_handler/request.go | 187 ++++++++++++++++++++ plugins/http/worker_handler/response.go | 105 ++++++++++++ plugins/http/worker_handler/uploads.go | 159 +++++++++++++++++ tests/plugins/http/configs/.rr-http-static.yaml | 4 +- tests/plugins/http/handler_test.go | 64 +++---- tests/plugins/http/http_plugin_test.go | 2 +- tests/plugins/http/parse_test.go | 6 +- tests/plugins/http/plugin_middleware.go | 12 +- tests/plugins/http/response_test.go | 18 +- tests/plugins/http/uploads_test.go | 10 +- tests/static/sample.txt | 2 +- 29 files changed, 1044 insertions(+), 1043 deletions(-) delete mode 100644 plugins/http/constants.go delete mode 100644 plugins/http/errors.go delete mode 100644 plugins/http/errors_windows.go delete mode 100644 plugins/http/handler.go delete mode 100644 plugins/http/parse.go delete mode 100644 plugins/http/request.go delete mode 100644 plugins/http/response.go delete mode 100644 plugins/http/static.go create mode 100644 plugins/http/static/static.go delete mode 100644 plugins/http/uploads.go create mode 100644 plugins/http/worker_handler/constants.go create mode 100644 plugins/http/worker_handler/errors.go create mode 100644 plugins/http/worker_handler/errors_windows.go create mode 100644 plugins/http/worker_handler/handler.go create mode 100644 plugins/http/worker_handler/parse.go create mode 100644 plugins/http/worker_handler/request.go create mode 100644 plugins/http/worker_handler/response.go create mode 100644 plugins/http/worker_handler/uploads.go diff --git a/plugins/gzip/plugin.go b/plugins/gzip/plugin.go index 949c6888..18ee7b88 100644 --- a/plugins/gzip/plugin.go +++ b/plugins/gzip/plugin.go @@ -15,10 +15,10 @@ func (g *Plugin) Init() error { return nil } -func (g *Plugin) Middleware(next http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func (g *Plugin) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gziphandler.GzipHandler(next).ServeHTTP(w, r) - } + }) } func (g *Plugin) Name() string { diff --git a/plugins/headers/plugin.go b/plugins/headers/plugin.go index a5ee702f..dea0d127 100644 --- a/plugins/headers/plugin.go +++ b/plugins/headers/plugin.go @@ -38,9 +38,9 @@ func (s *Plugin) Init(cfg config.Configurer) error { } // middleware must return true if request/response pair is handled within the middleware. -func (s *Plugin) Middleware(next http.Handler) http.HandlerFunc { +func (s *Plugin) Middleware(next http.Handler) http.Handler { // Define the http.HandlerFunc - return func(w http.ResponseWriter, r *http.Request) { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if s.cfg.Headers.Request != nil { for k, v := range s.cfg.Headers.Request { r.Header.Add(k, v) @@ -62,7 +62,7 @@ func (s *Plugin) Middleware(next http.Handler) http.HandlerFunc { } next.ServeHTTP(w, r) - } + }) } func (s *Plugin) Name() string { diff --git a/plugins/http/constants.go b/plugins/http/constants.go deleted file mode 100644 index c3d5c589..00000000 --- a/plugins/http/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -package http - -import "net/http" - -var http2pushHeaderKey = http.CanonicalHeaderKey("http2-push") - -// TrailerHeaderKey http header key -var TrailerHeaderKey = http.CanonicalHeaderKey("trailer") diff --git a/plugins/http/errors.go b/plugins/http/errors.go deleted file mode 100644 index 5889aa76..00000000 --- a/plugins/http/errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build !windows - -package http - -import ( - "errors" - "net" - "os" - "syscall" -) - -// Broken pipe -var errEPIPE = errors.New("EPIPE(32) -> connection reset by peer") - -// handleWriteError just check if error was caused by aborted connection on linux -func handleWriteError(err error) error { - if netErr, ok2 := err.(*net.OpError); ok2 { - if syscallErr, ok3 := netErr.Err.(*os.SyscallError); ok3 { - if errors.Is(syscallErr.Err, syscall.EPIPE) { - return errEPIPE - } - } - } - return err -} diff --git a/plugins/http/errors_windows.go b/plugins/http/errors_windows.go deleted file mode 100644 index 3d0ba04c..00000000 --- a/plugins/http/errors_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build windows - -package http - -import ( - "errors" - "net" - "os" - "syscall" -) - -//Software caused connection abort. -//An established connection was aborted by the software in your host computer, -//possibly due to a data transmission time-out or protocol error. -var errEPIPE = errors.New("WSAECONNABORTED (10053) -> an established connection was aborted by peer") - -// handleWriteError just check if error was caused by aborted connection on windows -func handleWriteError(err error) error { - if netErr, ok2 := err.(*net.OpError); ok2 { - if syscallErr, ok3 := netErr.Err.(*os.SyscallError); ok3 { - if syscallErr.Err == syscall.WSAECONNABORTED { - return errEPIPE - } - } - } - return err -} diff --git a/plugins/http/handler.go b/plugins/http/handler.go deleted file mode 100644 index d3c928aa..00000000 --- a/plugins/http/handler.go +++ /dev/null @@ -1,217 +0,0 @@ -package http - -import ( - "net" - "net/http" - "strconv" - "strings" - "sync" - "time" - - "github.com/spiral/errors" - "github.com/spiral/roadrunner/v2/pkg/events" - "github.com/spiral/roadrunner/v2/pkg/pool" - "github.com/spiral/roadrunner/v2/plugins/http/config" - "github.com/spiral/roadrunner/v2/plugins/logger" -) - -// MB is 1024 bytes -const MB uint64 = 1024 * 1024 - -// ErrorEvent represents singular http error event. -type ErrorEvent struct { - // Request contains client request, must not be stored. - Request *http.Request - - // Error - associated error, if any. - Error error - - // event timings - start time.Time - elapsed time.Duration -} - -// Elapsed returns duration of the invocation. -func (e *ErrorEvent) Elapsed() time.Duration { - return e.elapsed -} - -// ResponseEvent represents singular http response event. -type ResponseEvent struct { - // Request contains client request, must not be stored. - Request *Request - - // Response contains service response. - Response *Response - - // event timings - start time.Time - elapsed time.Duration -} - -// Elapsed returns duration of the invocation. -func (e *ResponseEvent) Elapsed() time.Duration { - return e.elapsed -} - -// Handler serves http connections to underlying PHP application using PSR-7 protocol. Context will include request headers, -// parsed files and query, payload will include parsed form dataTree (if any). -type Handler struct { - maxRequestSize uint64 - uploads config.Uploads - trusted config.Cidrs - log logger.Logger - pool pool.Pool - mul sync.Mutex - lsn events.Listener -} - -// NewHandler return handle interface implementation -func NewHandler(maxReqSize uint64, uploads config.Uploads, trusted config.Cidrs, pool pool.Pool) (*Handler, error) { - if pool == nil { - return nil, errors.E(errors.Str("pool should be initialized")) - } - return &Handler{ - maxRequestSize: maxReqSize * MB, - uploads: uploads, - pool: pool, - trusted: trusted, - }, nil -} - -// AddListener attaches handler event controller. -func (h *Handler) AddListener(l events.Listener) { - h.mul.Lock() - defer h.mul.Unlock() - - h.lsn = l -} - -// mdwr serve using PSR-7 requests passed to underlying application. Attempts to serve static files first if enabled. -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - const op = errors.Op("http_plugin_serve_http") - start := time.Now() - - // validating request size - if h.maxRequestSize != 0 { - const op = errors.Op("http_handler_max_size") - if length := r.Header.Get("content-length"); length != "" { - // try to parse the value from the `content-length` header - size, err := strconv.ParseInt(length, 10, 64) - if err != nil { - // if got an error while parsing -> assign 500 code to the writer and return - http.Error(w, errors.E(op, err).Error(), 500) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, errors.Str("error while parsing value from the `content-length` header")), start: start, elapsed: time.Since(start)}) - return - } - - if size > int64(h.maxRequestSize) { - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, errors.Str("request body max size is exceeded")), start: start, elapsed: time.Since(start)}) - http.Error(w, errors.E(op, errors.Str("request body max size is exceeded")).Error(), 500) - return - } - } - } - - req, err := NewRequest(r, h.uploads) - if err != nil { - // if pipe is broken, there is no sense to write the header - // in this case we just report about error - if err == errEPIPE { - h.sendEvent(ErrorEvent{Request: r, Error: err, start: start, elapsed: time.Since(start)}) - return - } - - http.Error(w, errors.E(op, err).Error(), 500) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) - return - } - - // proxy IP resolution - h.resolveIP(req) - - req.Open(h.log) - defer req.Close(h.log) - - p, err := req.Payload() - if err != nil { - http.Error(w, errors.E(op, err).Error(), 500) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) - return - } - - rsp, err := h.pool.Exec(p) - if err != nil { - http.Error(w, errors.E(op, err).Error(), 500) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) - return - } - - resp, err := NewResponse(rsp) - if err != nil { - http.Error(w, errors.E(op, err).Error(), resp.Status) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) - return - } - - h.handleResponse(req, resp, start) - err = resp.Write(w) - if err != nil { - http.Error(w, errors.E(op, err).Error(), 500) - h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) - } -} - -// handleResponse triggers response event. -func (h *Handler) handleResponse(req *Request, resp *Response, start time.Time) { - h.sendEvent(ResponseEvent{Request: req, Response: resp, start: start, elapsed: time.Since(start)}) -} - -// sendEvent invokes event handler if any. -func (h *Handler) sendEvent(event interface{}) { - if h.lsn != nil { - h.lsn(event) - } -} - -// get real ip passing multiple proxy -func (h *Handler) resolveIP(r *Request) { - if h.trusted.IsTrusted(r.RemoteAddr) == false { //nolint:gosimple - return - } - - if r.Header.Get("X-Forwarded-For") != "" { - ips := strings.Split(r.Header.Get("X-Forwarded-For"), ",") - ipCount := len(ips) - - for i := ipCount - 1; i >= 0; i-- { - addr := strings.TrimSpace(ips[i]) - if net.ParseIP(addr) != nil { - r.RemoteAddr = addr - return - } - } - - return - } - - // The logic here is the following: - // In general case, we only expect X-Real-Ip header. If it exist, we get the IP address from header and set request Remote address - // But, if there is no X-Real-Ip header, we also trying to check CloudFlare headers - // True-Client-IP is a general CF header in which copied information from X-Real-Ip in CF. - // CF-Connecting-IP is an Enterprise feature and we check it last in order. - // This operations are near O(1) because Headers struct are the map type -> type MIMEHeader map[string][]string - if r.Header.Get("X-Real-Ip") != "" { - r.RemoteAddr = fetchIP(r.Header.Get("X-Real-Ip")) - return - } - - if r.Header.Get("True-Client-IP") != "" { - r.RemoteAddr = fetchIP(r.Header.Get("True-Client-IP")) - return - } - - if r.Header.Get("CF-Connecting-IP") != "" { - r.RemoteAddr = fetchIP(r.Header.Get("CF-Connecting-IP")) - } -} diff --git a/plugins/http/parse.go b/plugins/http/parse.go deleted file mode 100644 index 780e1279..00000000 --- a/plugins/http/parse.go +++ /dev/null @@ -1,149 +0,0 @@ -package http - -import ( - "net/http" - - "github.com/spiral/roadrunner/v2/plugins/http/config" -) - -// MaxLevel defines maximum tree depth for incoming request data and files. -const MaxLevel = 127 - -type dataTree map[string]interface{} -type fileTree map[string]interface{} - -// parseData parses incoming request body into data tree. -func parseData(r *http.Request) dataTree { - data := make(dataTree) - if r.PostForm != nil { - for k, v := range r.PostForm { - data.push(k, v) - } - } - - if r.MultipartForm != nil { - for k, v := range r.MultipartForm.Value { - data.push(k, v) - } - } - - return data -} - -// pushes value into data tree. -func (d dataTree) push(k string, v []string) { - keys := FetchIndexes(k) - if len(keys) <= MaxLevel { - d.mount(keys, v) - } -} - -// mount mounts data tree recursively. -func (d dataTree) mount(i []string, v []string) { - if len(i) == 1 { - // single value context (last element) - d[i[0]] = v[len(v)-1] - return - } - - if len(i) == 2 && i[1] == "" { - // non associated array of elements - d[i[0]] = v - return - } - - if p, ok := d[i[0]]; ok { - p.(dataTree).mount(i[1:], v) - return - } - - d[i[0]] = make(dataTree) - d[i[0]].(dataTree).mount(i[1:], v) -} - -// parse incoming dataTree request into JSON (including contentMultipart form dataTree) -func parseUploads(r *http.Request, cfg config.Uploads) *Uploads { - u := &Uploads{ - cfg: cfg, - tree: make(fileTree), - list: make([]*FileUpload, 0), - } - - for k, v := range r.MultipartForm.File { - files := make([]*FileUpload, 0, len(v)) - for _, f := range v { - files = append(files, NewUpload(f)) - } - - u.list = append(u.list, files...) - u.tree.push(k, files) - } - - return u -} - -// pushes new file upload into it's proper place. -func (d fileTree) push(k string, v []*FileUpload) { - keys := FetchIndexes(k) - if len(keys) <= MaxLevel { - d.mount(keys, v) - } -} - -// mount mounts data tree recursively. -func (d fileTree) mount(i []string, v []*FileUpload) { - if len(i) == 1 { - // single value context - d[i[0]] = v[0] - return - } - - if len(i) == 2 && i[1] == "" { - // non associated array of elements - d[i[0]] = v - return - } - - if p, ok := d[i[0]]; ok { - p.(fileTree).mount(i[1:], v) - return - } - - d[i[0]] = make(fileTree) - d[i[0]].(fileTree).mount(i[1:], v) -} - -// FetchIndexes parses input name and splits it into separate indexes list. -func FetchIndexes(s string) []string { - var ( - pos int - ch string - keys = make([]string, 1) - ) - - for _, c := range s { - ch = string(c) - switch ch { - case " ": - // ignore all spaces - continue - case "[": - pos = 1 - continue - case "]": - if pos == 1 { - keys = append(keys, "") - } - pos = 2 - default: - if pos == 1 || pos == 2 { - keys = append(keys, "") - } - - keys[len(keys)-1] += ch - pos = 0 - } - } - - return keys -} diff --git a/plugins/http/plugin.go b/plugins/http/plugin.go index dcfb7ddb..33efaf37 100644 --- a/plugins/http/plugin.go +++ b/plugins/http/plugin.go @@ -22,6 +22,8 @@ import ( "github.com/spiral/roadrunner/v2/plugins/config" "github.com/spiral/roadrunner/v2/plugins/http/attributes" httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" + "github.com/spiral/roadrunner/v2/plugins/http/static" + handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" "github.com/spiral/roadrunner/v2/plugins/logger" "github.com/spiral/roadrunner/v2/plugins/server" "github.com/spiral/roadrunner/v2/plugins/status" @@ -35,16 +37,15 @@ const ( // PluginName declares plugin name. PluginName = "http" - // RR_HTTP env variable key (internal) if the HTTP presents - RR_MODE = "RR_MODE" //nolint:golint,stylecheck + // RrMode RR_HTTP env variable key (internal) if the HTTP presents + RrMode = "RR_MODE" - // HTTPS_SCHEME - HTTPS_SCHEME = "https" //nolint:golint,stylecheck + HTTPSScheme = "https" ) // Middleware interface type Middleware interface { - Middleware(f http.Handler) http.HandlerFunc + Middleware(f http.Handler) http.Handler } type middleware map[string]Middleware @@ -69,7 +70,7 @@ type Plugin struct { pool pool.Pool // servers RR handler - handler *Handler + handler *handler.Handler // servers http *http.Server @@ -111,14 +112,14 @@ func (s *Plugin) Init(cfg config.Configurer, rrLogger logger.Logger, server serv s.cfg.Env = make(map[string]string) } - s.cfg.Env[RR_MODE] = "http" + s.cfg.Env[RrMode] = "http" s.server = server return nil } func (s *Plugin) logCallback(event interface{}) { - if ev, ok := event.(ResponseEvent); ok { + if ev, ok := event.(handler.ResponseEvent); ok { s.log.Debug(fmt.Sprintf("%d %s %s", ev.Response.Status, ev.Request.Method, ev.Request.URI), "remote", ev.Request.RemoteAddr, "elapsed", ev.Elapsed().String(), @@ -156,7 +157,7 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit return } - s.handler, err = NewHandler( + s.handler, err = handler.NewHandler( s.cfg.MaxRequestSize, *s.cfg.Uploads, s.cfg.Cidrs, @@ -174,7 +175,7 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit // if we have static, handler here, create a fileserver if s.cfg.Static != nil { - h := http.FileServer(StaticFilesHandler(s.cfg.Static)) + h := http.FileServer(static.FS(s.cfg.Static)) // Static files handler mux.HandleFunc(s.cfg.Static.Pattern, func(w http.ResponseWriter, r *http.Request) { if s.cfg.Static.Request != nil { @@ -429,7 +430,7 @@ func (s *Plugin) Reset() error { s.log.Info("HTTP workers Pool successfully restarted") - s.handler, err = NewHandler( + s.handler, err = handler.NewHandler( s.cfg.MaxRequestSize, *s.cfg.Uploads, s.cfg.Cidrs, @@ -500,7 +501,7 @@ func (s *Plugin) Ready() status.Status { func (s *Plugin) redirect(w http.ResponseWriter, r *http.Request) { target := &url.URL{ - Scheme: HTTPS_SCHEME, + Scheme: HTTPSScheme, // host or host:port Host: s.tlsAddr(r.Host, false), Path: r.URL.Path, @@ -641,7 +642,7 @@ func (s *Plugin) tlsAddr(host string, forcePort bool) string { } func applyMiddlewares(server *http.Server, middlewares map[string]Middleware, order []string, log logger.Logger) { - for i := 0; i < len(order); i++ { + for i := len(order) - 1; i >= 0; i-- { if mdwr, ok := middlewares[order[i]]; ok { server.Handler = mdwr.Middleware(server.Handler) } else { diff --git a/plugins/http/request.go b/plugins/http/request.go deleted file mode 100644 index a1398819..00000000 --- a/plugins/http/request.go +++ /dev/null @@ -1,187 +0,0 @@ -package http - -import ( - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "strings" - - j "github.com/json-iterator/go" - "github.com/spiral/roadrunner/v2/pkg/payload" - "github.com/spiral/roadrunner/v2/plugins/http/attributes" - "github.com/spiral/roadrunner/v2/plugins/http/config" - "github.com/spiral/roadrunner/v2/plugins/logger" -) - -var json = j.ConfigCompatibleWithStandardLibrary - -const ( - defaultMaxMemory = 32 << 20 // 32 MB - contentNone = iota + 900 - contentStream - contentMultipart - contentFormData -) - -// Request maps net/http requests to PSR7 compatible structure and managed state of temporary uploaded files. -type Request struct { - // RemoteAddr contains ip address of client, make sure to check X-Real-Ip and X-Forwarded-For for real client address. - RemoteAddr string `json:"remoteAddr"` - - // Protocol includes HTTP protocol version. - Protocol string `json:"protocol"` - - // Method contains name of HTTP method used for the request. - Method string `json:"method"` - - // URI contains full request URI with scheme and query. - URI string `json:"uri"` - - // Header contains list of request headers. - Header http.Header `json:"headers"` - - // Cookies contains list of request cookies. - Cookies map[string]string `json:"cookies"` - - // RawQuery contains non parsed query string (to be parsed on php end). - RawQuery string `json:"rawQuery"` - - // Parsed indicates that request body has been parsed on RR end. - Parsed bool `json:"parsed"` - - // Uploads contains list of uploaded files, their names, sized and associations with temporary files. - Uploads *Uploads `json:"uploads"` - - // Attributes can be set by chained mdwr to safely pass value from Golang to PHP. See: GetAttribute, SetAttribute functions. - Attributes map[string]interface{} `json:"attributes"` - - // request body can be parsedData or []byte - body interface{} -} - -func fetchIP(pair string) string { - if !strings.ContainsRune(pair, ':') { - return pair - } - - addr, _, _ := net.SplitHostPort(pair) - return addr -} - -// NewRequest creates new PSR7 compatible request using net/http request. -func NewRequest(r *http.Request, cfg config.Uploads) (*Request, error) { - req := &Request{ - RemoteAddr: fetchIP(r.RemoteAddr), - Protocol: r.Proto, - Method: r.Method, - URI: uri(r), - Header: r.Header, - Cookies: make(map[string]string), - RawQuery: r.URL.RawQuery, - Attributes: attributes.All(r), - } - - for _, c := range r.Cookies() { - if v, err := url.QueryUnescape(c.Value); err == nil { - req.Cookies[c.Name] = v - } - } - - switch req.contentType() { - case contentNone: - return req, nil - - case contentStream: - var err error - req.body, err = ioutil.ReadAll(r.Body) - return req, err - - case contentMultipart: - if err := r.ParseMultipartForm(defaultMaxMemory); err != nil { - return nil, err - } - - req.Uploads = parseUploads(r, cfg) - fallthrough - case contentFormData: - if err := r.ParseForm(); err != nil { - return nil, err - } - - req.body = parseData(r) - } - - req.Parsed = true - return req, nil -} - -// Open moves all uploaded files to temporary directory so it can be given to php later. -func (r *Request) Open(log logger.Logger) { - if r.Uploads == nil { - return - } - - r.Uploads.Open(log) -} - -// Close clears all temp file uploads -func (r *Request) Close(log logger.Logger) { - if r.Uploads == nil { - return - } - - r.Uploads.Clear(log) -} - -// Payload request marshaled RoadRunner payload based on PSR7 data. values encode method is JSON. Make sure to open -// files prior to calling this method. -func (r *Request) Payload() (payload.Payload, error) { - p := payload.Payload{} - - var err error - if p.Context, err = json.Marshal(r); err != nil { - return payload.Payload{}, err - } - - if r.Parsed { - if p.Body, err = json.Marshal(r.body); err != nil { - return payload.Payload{}, err - } - } else if r.body != nil { - p.Body = r.body.([]byte) - } - - return p, nil -} - -// contentType returns the payload content type. -func (r *Request) contentType() int { - if r.Method == "HEAD" || r.Method == "OPTIONS" { - return contentNone - } - - ct := r.Header.Get("content-type") - if strings.Contains(ct, "application/x-www-form-urlencoded") { - return contentFormData - } - - if strings.Contains(ct, "multipart/form-data") { - return contentMultipart - } - - return contentStream -} - -// uri fetches full uri from request in a form of string (including https scheme if TLS connection is enabled). -func uri(r *http.Request) string { - if r.URL.Host != "" { - return r.URL.String() - } - if r.TLS != nil { - return fmt.Sprintf("https://%s%s", r.Host, r.URL.String()) - } - - return fmt.Sprintf("http://%s%s", r.Host, r.URL.String()) -} diff --git a/plugins/http/response.go b/plugins/http/response.go deleted file mode 100644 index 17049ce1..00000000 --- a/plugins/http/response.go +++ /dev/null @@ -1,105 +0,0 @@ -package http - -import ( - "io" - "net/http" - "strings" - "sync" - - "github.com/spiral/roadrunner/v2/pkg/payload" -) - -// Response handles PSR7 response logic. -type Response struct { - // Status contains response status. - Status int `json:"status"` - - // Header contains list of response headers. - Headers map[string][]string `json:"headers"` - - // associated Body payload. - Body interface{} - sync.Mutex -} - -// NewResponse creates new response based on given pool payload. -func NewResponse(p payload.Payload) (*Response, error) { - r := &Response{Body: p.Body} - if err := json.Unmarshal(p.Context, r); err != nil { - return nil, err - } - - return r, nil -} - -// Write writes response headers, status and body into ResponseWriter. -func (r *Response) Write(w http.ResponseWriter) error { - // INFO map is the reference type in golang - p := handlePushHeaders(r.Headers) - if pusher, ok := w.(http.Pusher); ok { - for _, v := range p { - err := pusher.Push(v, nil) - if err != nil { - return err - } - } - } - - handleTrailers(r.Headers) - for n, h := range r.Headers { - for _, v := range h { - w.Header().Add(n, v) - } - } - - w.WriteHeader(r.Status) - - if data, ok := r.Body.([]byte); ok { - _, err := w.Write(data) - if err != nil { - return handleWriteError(err) - } - } - - if rc, ok := r.Body.(io.Reader); ok { - if _, err := io.Copy(w, rc); err != nil { - return err - } - } - - return nil -} - -func handlePushHeaders(h map[string][]string) []string { - var p []string - pushHeader, ok := h[http2pushHeaderKey] - if !ok { - return p - } - - p = append(p, pushHeader...) - - delete(h, http2pushHeaderKey) - - return p -} - -func handleTrailers(h map[string][]string) { - trailers, ok := h[TrailerHeaderKey] - if !ok { - return - } - - for _, tr := range trailers { - for _, n := range strings.Split(tr, ",") { - n = strings.Trim(n, "\t ") - if v, ok := h[n]; ok { - h["Trailer:"+n] = v - - delete(h, n) - } - } - } - - delete(h, TrailerHeaderKey) -} diff --git a/plugins/http/static.go b/plugins/http/static.go deleted file mode 100644 index be977fb3..00000000 --- a/plugins/http/static.go +++ /dev/null @@ -1,88 +0,0 @@ -package http - -import ( - "io/fs" - "net/http" - "path/filepath" - "strings" - - httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" -) - -type ExtensionFilter struct { - allowed map[string]struct{} - forbidden map[string]struct{} -} - -func NewExtensionFilter(allow, forbid []string) *ExtensionFilter { - ef := &ExtensionFilter{ - allowed: make(map[string]struct{}, len(allow)), - forbidden: make(map[string]struct{}, len(forbid)), - } - - for i := 0; i < len(forbid); i++ { - // skip empty lines - if forbid[i] == "" { - continue - } - ef.forbidden[forbid[i]] = struct{}{} - } - - for i := 0; i < len(allow); i++ { - // skip empty lines - if allow[i] == "" { - continue - } - ef.allowed[allow[i]] = struct{}{} - } - - // check if any forbidden items presented in the allowed - // if presented, delete such items from allowed - for k := range ef.allowed { - if _, ok := ef.forbidden[k]; ok { - delete(ef.allowed, k) - } - } - - return ef -} - -type FileSystem struct { - ef *ExtensionFilter - // embedded - http.FileSystem -} - -// Open wrapper around http.FileSystem Open method, name here is the name of the -func (f FileSystem) Open(name string) (http.File, error) { - file, err := f.FileSystem.Open(name) - if err != nil { - return nil, err - } - - fstat, err := file.Stat() - if err != nil { - return nil, fs.ErrNotExist - } - - if fstat.IsDir() { - return nil, fs.ErrPermission - } - - ext := strings.ToLower(filepath.Ext(fstat.Name())) - if _, ok := f.ef.forbidden[ext]; ok { - return nil, fs.ErrPermission - } - - // if file extension is allowed, append it to the FileInfo slice - if _, ok := f.ef.allowed[ext]; ok { - return file, nil - } - - return nil, fs.ErrNotExist -} - -// StaticFilesHandler is a constructor for the http.FileSystem -func StaticFilesHandler(config *httpConfig.Static) http.FileSystem { - return FileSystem{NewExtensionFilter(config.Allow, config.Forbid), http.Dir(config.Dir)} -} diff --git a/plugins/http/static/static.go b/plugins/http/static/static.go new file mode 100644 index 00000000..d0278466 --- /dev/null +++ b/plugins/http/static/static.go @@ -0,0 +1,88 @@ +package static + +import ( + "io/fs" + "net/http" + "path/filepath" + "strings" + + httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" +) + +type ExtensionFilter struct { + allowed map[string]struct{} + forbidden map[string]struct{} +} + +func NewExtensionFilter(allow, forbid []string) *ExtensionFilter { + ef := &ExtensionFilter{ + allowed: make(map[string]struct{}, len(allow)), + forbidden: make(map[string]struct{}, len(forbid)), + } + + for i := 0; i < len(forbid); i++ { + // skip empty lines + if forbid[i] == "" { + continue + } + ef.forbidden[forbid[i]] = struct{}{} + } + + for i := 0; i < len(allow); i++ { + // skip empty lines + if allow[i] == "" { + continue + } + ef.allowed[allow[i]] = struct{}{} + } + + // check if any forbidden items presented in the allowed + // if presented, delete such items from allowed + for k := range ef.allowed { + if _, ok := ef.forbidden[k]; ok { + delete(ef.allowed, k) + } + } + + return ef +} + +type FileSystem struct { + ef *ExtensionFilter + // embedded + http.FileSystem +} + +// Open wrapper around http.FileSystem Open method, name here is the name of the +func (f FileSystem) Open(name string) (http.File, error) { + file, err := f.FileSystem.Open(name) + if err != nil { + return nil, err + } + + fstat, err := file.Stat() + if err != nil { + return nil, fs.ErrNotExist + } + + if fstat.IsDir() { + return nil, fs.ErrPermission + } + + ext := strings.ToLower(filepath.Ext(fstat.Name())) + if _, ok := f.ef.forbidden[ext]; ok { + return nil, fs.ErrPermission + } + + // if file extension is allowed, append it to the FileInfo slice + if _, ok := f.ef.allowed[ext]; ok { + return file, nil + } + + return nil, fs.ErrNotExist +} + +// FS is a constructor for the http.FileSystem +func FS(config *httpConfig.Static) http.FileSystem { + return FileSystem{NewExtensionFilter(config.Allow, config.Forbid), http.Dir(config.Dir)} +} diff --git a/plugins/http/uploads.go b/plugins/http/uploads.go deleted file mode 100644 index f9f8e1c8..00000000 --- a/plugins/http/uploads.go +++ /dev/null @@ -1,159 +0,0 @@ -package http - -import ( - "github.com/spiral/roadrunner/v2/plugins/http/config" - "github.com/spiral/roadrunner/v2/plugins/logger" - - "io" - "io/ioutil" - "mime/multipart" - "os" - "sync" -) - -const ( - // UploadErrorOK - no error, the file uploaded with success. - UploadErrorOK = 0 - - // UploadErrorNoFile - no file was uploaded. - UploadErrorNoFile = 4 - - // UploadErrorNoTmpDir - missing a temporary folder. - UploadErrorNoTmpDir = 6 - - // UploadErrorCantWrite - failed to write file to disk. - UploadErrorCantWrite = 7 - - // UploadErrorExtension - forbidden file extension. - UploadErrorExtension = 8 -) - -// Uploads tree manages uploaded files tree and temporary files. -type Uploads struct { - // associated temp directory and forbidden extensions. - cfg config.Uploads - - // pre processed data tree for Uploads. - tree fileTree - - // flat list of all file Uploads. - list []*FileUpload -} - -// MarshalJSON marshal tree tree into JSON. -func (u *Uploads) MarshalJSON() ([]byte, error) { - return json.Marshal(u.tree) -} - -// Open moves all uploaded files to temp directory, return error in case of issue with temp directory. File errors -// will be handled individually. -func (u *Uploads) Open(log logger.Logger) { - var wg sync.WaitGroup - for _, f := range u.list { - wg.Add(1) - go func(f *FileUpload) { - defer wg.Done() - err := f.Open(u.cfg) - if err != nil && log != nil { - log.Error("error opening the file", "err", err) - } - }(f) - } - - wg.Wait() -} - -// Clear deletes all temporary files. -func (u *Uploads) Clear(log logger.Logger) { - for _, f := range u.list { - if f.TempFilename != "" && exists(f.TempFilename) { - err := os.Remove(f.TempFilename) - if err != nil && log != nil { - log.Error("error removing the file", "err", err) - } - } - } -} - -// FileUpload represents singular file NewUpload. -type FileUpload struct { - // ID contains filename specified by the client. - Name string `json:"name"` - - // Mime contains mime-type provided by the client. - Mime string `json:"mime"` - - // Size of the uploaded file. - Size int64 `json:"size"` - - // Error indicates file upload error (if any). See http://php.net/manual/en/features.file-upload.errors.php - Error int `json:"error"` - - // TempFilename points to temporary file location. - TempFilename string `json:"tmpName"` - - // associated file header - header *multipart.FileHeader -} - -// NewUpload wraps net/http upload into PRS-7 compatible structure. -func NewUpload(f *multipart.FileHeader) *FileUpload { - return &FileUpload{ - Name: f.Filename, - Mime: f.Header.Get("Content-Type"), - Error: UploadErrorOK, - header: f, - } -} - -// Open moves file content into temporary file available for PHP. -// NOTE: -// There is 2 deferred functions, and in case of getting 2 errors from both functions -// error from close of temp file would be overwritten by error from the main file -// STACK -// DEFER FILE CLOSE (2) -// DEFER TMP CLOSE (1) -func (f *FileUpload) Open(cfg config.Uploads) (err error) { - if cfg.Forbids(f.Name) { - f.Error = UploadErrorExtension - return nil - } - - file, err := f.header.Open() - if err != nil { - f.Error = UploadErrorNoFile - return err - } - - defer func() { - // close the main file - err = file.Close() - }() - - tmp, err := ioutil.TempFile(cfg.TmpDir(), "upload") - if err != nil { - // most likely cause of this issue is missing tmp dir - f.Error = UploadErrorNoTmpDir - return err - } - - f.TempFilename = tmp.Name() - defer func() { - // close the temp file - err = tmp.Close() - }() - - if f.Size, err = io.Copy(tmp, file); err != nil { - f.Error = UploadErrorCantWrite - } - - return err -} - -// exists if file exists. -func exists(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { - return false - } - return true -} diff --git a/plugins/http/worker_handler/constants.go b/plugins/http/worker_handler/constants.go new file mode 100644 index 00000000..3355d9c2 --- /dev/null +++ b/plugins/http/worker_handler/constants.go @@ -0,0 +1,8 @@ +package handler + +import "net/http" + +var http2pushHeaderKey = http.CanonicalHeaderKey("http2-push") + +// TrailerHeaderKey http header key +var TrailerHeaderKey = http.CanonicalHeaderKey("trailer") diff --git a/plugins/http/worker_handler/errors.go b/plugins/http/worker_handler/errors.go new file mode 100644 index 00000000..5fa8e64e --- /dev/null +++ b/plugins/http/worker_handler/errors.go @@ -0,0 +1,25 @@ +// +build !windows + +package handler + +import ( + "errors" + "net" + "os" + "syscall" +) + +// Broken pipe +var errEPIPE = errors.New("EPIPE(32) -> connection reset by peer") + +// handleWriteError just check if error was caused by aborted connection on linux +func handleWriteError(err error) error { + if netErr, ok2 := err.(*net.OpError); ok2 { + if syscallErr, ok3 := netErr.Err.(*os.SyscallError); ok3 { + if errors.Is(syscallErr.Err, syscall.EPIPE) { + return errEPIPE + } + } + } + return err +} diff --git a/plugins/http/worker_handler/errors_windows.go b/plugins/http/worker_handler/errors_windows.go new file mode 100644 index 00000000..390cc7d1 --- /dev/null +++ b/plugins/http/worker_handler/errors_windows.go @@ -0,0 +1,27 @@ +// +build windows + +package handler + +import ( + "errors" + "net" + "os" + "syscall" +) + +//Software caused connection abort. +//An established connection was aborted by the software in your host computer, +//possibly due to a data transmission time-out or protocol error. +var errEPIPE = errors.New("WSAECONNABORTED (10053) -> an established connection was aborted by peer") + +// handleWriteError just check if error was caused by aborted connection on windows +func handleWriteError(err error) error { + if netErr, ok2 := err.(*net.OpError); ok2 { + if syscallErr, ok3 := netErr.Err.(*os.SyscallError); ok3 { + if syscallErr.Err == syscall.WSAECONNABORTED { + return errEPIPE + } + } + } + return err +} diff --git a/plugins/http/worker_handler/handler.go b/plugins/http/worker_handler/handler.go new file mode 100644 index 00000000..be53fc12 --- /dev/null +++ b/plugins/http/worker_handler/handler.go @@ -0,0 +1,217 @@ +package handler + +import ( + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/spiral/errors" + "github.com/spiral/roadrunner/v2/pkg/events" + "github.com/spiral/roadrunner/v2/pkg/pool" + "github.com/spiral/roadrunner/v2/plugins/http/config" + "github.com/spiral/roadrunner/v2/plugins/logger" +) + +// MB is 1024 bytes +const MB uint64 = 1024 * 1024 + +// ErrorEvent represents singular http error event. +type ErrorEvent struct { + // Request contains client request, must not be stored. + Request *http.Request + + // Error - associated error, if any. + Error error + + // event timings + start time.Time + elapsed time.Duration +} + +// Elapsed returns duration of the invocation. +func (e *ErrorEvent) Elapsed() time.Duration { + return e.elapsed +} + +// ResponseEvent represents singular http response event. +type ResponseEvent struct { + // Request contains client request, must not be stored. + Request *Request + + // Response contains service response. + Response *Response + + // event timings + start time.Time + elapsed time.Duration +} + +// Elapsed returns duration of the invocation. +func (e *ResponseEvent) Elapsed() time.Duration { + return e.elapsed +} + +// Handler serves http connections to underlying PHP application using PSR-7 protocol. Context will include request headers, +// parsed files and query, payload will include parsed form dataTree (if any). +type Handler struct { + maxRequestSize uint64 + uploads config.Uploads + trusted config.Cidrs + log logger.Logger + pool pool.Pool + mul sync.Mutex + lsn events.Listener +} + +// NewHandler return handle interface implementation +func NewHandler(maxReqSize uint64, uploads config.Uploads, trusted config.Cidrs, pool pool.Pool) (*Handler, error) { + if pool == nil { + return nil, errors.E(errors.Str("pool should be initialized")) + } + return &Handler{ + maxRequestSize: maxReqSize * MB, + uploads: uploads, + pool: pool, + trusted: trusted, + }, nil +} + +// AddListener attaches handler event controller. +func (h *Handler) AddListener(l events.Listener) { + h.mul.Lock() + defer h.mul.Unlock() + + h.lsn = l +} + +// mdwr serve using PSR-7 requests passed to underlying application. Attempts to serve static files first if enabled. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + const op = errors.Op("http_plugin_serve_http") + start := time.Now() + + // validating request size + if h.maxRequestSize != 0 { + const op = errors.Op("http_handler_max_size") + if length := r.Header.Get("content-length"); length != "" { + // try to parse the value from the `content-length` header + size, err := strconv.ParseInt(length, 10, 64) + if err != nil { + // if got an error while parsing -> assign 500 code to the writer and return + http.Error(w, errors.E(op, err).Error(), 500) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, errors.Str("error while parsing value from the `content-length` header")), start: start, elapsed: time.Since(start)}) + return + } + + if size > int64(h.maxRequestSize) { + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, errors.Str("request body max size is exceeded")), start: start, elapsed: time.Since(start)}) + http.Error(w, errors.E(op, errors.Str("request body max size is exceeded")).Error(), 500) + return + } + } + } + + req, err := NewRequest(r, h.uploads) + if err != nil { + // if pipe is broken, there is no sense to write the header + // in this case we just report about error + if err == errEPIPE { + h.sendEvent(ErrorEvent{Request: r, Error: err, start: start, elapsed: time.Since(start)}) + return + } + + http.Error(w, errors.E(op, err).Error(), 500) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) + return + } + + // proxy IP resolution + h.resolveIP(req) + + req.Open(h.log) + defer req.Close(h.log) + + p, err := req.Payload() + if err != nil { + http.Error(w, errors.E(op, err).Error(), 500) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) + return + } + + rsp, err := h.pool.Exec(p) + if err != nil { + http.Error(w, errors.E(op, err).Error(), 500) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) + return + } + + resp, err := NewResponse(rsp) + if err != nil { + http.Error(w, errors.E(op, err).Error(), resp.Status) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) + return + } + + h.handleResponse(req, resp, start) + err = resp.Write(w) + if err != nil { + http.Error(w, errors.E(op, err).Error(), 500) + h.sendEvent(ErrorEvent{Request: r, Error: errors.E(op, err), start: start, elapsed: time.Since(start)}) + } +} + +// handleResponse triggers response event. +func (h *Handler) handleResponse(req *Request, resp *Response, start time.Time) { + h.sendEvent(ResponseEvent{Request: req, Response: resp, start: start, elapsed: time.Since(start)}) +} + +// sendEvent invokes event handler if any. +func (h *Handler) sendEvent(event interface{}) { + if h.lsn != nil { + h.lsn(event) + } +} + +// get real ip passing multiple proxy +func (h *Handler) resolveIP(r *Request) { + if h.trusted.IsTrusted(r.RemoteAddr) == false { //nolint:gosimple + return + } + + if r.Header.Get("X-Forwarded-For") != "" { + ips := strings.Split(r.Header.Get("X-Forwarded-For"), ",") + ipCount := len(ips) + + for i := ipCount - 1; i >= 0; i-- { + addr := strings.TrimSpace(ips[i]) + if net.ParseIP(addr) != nil { + r.RemoteAddr = addr + return + } + } + + return + } + + // The logic here is the following: + // In general case, we only expect X-Real-Ip header. If it exist, we get the IP address from header and set request Remote address + // But, if there is no X-Real-Ip header, we also trying to check CloudFlare headers + // True-Client-IP is a general CF header in which copied information from X-Real-Ip in CF. + // CF-Connecting-IP is an Enterprise feature and we check it last in order. + // This operations are near O(1) because Headers struct are the map type -> type MIMEHeader map[string][]string + if r.Header.Get("X-Real-Ip") != "" { + r.RemoteAddr = fetchIP(r.Header.Get("X-Real-Ip")) + return + } + + if r.Header.Get("True-Client-IP") != "" { + r.RemoteAddr = fetchIP(r.Header.Get("True-Client-IP")) + return + } + + if r.Header.Get("CF-Connecting-IP") != "" { + r.RemoteAddr = fetchIP(r.Header.Get("CF-Connecting-IP")) + } +} diff --git a/plugins/http/worker_handler/parse.go b/plugins/http/worker_handler/parse.go new file mode 100644 index 00000000..2790da2a --- /dev/null +++ b/plugins/http/worker_handler/parse.go @@ -0,0 +1,149 @@ +package handler + +import ( + "net/http" + + "github.com/spiral/roadrunner/v2/plugins/http/config" +) + +// MaxLevel defines maximum tree depth for incoming request data and files. +const MaxLevel = 127 + +type dataTree map[string]interface{} +type fileTree map[string]interface{} + +// parseData parses incoming request body into data tree. +func parseData(r *http.Request) dataTree { + data := make(dataTree) + if r.PostForm != nil { + for k, v := range r.PostForm { + data.push(k, v) + } + } + + if r.MultipartForm != nil { + for k, v := range r.MultipartForm.Value { + data.push(k, v) + } + } + + return data +} + +// pushes value into data tree. +func (d dataTree) push(k string, v []string) { + keys := FetchIndexes(k) + if len(keys) <= MaxLevel { + d.mount(keys, v) + } +} + +// mount mounts data tree recursively. +func (d dataTree) mount(i []string, v []string) { + if len(i) == 1 { + // single value context (last element) + d[i[0]] = v[len(v)-1] + return + } + + if len(i) == 2 && i[1] == "" { + // non associated array of elements + d[i[0]] = v + return + } + + if p, ok := d[i[0]]; ok { + p.(dataTree).mount(i[1:], v) + return + } + + d[i[0]] = make(dataTree) + d[i[0]].(dataTree).mount(i[1:], v) +} + +// parse incoming dataTree request into JSON (including contentMultipart form dataTree) +func parseUploads(r *http.Request, cfg config.Uploads) *Uploads { + u := &Uploads{ + cfg: cfg, + tree: make(fileTree), + list: make([]*FileUpload, 0), + } + + for k, v := range r.MultipartForm.File { + files := make([]*FileUpload, 0, len(v)) + for _, f := range v { + files = append(files, NewUpload(f)) + } + + u.list = append(u.list, files...) + u.tree.push(k, files) + } + + return u +} + +// pushes new file upload into it's proper place. +func (d fileTree) push(k string, v []*FileUpload) { + keys := FetchIndexes(k) + if len(keys) <= MaxLevel { + d.mount(keys, v) + } +} + +// mount mounts data tree recursively. +func (d fileTree) mount(i []string, v []*FileUpload) { + if len(i) == 1 { + // single value context + d[i[0]] = v[0] + return + } + + if len(i) == 2 && i[1] == "" { + // non associated array of elements + d[i[0]] = v + return + } + + if p, ok := d[i[0]]; ok { + p.(fileTree).mount(i[1:], v) + return + } + + d[i[0]] = make(fileTree) + d[i[0]].(fileTree).mount(i[1:], v) +} + +// FetchIndexes parses input name and splits it into separate indexes list. +func FetchIndexes(s string) []string { + var ( + pos int + ch string + keys = make([]string, 1) + ) + + for _, c := range s { + ch = string(c) + switch ch { + case " ": + // ignore all spaces + continue + case "[": + pos = 1 + continue + case "]": + if pos == 1 { + keys = append(keys, "") + } + pos = 2 + default: + if pos == 1 || pos == 2 { + keys = append(keys, "") + } + + keys[len(keys)-1] += ch + pos = 0 + } + } + + return keys +} diff --git a/plugins/http/worker_handler/request.go b/plugins/http/worker_handler/request.go new file mode 100644 index 00000000..178bc827 --- /dev/null +++ b/plugins/http/worker_handler/request.go @@ -0,0 +1,187 @@ +package handler + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "strings" + + j "github.com/json-iterator/go" + "github.com/spiral/roadrunner/v2/pkg/payload" + "github.com/spiral/roadrunner/v2/plugins/http/attributes" + "github.com/spiral/roadrunner/v2/plugins/http/config" + "github.com/spiral/roadrunner/v2/plugins/logger" +) + +var json = j.ConfigCompatibleWithStandardLibrary + +const ( + defaultMaxMemory = 32 << 20 // 32 MB + contentNone = iota + 900 + contentStream + contentMultipart + contentFormData +) + +// Request maps net/http requests to PSR7 compatible structure and managed state of temporary uploaded files. +type Request struct { + // RemoteAddr contains ip address of client, make sure to check X-Real-Ip and X-Forwarded-For for real client address. + RemoteAddr string `json:"remoteAddr"` + + // Protocol includes HTTP protocol version. + Protocol string `json:"protocol"` + + // Method contains name of HTTP method used for the request. + Method string `json:"method"` + + // URI contains full request URI with scheme and query. + URI string `json:"uri"` + + // Header contains list of request headers. + Header http.Header `json:"headers"` + + // Cookies contains list of request cookies. + Cookies map[string]string `json:"cookies"` + + // RawQuery contains non parsed query string (to be parsed on php end). + RawQuery string `json:"rawQuery"` + + // Parsed indicates that request body has been parsed on RR end. + Parsed bool `json:"parsed"` + + // Uploads contains list of uploaded files, their names, sized and associations with temporary files. + Uploads *Uploads `json:"uploads"` + + // Attributes can be set by chained mdwr to safely pass value from Golang to PHP. See: GetAttribute, SetAttribute functions. + Attributes map[string]interface{} `json:"attributes"` + + // request body can be parsedData or []byte + body interface{} +} + +func fetchIP(pair string) string { + if !strings.ContainsRune(pair, ':') { + return pair + } + + addr, _, _ := net.SplitHostPort(pair) + return addr +} + +// NewRequest creates new PSR7 compatible request using net/http request. +func NewRequest(r *http.Request, cfg config.Uploads) (*Request, error) { + req := &Request{ + RemoteAddr: fetchIP(r.RemoteAddr), + Protocol: r.Proto, + Method: r.Method, + URI: uri(r), + Header: r.Header, + Cookies: make(map[string]string), + RawQuery: r.URL.RawQuery, + Attributes: attributes.All(r), + } + + for _, c := range r.Cookies() { + if v, err := url.QueryUnescape(c.Value); err == nil { + req.Cookies[c.Name] = v + } + } + + switch req.contentType() { + case contentNone: + return req, nil + + case contentStream: + var err error + req.body, err = ioutil.ReadAll(r.Body) + return req, err + + case contentMultipart: + if err := r.ParseMultipartForm(defaultMaxMemory); err != nil { + return nil, err + } + + req.Uploads = parseUploads(r, cfg) + fallthrough + case contentFormData: + if err := r.ParseForm(); err != nil { + return nil, err + } + + req.body = parseData(r) + } + + req.Parsed = true + return req, nil +} + +// Open moves all uploaded files to temporary directory so it can be given to php later. +func (r *Request) Open(log logger.Logger) { + if r.Uploads == nil { + return + } + + r.Uploads.Open(log) +} + +// Close clears all temp file uploads +func (r *Request) Close(log logger.Logger) { + if r.Uploads == nil { + return + } + + r.Uploads.Clear(log) +} + +// Payload request marshaled RoadRunner payload based on PSR7 data. values encode method is JSON. Make sure to open +// files prior to calling this method. +func (r *Request) Payload() (payload.Payload, error) { + p := payload.Payload{} + + var err error + if p.Context, err = json.Marshal(r); err != nil { + return payload.Payload{}, err + } + + if r.Parsed { + if p.Body, err = json.Marshal(r.body); err != nil { + return payload.Payload{}, err + } + } else if r.body != nil { + p.Body = r.body.([]byte) + } + + return p, nil +} + +// contentType returns the payload content type. +func (r *Request) contentType() int { + if r.Method == "HEAD" || r.Method == "OPTIONS" { + return contentNone + } + + ct := r.Header.Get("content-type") + if strings.Contains(ct, "application/x-www-form-urlencoded") { + return contentFormData + } + + if strings.Contains(ct, "multipart/form-data") { + return contentMultipart + } + + return contentStream +} + +// uri fetches full uri from request in a form of string (including https scheme if TLS connection is enabled). +func uri(r *http.Request) string { + if r.URL.Host != "" { + return r.URL.String() + } + if r.TLS != nil { + return fmt.Sprintf("https://%s%s", r.Host, r.URL.String()) + } + + return fmt.Sprintf("http://%s%s", r.Host, r.URL.String()) +} diff --git a/plugins/http/worker_handler/response.go b/plugins/http/worker_handler/response.go new file mode 100644 index 00000000..1763d304 --- /dev/null +++ b/plugins/http/worker_handler/response.go @@ -0,0 +1,105 @@ +package handler + +import ( + "io" + "net/http" + "strings" + "sync" + + "github.com/spiral/roadrunner/v2/pkg/payload" +) + +// Response handles PSR7 response logic. +type Response struct { + // Status contains response status. + Status int `json:"status"` + + // Header contains list of response headers. + Headers map[string][]string `json:"headers"` + + // associated Body payload. + Body interface{} + sync.Mutex +} + +// NewResponse creates new response based on given pool payload. +func NewResponse(p payload.Payload) (*Response, error) { + r := &Response{Body: p.Body} + if err := json.Unmarshal(p.Context, r); err != nil { + return nil, err + } + + return r, nil +} + +// Write writes response headers, status and body into ResponseWriter. +func (r *Response) Write(w http.ResponseWriter) error { + // INFO map is the reference type in golang + p := handlePushHeaders(r.Headers) + if pusher, ok := w.(http.Pusher); ok { + for _, v := range p { + err := pusher.Push(v, nil) + if err != nil { + return err + } + } + } + + handleTrailers(r.Headers) + for n, h := range r.Headers { + for _, v := range h { + w.Header().Add(n, v) + } + } + + w.WriteHeader(r.Status) + + if data, ok := r.Body.([]byte); ok { + _, err := w.Write(data) + if err != nil { + return handleWriteError(err) + } + } + + if rc, ok := r.Body.(io.Reader); ok { + if _, err := io.Copy(w, rc); err != nil { + return err + } + } + + return nil +} + +func handlePushHeaders(h map[string][]string) []string { + var p []string + pushHeader, ok := h[http2pushHeaderKey] + if !ok { + return p + } + + p = append(p, pushHeader...) + + delete(h, http2pushHeaderKey) + + return p +} + +func handleTrailers(h map[string][]string) { + trailers, ok := h[TrailerHeaderKey] + if !ok { + return + } + + for _, tr := range trailers { + for _, n := range strings.Split(tr, ",") { + n = strings.Trim(n, "\t ") + if v, ok := h[n]; ok { + h["Trailer:"+n] = v + + delete(h, n) + } + } + } + + delete(h, TrailerHeaderKey) +} diff --git a/plugins/http/worker_handler/uploads.go b/plugins/http/worker_handler/uploads.go new file mode 100644 index 00000000..e695000e --- /dev/null +++ b/plugins/http/worker_handler/uploads.go @@ -0,0 +1,159 @@ +package handler + +import ( + "github.com/spiral/roadrunner/v2/plugins/http/config" + "github.com/spiral/roadrunner/v2/plugins/logger" + + "io" + "io/ioutil" + "mime/multipart" + "os" + "sync" +) + +const ( + // UploadErrorOK - no error, the file uploaded with success. + UploadErrorOK = 0 + + // UploadErrorNoFile - no file was uploaded. + UploadErrorNoFile = 4 + + // UploadErrorNoTmpDir - missing a temporary folder. + UploadErrorNoTmpDir = 6 + + // UploadErrorCantWrite - failed to write file to disk. + UploadErrorCantWrite = 7 + + // UploadErrorExtension - forbidden file extension. + UploadErrorExtension = 8 +) + +// Uploads tree manages uploaded files tree and temporary files. +type Uploads struct { + // associated temp directory and forbidden extensions. + cfg config.Uploads + + // pre processed data tree for Uploads. + tree fileTree + + // flat list of all file Uploads. + list []*FileUpload +} + +// MarshalJSON marshal tree tree into JSON. +func (u *Uploads) MarshalJSON() ([]byte, error) { + return json.Marshal(u.tree) +} + +// Open moves all uploaded files to temp directory, return error in case of issue with temp directory. File errors +// will be handled individually. +func (u *Uploads) Open(log logger.Logger) { + var wg sync.WaitGroup + for _, f := range u.list { + wg.Add(1) + go func(f *FileUpload) { + defer wg.Done() + err := f.Open(u.cfg) + if err != nil && log != nil { + log.Error("error opening the file", "err", err) + } + }(f) + } + + wg.Wait() +} + +// Clear deletes all temporary files. +func (u *Uploads) Clear(log logger.Logger) { + for _, f := range u.list { + if f.TempFilename != "" && exists(f.TempFilename) { + err := os.Remove(f.TempFilename) + if err != nil && log != nil { + log.Error("error removing the file", "err", err) + } + } + } +} + +// FileUpload represents singular file NewUpload. +type FileUpload struct { + // ID contains filename specified by the client. + Name string `json:"name"` + + // Mime contains mime-type provided by the client. + Mime string `json:"mime"` + + // Size of the uploaded file. + Size int64 `json:"size"` + + // Error indicates file upload error (if any). See http://php.net/manual/en/features.file-upload.errors.php + Error int `json:"error"` + + // TempFilename points to temporary file location. + TempFilename string `json:"tmpName"` + + // associated file header + header *multipart.FileHeader +} + +// NewUpload wraps net/http upload into PRS-7 compatible structure. +func NewUpload(f *multipart.FileHeader) *FileUpload { + return &FileUpload{ + Name: f.Filename, + Mime: f.Header.Get("Content-Type"), + Error: UploadErrorOK, + header: f, + } +} + +// Open moves file content into temporary file available for PHP. +// NOTE: +// There is 2 deferred functions, and in case of getting 2 errors from both functions +// error from close of temp file would be overwritten by error from the main file +// STACK +// DEFER FILE CLOSE (2) +// DEFER TMP CLOSE (1) +func (f *FileUpload) Open(cfg config.Uploads) (err error) { + if cfg.Forbids(f.Name) { + f.Error = UploadErrorExtension + return nil + } + + file, err := f.header.Open() + if err != nil { + f.Error = UploadErrorNoFile + return err + } + + defer func() { + // close the main file + err = file.Close() + }() + + tmp, err := ioutil.TempFile(cfg.TmpDir(), "upload") + if err != nil { + // most likely cause of this issue is missing tmp dir + f.Error = UploadErrorNoTmpDir + return err + } + + f.TempFilename = tmp.Name() + defer func() { + // close the temp file + err = tmp.Close() + }() + + if f.Size, err = io.Copy(tmp, file); err != nil { + f.Error = UploadErrorCantWrite + } + + return err +} + +// exists if file exists. +func exists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} diff --git a/tests/plugins/http/configs/.rr-http-static.yaml b/tests/plugins/http/configs/.rr-http-static.yaml index 9bb2abc0..9351f020 100644 --- a/tests/plugins/http/configs/.rr-http-static.yaml +++ b/tests/plugins/http/configs/.rr-http-static.yaml @@ -1,5 +1,5 @@ server: - command: "php ../../http/client.php pid pipes" + command: "php ../../psr-worker-bench.php" user: "" group: "" env: @@ -24,7 +24,7 @@ http: response: "output": "output-header" pool: - num_workers: 2 + num_workers: 12 max_jobs: 0 allocate_timeout: 60s destroy_timeout: 60s diff --git a/tests/plugins/http/handler_test.go b/tests/plugins/http/handler_test.go index cf445aad..575fe656 100644 --- a/tests/plugins/http/handler_test.go +++ b/tests/plugins/http/handler_test.go @@ -12,8 +12,8 @@ import ( "github.com/spiral/roadrunner/v2/pkg/pool" "github.com/spiral/roadrunner/v2/pkg/transport/pipe" - httpPlugin "github.com/spiral/roadrunner/v2/plugins/http" "github.com/spiral/roadrunner/v2/plugins/http/config" + handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" "github.com/stretchr/testify/assert" "net/http" @@ -35,7 +35,7 @@ func TestHandler_Echo(t *testing.T) { t.Fatal(err) } - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -66,7 +66,7 @@ func TestHandler_Echo(t *testing.T) { } func Test_HandlerErrors(t *testing.T) { - _, err := httpPlugin.NewHandler(1024, config.Uploads{ + _, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, nil) @@ -89,7 +89,7 @@ func TestHandler_Headers(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -150,7 +150,7 @@ func TestHandler_Empty_User_Agent(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -210,7 +210,7 @@ func TestHandler_User_Agent(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -270,7 +270,7 @@ func TestHandler_Cookies(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -335,7 +335,7 @@ func TestHandler_JsonPayload_POST(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -399,7 +399,7 @@ func TestHandler_JsonPayload_PUT(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -459,7 +459,7 @@ func TestHandler_JsonPayload_PATCH(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -519,7 +519,7 @@ func TestHandler_FormData_POST(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -592,7 +592,7 @@ func TestHandler_FormData_POST_Overwrite(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -665,7 +665,7 @@ func TestHandler_FormData_POST_Form_UrlEncoded_Charset(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -737,7 +737,7 @@ func TestHandler_FormData_PUT(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -809,7 +809,7 @@ func TestHandler_FormData_PATCH(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -881,7 +881,7 @@ func TestHandler_Multipart_POST(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -995,7 +995,7 @@ func TestHandler_Multipart_PUT(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1109,7 +1109,7 @@ func TestHandler_Multipart_PATCH(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1225,7 +1225,7 @@ func TestHandler_Error(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1271,7 +1271,7 @@ func TestHandler_Error2(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1317,7 +1317,7 @@ func TestHandler_Error3(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1, config.Uploads{ + h, err := handler.NewHandler(1, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1376,7 +1376,7 @@ func TestHandler_ResponseDuration(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1401,7 +1401,7 @@ func TestHandler_ResponseDuration(t *testing.T) { gotresp := make(chan interface{}) h.AddListener(func(event interface{}) { switch t := event.(type) { - case httpPlugin.ResponseEvent: + case handler.ResponseEvent: if t.Elapsed() > 0 { close(gotresp) } @@ -1437,7 +1437,7 @@ func TestHandler_ResponseDurationDelayed(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1462,7 +1462,7 @@ func TestHandler_ResponseDurationDelayed(t *testing.T) { gotresp := make(chan interface{}) h.AddListener(func(event interface{}) { switch tp := event.(type) { - case httpPlugin.ResponseEvent: + case handler.ResponseEvent: if tp.Elapsed() > time.Second { close(gotresp) } @@ -1497,7 +1497,7 @@ func TestHandler_ErrorDuration(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) @@ -1522,7 +1522,7 @@ func TestHandler_ErrorDuration(t *testing.T) { goterr := make(chan interface{}) h.AddListener(func(event interface{}) { switch tp := event.(type) { - case httpPlugin.ErrorEvent: + case handler.ErrorEvent: if tp.Elapsed() > 0 { close(goterr) } @@ -1571,7 +1571,7 @@ func TestHandler_IP(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, cidrs, p) @@ -1632,7 +1632,7 @@ func TestHandler_XRealIP(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, cidrs, p) @@ -1698,7 +1698,7 @@ func TestHandler_XForwardedFor(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, cidrs, p) @@ -1763,7 +1763,7 @@ func TestHandler_XForwardedFor_NotTrustedRemoteIp(t *testing.T) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, cidrs, p) @@ -1811,7 +1811,7 @@ func BenchmarkHandler_Listen_Echo(b *testing.B) { p.Destroy(context.Background()) }() - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, p) diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index 000fd25d..61486eef 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -1671,7 +1671,7 @@ func staticNotForbid(t *testing.T) { func serveStaticSample(t *testing.T) { b, r, err := get("http://localhost:21603/tests/static/sample.txt") assert.NoError(t, err) - assert.Equal(t, "sample", b) + assert.Equal(t, "sample\n", b) _ = r.Body.Close() } diff --git a/tests/plugins/http/parse_test.go b/tests/plugins/http/parse_test.go index 5cc1ce32..15c82839 100644 --- a/tests/plugins/http/parse_test.go +++ b/tests/plugins/http/parse_test.go @@ -3,7 +3,7 @@ package http import ( "testing" - "github.com/spiral/roadrunner/v2/plugins/http" + handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" ) var samples = []struct { @@ -21,7 +21,7 @@ var samples = []struct { func Test_FetchIndexes(t *testing.T) { for i := 0; i < len(samples); i++ { - r := http.FetchIndexes(samples[i].in) + r := handler.FetchIndexes(samples[i].in) if !same(r, samples[i].out) { t.Errorf("got %q, want %q", r, samples[i].out) } @@ -31,7 +31,7 @@ func Test_FetchIndexes(t *testing.T) { func BenchmarkConfig_FetchIndexes(b *testing.B) { for _, tt := range samples { for n := 0; n < b.N; n++ { - r := http.FetchIndexes(tt.in) + r := handler.FetchIndexes(tt.in) if !same(r, tt.out) { b.Fail() } diff --git a/tests/plugins/http/plugin_middleware.go b/tests/plugins/http/plugin_middleware.go index 00640b69..9f04d6db 100644 --- a/tests/plugins/http/plugin_middleware.go +++ b/tests/plugins/http/plugin_middleware.go @@ -18,8 +18,8 @@ func (p *PluginMiddleware) Init(cfg config.Configurer) error { } // Middleware test -func (p *PluginMiddleware) Middleware(next http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func (p *PluginMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/halt" { w.WriteHeader(500) _, err := w.Write([]byte("halted")) @@ -29,7 +29,7 @@ func (p *PluginMiddleware) Middleware(next http.Handler) http.HandlerFunc { } else { next.ServeHTTP(w, r) } - } + }) } // Name test @@ -49,8 +49,8 @@ func (p *PluginMiddleware2) Init(cfg config.Configurer) error { } // Middleware test -func (p *PluginMiddleware2) Middleware(next http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func (p *PluginMiddleware2) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/boom" { w.WriteHeader(555) _, err := w.Write([]byte("boom")) @@ -60,7 +60,7 @@ func (p *PluginMiddleware2) Middleware(next http.Handler) http.HandlerFunc { } else { next.ServeHTTP(w, r) } - } + }) } // Name test diff --git a/tests/plugins/http/response_test.go b/tests/plugins/http/response_test.go index dc9856ac..3564d9cd 100644 --- a/tests/plugins/http/response_test.go +++ b/tests/plugins/http/response_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/spiral/roadrunner/v2/pkg/payload" - httpPlugin "github.com/spiral/roadrunner/v2/plugins/http" + handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" "github.com/stretchr/testify/assert" ) @@ -45,13 +45,13 @@ func (tw *testWriter) Push(target string, opts *http.PushOptions) error { } func TestNewResponse_Error(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{Context: []byte(`invalid payload`)}) + r, err := handler.NewResponse(payload.Payload{Context: []byte(`invalid payload`)}) assert.Error(t, err) assert.Nil(t, r) } func TestNewResponse_Write(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte(`{"headers":{"key":["value"]},"status": 301}`), Body: []byte(`sample body`), }) @@ -68,7 +68,7 @@ func TestNewResponse_Write(t *testing.T) { } func TestNewResponse_Stream(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte(`{"headers":{"key":["value"]},"status": 301}`), }) @@ -93,7 +93,7 @@ func TestNewResponse_Stream(t *testing.T) { } func TestNewResponse_StreamError(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte(`{"headers":{"key":["value"]},"status": 301}`), }) @@ -114,7 +114,7 @@ func TestNewResponse_StreamError(t *testing.T) { } func TestWrite_HandlesPush(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte(`{"headers":{"Http2-Push":["/test.js"],"content-type":["text/html"]},"status": 200}`), }) @@ -129,7 +129,7 @@ func TestWrite_HandlesPush(t *testing.T) { } func TestWrite_HandlesTrailers(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte(`{"headers":{"Trailer":["foo, bar", "baz"],"foo":["test"],"bar":["demo"]},"status": 200}`), }) @@ -139,7 +139,7 @@ func TestWrite_HandlesTrailers(t *testing.T) { w := &testWriter{h: http.Header(make(map[string][]string))} assert.NoError(t, r.Write(w)) - assert.Nil(t, w.h[httpPlugin.TrailerHeaderKey]) + assert.Nil(t, w.h[handler.TrailerHeaderKey]) assert.Nil(t, w.h["foo"]) //nolint:staticcheck assert.Nil(t, w.h["baz"]) //nolint:staticcheck @@ -148,7 +148,7 @@ func TestWrite_HandlesTrailers(t *testing.T) { } func TestWrite_HandlesHandlesWhitespacesInTrailer(t *testing.T) { - r, err := httpPlugin.NewResponse(payload.Payload{ + r, err := handler.NewResponse(payload.Payload{ Context: []byte( `{"headers":{"Trailer":["foo\t,bar , baz"],"foo":["a"],"bar":["b"],"baz":["c"]},"status": 200}`), }) diff --git a/tests/plugins/http/uploads_test.go b/tests/plugins/http/uploads_test.go index bc7e17df..5c39589c 100644 --- a/tests/plugins/http/uploads_test.go +++ b/tests/plugins/http/uploads_test.go @@ -18,8 +18,8 @@ import ( j "github.com/json-iterator/go" poolImpl "github.com/spiral/roadrunner/v2/pkg/pool" "github.com/spiral/roadrunner/v2/pkg/transport/pipe" - httpPlugin "github.com/spiral/roadrunner/v2/plugins/http" "github.com/spiral/roadrunner/v2/plugins/http/config" + handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" "github.com/stretchr/testify/assert" ) @@ -40,7 +40,7 @@ func TestHandler_Upload_File(t *testing.T) { t.Fatal(err) } - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, pool) @@ -123,7 +123,7 @@ func TestHandler_Upload_NestedFile(t *testing.T) { t.Fatal(err) } - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{}, }, nil, pool) @@ -206,7 +206,7 @@ func TestHandler_Upload_File_NoTmpDir(t *testing.T) { t.Fatal(err) } - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: "-------", Forbid: []string{}, }, nil, pool) @@ -289,7 +289,7 @@ func TestHandler_Upload_File_Forbids(t *testing.T) { t.Fatal(err) } - h, err := httpPlugin.NewHandler(1024, config.Uploads{ + h, err := handler.NewHandler(1024, config.Uploads{ Dir: os.TempDir(), Forbid: []string{".go"}, }, nil, pool) diff --git a/tests/static/sample.txt b/tests/static/sample.txt index eed7e79a..d64a3d96 100644 --- a/tests/static/sample.txt +++ b/tests/static/sample.txt @@ -1 +1 @@ -sample \ No newline at end of file +sample -- cgit v1.2.3 From b789df7dcc9268f98f2bacfb40f753b10d521e4f Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Wed, 28 Apr 2021 16:43:59 +0300 Subject: - Update CHANGELOG - Add tests for the etags functionality Signed-off-by: Valery Piashchynski --- CHANGELOG.md | 18 ++ pkg/worker_watcher/worker_watcher.go | 4 +- plugins/http/config/static.go | 6 + plugins/http/plugin.go | 246 ++------------------- plugins/http/serve.go | 242 ++++++++++++++++++++ plugins/http/static/etag.go | 71 ++++++ .../http/configs/.rr-http-static-etags.yaml | 35 +++ tests/plugins/http/configs/.rr-http-static.yaml | 6 +- tests/plugins/http/http_plugin_test.go | 98 ++++++++ 9 files changed, 490 insertions(+), 236 deletions(-) create mode 100644 plugins/http/serve.go create mode 100644 plugins/http/static/etag.go create mode 100644 tests/plugins/http/configs/.rr-http-static-etags.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e2a940..ca0758b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,24 @@ CHANGELOG ========= +v2.2.0 (11.05.2021) +------------------- + +## 👀 New: + +- ✏️ Reworked `static` plugin. Now, it does not affect the performance of the main route and persist on the separate + file server (within the `http` plugin). Looong awaited feature: `Etag` (+ weak Etags) as well with the `If-Mach` + , `If-None-Match`, `If-Range`, `Last-Modified` + and `If-Modified-Since` tags supported. Static plugin has a bunch of new options such as: `allow`, `calculate_etag` + , `weak` and `pattern`. + ### Option `always` was deleted from the plugin. + +- ✏️ + +## 🩹 Fixes: + +- 🐛 Fix: issue with wrong ordered middlewares (reverse). Now the order is correct. + v2.1.0 (27.04.2021) ------------------- diff --git a/pkg/worker_watcher/worker_watcher.go b/pkg/worker_watcher/worker_watcher.go index a6dfe43e..5aec4ee6 100755 --- a/pkg/worker_watcher/worker_watcher.go +++ b/pkg/worker_watcher/worker_watcher.go @@ -153,7 +153,7 @@ func (ww *workerWatcher) Allocate() error { return nil } -// Remove +// Remove worker func (ww *workerWatcher) Remove(wb worker.BaseProcess) { ww.Lock() defer ww.Unlock() @@ -172,7 +172,7 @@ func (ww *workerWatcher) Remove(wb worker.BaseProcess) { } } -// O(1) operation +// Push O(1) operation func (ww *workerWatcher) Push(w worker.BaseProcess) { if w.State().Value() != worker.StateReady { _ = w.Kill() diff --git a/plugins/http/config/static.go b/plugins/http/config/static.go index e9acc3e4..4b7b3a9b 100644 --- a/plugins/http/config/static.go +++ b/plugins/http/config/static.go @@ -17,6 +17,12 @@ type Static struct { // Default - /static/ Pattern string + // CalculateEtag can be true/false and used to calculate etag for the static + CalculateEtag bool `mapstructure:"calculate_etag"` + + // Weak etag `W/` + Weak bool + // forbid specifies list of file extensions which are forbidden for access. // example: .php, .exe, .bat, .htaccess and etc. Forbid []string diff --git a/plugins/http/plugin.go b/plugins/http/plugin.go index 33efaf37..58336c17 100644 --- a/plugins/http/plugin.go +++ b/plugins/http/plugin.go @@ -2,15 +2,11 @@ package http import ( "context" - "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" "log" "net/http" - "net/http/fcgi" - "net/url" - "strings" + "os" + "path/filepath" "sync" "github.com/hashicorp/go-multierror" @@ -27,10 +23,8 @@ import ( "github.com/spiral/roadrunner/v2/plugins/logger" "github.com/spiral/roadrunner/v2/plugins/server" "github.com/spiral/roadrunner/v2/plugins/status" - "github.com/spiral/roadrunner/v2/utils" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" - "golang.org/x/sys/cpu" ) const ( @@ -190,10 +184,22 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit } } + // calculate etag for the resource + if s.cfg.Static.CalculateEtag { + f, errS := os.Open(filepath.Join(s.cfg.Static.Dir, r.URL.Path)) + if errS != nil { + s.log.Warn("error opening file to calculate the Etag", "provided path", r.URL.Path) + } + + // Set etag value to the ResponseWriter + static.SetEtag(s.cfg.Static, f, w) + } + h.ServeHTTP(w, r) }) } + // handle main route mux.HandleFunc("/", s.ServeHTTP) if s.cfg.EnableHTTP() { @@ -241,78 +247,6 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit }() } -func (s *Plugin) serveHTTP(errCh chan error) { - if s.http == nil { - return - } - const op = errors.Op("http_plugin_serve_http") - - if len(s.mdwr) > 0 { - applyMiddlewares(s.http, s.mdwr, s.cfg.Middleware, s.log) - } - l, err := utils.CreateListener(s.cfg.Address) - if err != nil { - errCh <- errors.E(op, err) - return - } - - err = s.http.Serve(l) - if err != nil && err != http.ErrServerClosed { - errCh <- errors.E(op, err) - return - } -} - -func (s *Plugin) serveHTTPS(errCh chan error) { - if s.https == nil { - return - } - const op = errors.Op("http_plugin_serve_https") - if len(s.mdwr) > 0 { - applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) - } - l, err := utils.CreateListener(s.cfg.SSLConfig.Address) - if err != nil { - errCh <- errors.E(op, err) - return - } - - err = s.https.ServeTLS( - l, - s.cfg.SSLConfig.Cert, - s.cfg.SSLConfig.Key, - ) - - if err != nil && err != http.ErrServerClosed { - errCh <- errors.E(op, err) - return - } -} - -// serveFCGI starts FastCGI server. -func (s *Plugin) serveFCGI(errCh chan error) { - if s.fcgi == nil { - return - } - const op = errors.Op("http_plugin_serve_fcgi") - - if len(s.mdwr) > 0 { - applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) - } - - l, err := utils.CreateListener(s.cfg.FCGIConfig.Address) - if err != nil { - errCh <- errors.E(op, err) - return - } - - err = fcgi.Serve(l, s.fcgi.Handler) - if err != nil && err != http.ErrServerClosed { - errCh <- errors.E(op, err) - return - } -} - // Stop stops the http. func (s *Plugin) Stop() error { s.Lock() @@ -498,155 +432,3 @@ func (s *Plugin) Ready() status.Status { Code: http.StatusServiceUnavailable, } } - -func (s *Plugin) redirect(w http.ResponseWriter, r *http.Request) { - target := &url.URL{ - Scheme: HTTPSScheme, - // host or host:port - Host: s.tlsAddr(r.Host, false), - Path: r.URL.Path, - RawQuery: r.URL.RawQuery, - } - - http.Redirect(w, r, target.String(), http.StatusPermanentRedirect) -} - -// https://golang.org/pkg/net/http/#Hijacker -//go:inline -func headerContainsUpgrade(r *http.Request) bool { - if _, ok := r.Header["Upgrade"]; ok { - return true - } - return false -} - -// append RootCA to the https server TLS config -func (s *Plugin) appendRootCa() error { - const op = errors.Op("http_plugin_append_root_ca") - rootCAs, err := x509.SystemCertPool() - if err != nil { - return nil - } - if rootCAs == nil { - rootCAs = x509.NewCertPool() - } - - CA, err := ioutil.ReadFile(s.cfg.SSLConfig.RootCA) - if err != nil { - return err - } - - // should append our CA cert - ok := rootCAs.AppendCertsFromPEM(CA) - if !ok { - return errors.E(op, errors.Str("could not append Certs from PEM")) - } - // disable "G402 (CWE-295): TLS MinVersion too low. (Confidence: HIGH, Severity: HIGH)" - // #nosec G402 - cfg := &tls.Config{ - InsecureSkipVerify: false, - RootCAs: rootCAs, - } - s.http.TLSConfig = cfg - - return nil -} - -// Init https server -func (s *Plugin) initSSL() *http.Server { - var topCipherSuites []uint16 - var defaultCipherSuitesTLS13 []uint16 - - hasGCMAsmAMD64 := cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ - hasGCMAsmARM64 := cpu.ARM64.HasAES && cpu.ARM64.HasPMULL - // Keep in sync with crypto/aes/cipher_s390x.go. - hasGCMAsmS390X := cpu.S390X.HasAES && cpu.S390X.HasAESCBC && cpu.S390X.HasAESCTR && (cpu.S390X.HasGHASH || cpu.S390X.HasAESGCM) - - hasGCMAsm := hasGCMAsmAMD64 || hasGCMAsmARM64 || hasGCMAsmS390X - - if hasGCMAsm { - // If AES-GCM hardware is provided then priorities AES-GCM - // cipher suites. - topCipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - } - defaultCipherSuitesTLS13 = []uint16{ - tls.TLS_AES_128_GCM_SHA256, - tls.TLS_CHACHA20_POLY1305_SHA256, - tls.TLS_AES_256_GCM_SHA384, - } - } else { - // Without AES-GCM hardware, we put the ChaCha20-Poly1305 - // cipher suites first. - topCipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - } - defaultCipherSuitesTLS13 = []uint16{ - tls.TLS_CHACHA20_POLY1305_SHA256, - tls.TLS_AES_128_GCM_SHA256, - tls.TLS_AES_256_GCM_SHA384, - } - } - - DefaultCipherSuites := make([]uint16, 0, 22) - DefaultCipherSuites = append(DefaultCipherSuites, topCipherSuites...) - DefaultCipherSuites = append(DefaultCipherSuites, defaultCipherSuitesTLS13...) - - sslServer := &http.Server{ - Addr: s.tlsAddr(s.cfg.Address, true), - Handler: s, - ErrorLog: s.stdLog, - TLSConfig: &tls.Config{ - CurvePreferences: []tls.CurveID{ - tls.CurveP256, - tls.CurveP384, - tls.CurveP521, - tls.X25519, - }, - CipherSuites: DefaultCipherSuites, - MinVersion: tls.VersionTLS12, - PreferServerCipherSuites: true, - }, - } - - return sslServer -} - -// init http/2 server -func (s *Plugin) initHTTP2() error { - return http2.ConfigureServer(s.https, &http2.Server{ - MaxConcurrentStreams: s.cfg.HTTP2Config.MaxConcurrentStreams, - }) -} - -// tlsAddr replaces listen or host port with port configured by SSLConfig config. -func (s *Plugin) tlsAddr(host string, forcePort bool) string { - // remove current forcePort first - host = strings.Split(host, ":")[0] - - if forcePort || s.cfg.SSLConfig.Port != 443 { - host = fmt.Sprintf("%s:%v", host, s.cfg.SSLConfig.Port) - } - - return host -} - -func applyMiddlewares(server *http.Server, middlewares map[string]Middleware, order []string, log logger.Logger) { - for i := len(order) - 1; i >= 0; i-- { - if mdwr, ok := middlewares[order[i]]; ok { - server.Handler = mdwr.Middleware(server.Handler) - } else { - log.Warn("requested middleware does not exist", "requested", order[i]) - } - } -} diff --git a/plugins/http/serve.go b/plugins/http/serve.go new file mode 100644 index 00000000..338d4339 --- /dev/null +++ b/plugins/http/serve.go @@ -0,0 +1,242 @@ +package http + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/http/fcgi" + "net/url" + "os" + "strings" + + "github.com/spiral/errors" + "github.com/spiral/roadrunner/v2/plugins/logger" + "github.com/spiral/roadrunner/v2/utils" + "golang.org/x/net/http2" + "golang.org/x/sys/cpu" +) + +func (s *Plugin) serveHTTP(errCh chan error) { + if s.http == nil { + return + } + const op = errors.Op("http_plugin_serve_http") + + if len(s.mdwr) > 0 { + applyMiddlewares(s.http, s.mdwr, s.cfg.Middleware, s.log) + } + l, err := utils.CreateListener(s.cfg.Address) + if err != nil { + errCh <- errors.E(op, err) + return + } + + err = s.http.Serve(l) + if err != nil && err != http.ErrServerClosed { + errCh <- errors.E(op, err) + return + } +} + +func (s *Plugin) serveHTTPS(errCh chan error) { + if s.https == nil { + return + } + const op = errors.Op("http_plugin_serve_https") + if len(s.mdwr) > 0 { + applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) + } + l, err := utils.CreateListener(s.cfg.SSLConfig.Address) + if err != nil { + errCh <- errors.E(op, err) + return + } + + err = s.https.ServeTLS( + l, + s.cfg.SSLConfig.Cert, + s.cfg.SSLConfig.Key, + ) + + if err != nil && err != http.ErrServerClosed { + errCh <- errors.E(op, err) + return + } +} + +// serveFCGI starts FastCGI server. +func (s *Plugin) serveFCGI(errCh chan error) { + if s.fcgi == nil { + return + } + const op = errors.Op("http_plugin_serve_fcgi") + + if len(s.mdwr) > 0 { + applyMiddlewares(s.https, s.mdwr, s.cfg.Middleware, s.log) + } + + l, err := utils.CreateListener(s.cfg.FCGIConfig.Address) + if err != nil { + errCh <- errors.E(op, err) + return + } + + err = fcgi.Serve(l, s.fcgi.Handler) + if err != nil && err != http.ErrServerClosed { + errCh <- errors.E(op, err) + return + } +} + +func (s *Plugin) redirect(w http.ResponseWriter, r *http.Request) { + target := &url.URL{ + Scheme: HTTPSScheme, + // host or host:port + Host: s.tlsAddr(r.Host, false), + Path: r.URL.Path, + RawQuery: r.URL.RawQuery, + } + + http.Redirect(w, r, target.String(), http.StatusPermanentRedirect) +} + +// https://golang.org/pkg/net/http/#Hijacker +//go:inline +func headerContainsUpgrade(r *http.Request) bool { + if _, ok := r.Header["Upgrade"]; ok { + return true + } + return false +} + +// append RootCA to the https server TLS config +func (s *Plugin) appendRootCa() error { + const op = errors.Op("http_plugin_append_root_ca") + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil + } + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + CA, err := os.ReadFile(s.cfg.SSLConfig.RootCA) + if err != nil { + return err + } + + // should append our CA cert + ok := rootCAs.AppendCertsFromPEM(CA) + if !ok { + return errors.E(op, errors.Str("could not append Certs from PEM")) + } + // disable "G402 (CWE-295): TLS MinVersion too low. (Confidence: HIGH, Severity: HIGH)" + // #nosec G402 + cfg := &tls.Config{ + InsecureSkipVerify: false, + RootCAs: rootCAs, + } + s.http.TLSConfig = cfg + + return nil +} + +// Init https server +func (s *Plugin) initSSL() *http.Server { + var topCipherSuites []uint16 + var defaultCipherSuitesTLS13 []uint16 + + hasGCMAsmAMD64 := cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ + hasGCMAsmARM64 := cpu.ARM64.HasAES && cpu.ARM64.HasPMULL + // Keep in sync with crypto/aes/cipher_s390x.go. + hasGCMAsmS390X := cpu.S390X.HasAES && cpu.S390X.HasAESCBC && cpu.S390X.HasAESCTR && (cpu.S390X.HasGHASH || cpu.S390X.HasAESGCM) + + hasGCMAsm := hasGCMAsmAMD64 || hasGCMAsmARM64 || hasGCMAsmS390X + + if hasGCMAsm { + // If AES-GCM hardware is provided then priorities AES-GCM + // cipher suites. + topCipherSuites = []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + } + defaultCipherSuitesTLS13 = []uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_AES_256_GCM_SHA384, + } + } else { + // Without AES-GCM hardware, we put the ChaCha20-Poly1305 + // cipher suites first. + topCipherSuites = []uint16{ + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + } + defaultCipherSuitesTLS13 = []uint16{ + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + } + } + + DefaultCipherSuites := make([]uint16, 0, 22) + DefaultCipherSuites = append(DefaultCipherSuites, topCipherSuites...) + DefaultCipherSuites = append(DefaultCipherSuites, defaultCipherSuitesTLS13...) + + sslServer := &http.Server{ + Addr: s.tlsAddr(s.cfg.Address, true), + Handler: s, + ErrorLog: s.stdLog, + TLSConfig: &tls.Config{ + CurvePreferences: []tls.CurveID{ + tls.CurveP256, + tls.CurveP384, + tls.CurveP521, + tls.X25519, + }, + CipherSuites: DefaultCipherSuites, + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + }, + } + + return sslServer +} + +// init http/2 server +func (s *Plugin) initHTTP2() error { + return http2.ConfigureServer(s.https, &http2.Server{ + MaxConcurrentStreams: s.cfg.HTTP2Config.MaxConcurrentStreams, + }) +} + +// tlsAddr replaces listen or host port with port configured by SSLConfig config. +func (s *Plugin) tlsAddr(host string, forcePort bool) string { + // remove current forcePort first + host = strings.Split(host, ":")[0] + + if forcePort || s.cfg.SSLConfig.Port != 443 { + host = fmt.Sprintf("%s:%v", host, s.cfg.SSLConfig.Port) + } + + return host +} + +func applyMiddlewares(server *http.Server, middlewares map[string]Middleware, order []string, log logger.Logger) { + for i := len(order) - 1; i >= 0; i-- { + if mdwr, ok := middlewares[order[i]]; ok { + server.Handler = mdwr.Middleware(server.Handler) + } else { + log.Warn("requested middleware does not exist", "requested", order[i]) + } + } +} diff --git a/plugins/http/static/etag.go b/plugins/http/static/etag.go new file mode 100644 index 00000000..5d41cc53 --- /dev/null +++ b/plugins/http/static/etag.go @@ -0,0 +1,71 @@ +package static + +import ( + "hash/crc32" + "io" + "net/http" + "os" + "unsafe" + + httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" +) + +const etag string = "Etag" + +// weak Etag prefix +var weakPrefix = []byte(`W/`) + +// CRC32 table +var crc32q = crc32.MakeTable(0x48D90782) + +func SetEtag(cfg *httpConfig.Static, f *os.File, w http.ResponseWriter) { + // read the file content + body, err := io.ReadAll(f) + if err != nil { + return + } + + // skip for 0 body + if len(body) == 0 { + return + } + + // preallocate + calculatedEtag := make([]byte, 0, 64) + + // write weak + if cfg.Weak { + calculatedEtag = append(calculatedEtag, weakPrefix...) + } + + calculatedEtag = append(calculatedEtag, '"') + calculatedEtag = appendUint(calculatedEtag, uint32(len(body))) + calculatedEtag = append(calculatedEtag, '-') + calculatedEtag = appendUint(calculatedEtag, crc32.Checksum(body, crc32q)) + calculatedEtag = append(calculatedEtag, '"') + + w.Header().Set(etag, byteToSrt(calculatedEtag)) +} + +// appendUint appends n to dst and returns the extended dst. +func appendUint(dst []byte, n uint32) []byte { + var b [20]byte + buf := b[:] + i := len(buf) + var q uint32 + for n >= 10 { + i-- + q = n / 10 + buf[i] = '0' + byte(n-q*10) + n = q + } + i-- + buf[i] = '0' + byte(n) + + dst = append(dst, buf[i:]...) + return dst +} + +func byteToSrt(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/tests/plugins/http/configs/.rr-http-static-etags.yaml b/tests/plugins/http/configs/.rr-http-static-etags.yaml new file mode 100644 index 00000000..e18c50dd --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static-etags.yaml @@ -0,0 +1,35 @@ +server: + command: "php ../../http/client.php pid pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:21603 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + static: + dir: "../../../" + pattern: "/tests/" + forbid: [ "" ] + allow: [ ".txt", ".php" ] + calculate_etag: true + weak: true + request: + "input": "custom-header" + response: + "output": "output-header" + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/configs/.rr-http-static.yaml b/tests/plugins/http/configs/.rr-http-static.yaml index 9351f020..bbec13f9 100644 --- a/tests/plugins/http/configs/.rr-http-static.yaml +++ b/tests/plugins/http/configs/.rr-http-static.yaml @@ -1,5 +1,5 @@ server: - command: "php ../../psr-worker-bench.php" + command: "php ../../http/client.php pid pipes" user: "" group: "" env: @@ -19,12 +19,14 @@ http: pattern: "/tests/" forbid: [ "" ] allow: [ ".txt", ".php" ] + calculate_etag: true + weak: false request: "input": "custom-header" response: "output": "output-header" pool: - num_workers: 12 + num_workers: 2 max_jobs: 0 allocate_timeout: 60s destroy_timeout: 60s diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index 61486eef..f48194c9 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "net/rpc" + "net/url" "os" "os/signal" "sync" @@ -1562,6 +1563,103 @@ func bigEchoHTTP(t *testing.T) { assert.NoError(t, err) } +func TestStaticEtagPlugin(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static.yaml", + Prefix: "rr", + } + + err = cont.RegisterAll( + cfg, + &logger.ZapLogger{}, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + + err = cont.Init() + if err != nil { + t.Fatal(err) + } + + ch, err := cont.Serve() + assert.NoError(t, err) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + wg := &sync.WaitGroup{} + wg.Add(1) + + stopCh := make(chan struct{}, 1) + + go func() { + defer wg.Done() + for { + select { + case e := <-ch: + assert.Fail(t, "error", e.Error.Error()) + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + case <-sig: + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + case <-stopCh: + // timeout + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + } + } + }() + + time.Sleep(time.Second) + t.Run("ServeSampleEtag", serveStaticSampleEtag) + + stopCh <- struct{}{} + wg.Wait() +} + +func serveStaticSampleEtag(t *testing.T) { + // OK 200 response + b, r, err := get("http://localhost:21603/tests/static/sample.txt") + assert.NoError(t, err) + assert.Equal(t, "sample\n", b) + assert.Equal(t, r.StatusCode, http.StatusOK) + etag := r.Header.Get("Etag") + + _ = r.Body.Close() + + // Should be 304 response with same etag + c := http.Client{ + Timeout: time.Second * 5, + } + + parsedURL, _ := url.Parse("http://localhost:21603/tests/static/sample.txt") + + req := &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + Header: map[string][]string{"If-None-Match": {etag}}, + } + + resp, err := c.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusNotModified, resp.StatusCode) + _ = resp.Body.Close() +} + func TestStaticPlugin(t *testing.T) { cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) assert.NoError(t, err) -- cgit v1.2.3 From 2812157be7a9c1411d02872f0b9fa567bcf7a9b7 Mon Sep 17 00:00:00 2001 From: Valery Piashchynski Date: Wed, 28 Apr 2021 17:37:54 +0300 Subject: - Add r.URL.Path protection Signed-off-by: Valery Piashchynski --- plugins/http/plugin.go | 8 ++ .../http/configs/.rr-http-static-security.yaml | 35 +++++ tests/plugins/http/http_plugin_test.go | 148 +++++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 tests/plugins/http/configs/.rr-http-static-security.yaml diff --git a/plugins/http/plugin.go b/plugins/http/plugin.go index 58336c17..2b1dec89 100644 --- a/plugins/http/plugin.go +++ b/plugins/http/plugin.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" "github.com/hashicorp/go-multierror" @@ -186,6 +187,13 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit // calculate etag for the resource if s.cfg.Static.CalculateEtag { + // do not allow paths like ../../resource + // only specified folder and resources in it + // https://lgtm.com/rules/1510366186013/ + if strings.Contains(r.URL.Path, "..") { + w.WriteHeader(http.StatusForbidden) + return + } f, errS := os.Open(filepath.Join(s.cfg.Static.Dir, r.URL.Path)) if errS != nil { s.log.Warn("error opening file to calculate the Etag", "provided path", r.URL.Path) diff --git a/tests/plugins/http/configs/.rr-http-static-security.yaml b/tests/plugins/http/configs/.rr-http-static-security.yaml new file mode 100644 index 00000000..bbec13f9 --- /dev/null +++ b/tests/plugins/http/configs/.rr-http-static-security.yaml @@ -0,0 +1,35 @@ +server: + command: "php ../../http/client.php pid pipes" + user: "" + group: "" + env: + "RR_HTTP": "true" + relay: "pipes" + relay_timeout: "20s" + +http: + address: 127.0.0.1:21603 + max_request_size: 1024 + middleware: [ "gzip" ] + trusted_subnets: [ "10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10" ] + uploads: + forbid: [ ".php", ".exe", ".bat" ] + static: + dir: "../../../" + pattern: "/tests/" + forbid: [ "" ] + allow: [ ".txt", ".php" ] + calculate_etag: true + weak: false + request: + "input": "custom-header" + response: + "output": "output-header" + pool: + num_workers: 2 + max_jobs: 0 + allocate_timeout: 60s + destroy_timeout: 60s +logs: + mode: development + level: error diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index f48194c9..8f76e3ba 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -1660,6 +1660,154 @@ func serveStaticSampleEtag(t *testing.T) { _ = resp.Body.Close() } +func TestStaticPluginSecurity(t *testing.T) { + cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) + assert.NoError(t, err) + + cfg := &config.Viper{ + Path: "configs/.rr-http-static-security.yaml", + Prefix: "rr", + } + + err = cont.RegisterAll( + cfg, + &logger.ZapLogger{}, + &server.Plugin{}, + &httpPlugin.Plugin{}, + &gzip.Plugin{}, + ) + assert.NoError(t, err) + + err = cont.Init() + if err != nil { + t.Fatal(err) + } + + ch, err := cont.Serve() + assert.NoError(t, err) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + wg := &sync.WaitGroup{} + wg.Add(1) + + stopCh := make(chan struct{}, 1) + + go func() { + defer wg.Done() + for { + select { + case e := <-ch: + assert.Fail(t, "error", e.Error.Error()) + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + case <-sig: + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + case <-stopCh: + // timeout + err = cont.Stop() + if err != nil { + assert.FailNow(t, "error", err.Error()) + } + return + } + } + }() + + time.Sleep(time.Second) + t.Run("ServeSampleNotAllowedPath", serveStaticSampleNotAllowedPath) + + stopCh <- struct{}{} + wg.Wait() +} + +func serveStaticSampleNotAllowedPath(t *testing.T) { + // Should be 304 response with same etag + c := http.Client{ + Timeout: time.Second * 5, + } + + parsedURL := &url.URL{ + Scheme: "http", + User: nil, + Host: "localhost:21603", + Path: "%2e%2e%/tests/", + } + + req := &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + } + + resp, err := c.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + _ = resp.Body.Close() + + parsedURL = &url.URL{ + Scheme: "http", + User: nil, + Host: "localhost:21603", + Path: "%2e%2e%5ctests/", + } + + req = &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + } + + resp, err = c.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + _ = resp.Body.Close() + + parsedURL = &url.URL{ + Scheme: "http", + User: nil, + Host: "localhost:21603", + Path: "..%2ftests/", + } + + req = &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + } + + resp, err = c.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + _ = resp.Body.Close() + + parsedURL = &url.URL{ + Scheme: "http", + User: nil, + Host: "localhost:21603", + Path: "%2e%2e%2ftests/", + } + + req = &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + } + + resp, err = c.Do(req) + assert.Nil(t, err) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + _ = resp.Body.Close() + + _, r, err := get("http://localhost:21603/../../../../tests/../static/sample.txt") + assert.NoError(t, err) + assert.Equal(t, r.StatusCode, 200) + _ = r.Body.Close() +} + func TestStaticPlugin(t *testing.T) { cont, err := endure.NewContainer(nil, endure.SetLogLevel(endure.ErrorLevel)) assert.NoError(t, err) -- cgit v1.2.3