diff options
author | Valery Piashchynski <[email protected]> | 2021-05-13 20:15:00 +0300 |
---|---|---|
committer | GitHub <[email protected]> | 2021-05-13 20:15:00 +0300 |
commit | e1ff9daead5033b537296ffb071e551b95af91ab (patch) | |
tree | c3755967c7c40a05f12d81d1f7043ccc0bc3da0c | |
parent | 705b69631dc91323c64a19594dcfeca06ea4fa5a (diff) | |
parent | 1dd0db287da4106d99578338ea252004def788a9 (diff) |
#671 fix(static): revert static pluginv2.2.1
#671 fix(static): revert static plugin
29 files changed, 389 insertions, 344 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f84c00..5afdfdc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,12 @@ CHANGELOG ========= -v2.3.0 (01.06.2021) +v2.2.1 (13.05.2021) ------------------- -## 👀 New: - -- ✏️ - ## 🩹 Fixes: +- 🐛 Fix: revert static plugin. It stays as a separate plugin on the main route (`/`) and supports all the previously announced features. - 🐛 Fix: remove `build` and other old targets from the Makefile. --- diff --git a/pkg/payload/payload.go b/pkg/payload/payload.go index bf3972aa..e1e45ac1 100755 --- a/pkg/payload/payload.go +++ b/pkg/payload/payload.go @@ -1,6 +1,8 @@ package payload -import "unsafe" +import ( + "github.com/spiral/roadrunner/v2/utils" +) // Payload carries binary header and body to stack and // back to the server. @@ -14,10 +16,5 @@ type Payload struct { // String returns payload body as string func (p *Payload) String() string { - return toString(p.Body) -} - -// unsafe, but lightning fast []byte to string conversion -func toString(data []byte) string { - return *(*string)(unsafe.Pointer(&data)) + return utils.AsString(p.Body) } diff --git a/pkg/pool/static_pool.go b/pkg/pool/static_pool.go index 6ef2373a..d57cc95c 100755 --- a/pkg/pool/static_pool.go +++ b/pkg/pool/static_pool.go @@ -4,7 +4,6 @@ import ( "context" "os/exec" "time" - "unsafe" "github.com/spiral/errors" "github.com/spiral/roadrunner/v2/pkg/events" @@ -12,6 +11,7 @@ import ( "github.com/spiral/roadrunner/v2/pkg/transport" "github.com/spiral/roadrunner/v2/pkg/worker" workerWatcher "github.com/spiral/roadrunner/v2/pkg/worker_watcher" + "github.com/spiral/roadrunner/v2/utils" ) // StopRequest can be sent by worker to indicate that restart is required. @@ -153,7 +153,7 @@ func (sp *StaticPool) Exec(p payload.Payload) (payload.Payload, error) { } // worker want's to be terminated - if len(rsp.Body) == 0 && toString(rsp.Context) == StopRequest { + if len(rsp.Body) == 0 && utils.AsString(rsp.Context) == StopRequest { sp.stopWorker(w) return sp.Exec(p) } @@ -187,7 +187,7 @@ func (sp *StaticPool) execWithTTL(ctx context.Context, p payload.Payload) (paylo } // worker want's to be terminated - if len(rsp.Body) == 0 && toString(rsp.Context) == StopRequest { + if len(rsp.Body) == 0 && utils.AsString(rsp.Context) == StopRequest { sp.stopWorker(w) return sp.execWithTTL(ctx, p) } @@ -347,8 +347,3 @@ func (sp *StaticPool) allocateWorkers(numWorkers uint64) ([]worker.BaseProcess, } return workers, nil } - -// unsafe, but lightning fast []byte to string conversion -func toString(data []byte) string { - return *(*string)(unsafe.Pointer(&data)) -} diff --git a/pkg/pool/static_pool_test.go b/pkg/pool/static_pool_test.go index 69527815..6667bfea 100755 --- a/pkg/pool/static_pool_test.go +++ b/pkg/pool/static_pool_test.go @@ -16,6 +16,7 @@ import ( "github.com/spiral/roadrunner/v2/pkg/payload" "github.com/spiral/roadrunner/v2/pkg/transport/pipe" "github.com/spiral/roadrunner/v2/pkg/worker" + "github.com/spiral/roadrunner/v2/utils" "github.com/stretchr/testify/assert" ) @@ -646,19 +647,19 @@ func Benchmark_Pool_Echo_Replaced(b *testing.B) { } // BenchmarkToStringUnsafe-12 566317729 1.91 ns/op 0 B/op 0 allocs/op -// inline BenchmarkToStringUnsafe-12 1000000000 0.295 ns/op 0 B/op 0 allocs/op +// BenchmarkToStringUnsafe-32 1000000000 0.4434 ns/op 0 B/op 0 allocs/op func BenchmarkToStringUnsafe(b *testing.B) { testPayload := []byte("falsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtoj") b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - res := toString(testPayload) + res := utils.AsString(testPayload) _ = res } } -// BenchmarkToStringSafe-12 28584489 39.1 ns/op 112 B/op 1 allocs/op +// BenchmarkToStringSafe-32 8017846 182.5 ns/op 896 B/op 1 allocs/op // inline BenchmarkToStringSafe-12 28926276 46.6 ns/op 128 B/op 1 allocs/op func BenchmarkToStringSafe(b *testing.B) { testPayload := []byte("falsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtojfalsflasjlifjwpoihejfoiwejow{}{}{}{}jelfjasjfhwaopiehjtopwhtgohrgouahsgkljasdlfjasl;fjals;jdflkndgouwhetopwqhjtoj") diff --git a/plugins/http/config/http.go b/plugins/http/config/http.go index 59735e2e..8b63395f 100644 --- a/plugins/http/config/http.go +++ b/plugins/http/config/http.go @@ -33,9 +33,6 @@ type HTTP struct { // Uploads configures uploads configuration. Uploads *Uploads `mapstructure:"uploads"` - // static configuration - Static *Static `mapstructure:"static"` - // Pool configures worker pool. Pool *poolImpl.Config `mapstructure:"pool"` @@ -103,16 +100,6 @@ func (c *HTTP) InitDefaults() error { c.SSLConfig.Address = "127.0.0.1:443" } - // static files - if c.Static != nil { - if c.Static.Pattern == "" { - c.Static.Pattern = "/static/" - } - if c.Static.Dir == "" { - c.Static.Dir = "." - } - } - err := c.HTTP2Config.InitDefaults() if err != nil { return err @@ -189,13 +176,5 @@ func (c *HTTP) Valid() error { } } - // validate static - if c.Static != nil { - err := c.Static.Valid() - if err != nil { - return errors.E(op, err) - } - } - return nil } diff --git a/plugins/http/config/static.go b/plugins/http/config/static.go deleted file mode 100644 index 4b7b3a9b..00000000 --- a/plugins/http/config/static.go +++ /dev/null @@ -1,58 +0,0 @@ -package config - -import ( - "os" - - "github.com/spiral/errors" -) - -// Static describes file location and controls access to them. -type Static struct { - // Dir contains name of directory to control access to. - // Default - "." - Dir string - - // HTTP pattern, where to serve static files - // for example - `/static/`, `/my-files/static/`, etc - // Default - /static/ - Pattern string - - // CalculateEtag can be true/false and used to calculate etag for the static - CalculateEtag bool `mapstructure:"calculate_etag"` - - // Weak etag `W/` - Weak bool - - // forbid specifies list of file extensions which are forbidden for access. - // example: .php, .exe, .bat, .htaccess and etc. - Forbid []string - - // Allow specifies list of file extensions which are allowed for access. - // example: .php, .exe, .bat, .htaccess and etc. - Allow []string - - // Request headers to add to every static. - Request map[string]string - - // Response headers to add to every static. - Response map[string]string -} - -// Valid returns nil if config is valid. -func (c *Static) Valid() error { - const op = errors.Op("static_plugin_valid") - st, err := os.Stat(c.Dir) - if err != nil { - if os.IsNotExist(err) { - return errors.E(op, errors.Errorf("root directory '%s' does not exists", c.Dir)) - } - - return err - } - - if !st.IsDir() { - return errors.E(op, errors.Errorf("invalid root directory '%s'", c.Dir)) - } - - return nil -} diff --git a/plugins/http/plugin.go b/plugins/http/plugin.go index 344102f4..2b68bbe5 100644 --- a/plugins/http/plugin.go +++ b/plugins/http/plugin.go @@ -5,9 +5,6 @@ import ( "fmt" "log" "net/http" - "os" - "path/filepath" - "strings" "sync" "github.com/hashicorp/go-multierror" @@ -19,7 +16,6 @@ import ( "github.com/spiral/roadrunner/v2/plugins/config" "github.com/spiral/roadrunner/v2/plugins/http/attributes" httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" - "github.com/spiral/roadrunner/v2/plugins/http/static" handler "github.com/spiral/roadrunner/v2/plugins/http/worker_handler" "github.com/spiral/roadrunner/v2/plugins/logger" "github.com/spiral/roadrunner/v2/plugins/server" @@ -136,7 +132,7 @@ func (s *Plugin) Serve() chan error { return errCh } -func (s *Plugin) serve(errCh chan error) { //nolint:gocognit +func (s *Plugin) serve(errCh chan error) { var err error const op = errors.Op("http_plugin_serve") s.pool, err = s.server.NewWorkerPool(context.Background(), pool.Config{ @@ -165,56 +161,11 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit s.handler.AddListener(s.logCallback) - // Create new HTTP Multiplexer - mux := http.NewServeMux() - - // if we have static, handler here, create a fileserver - if s.cfg.Static != nil { - h := http.FileServer(static.FS(s.cfg.Static)) - // Static files handler - mux.HandleFunc(s.cfg.Static.Pattern, func(w http.ResponseWriter, r *http.Request) { - if s.cfg.Static.Request != nil { - for k, v := range s.cfg.Static.Request { - r.Header.Add(k, v) - } - } - - if s.cfg.Static.Response != nil { - for k, v := range s.cfg.Static.Response { - w.Header().Set(k, v) - } - } - - // calculate etag for the resource - if s.cfg.Static.CalculateEtag { - // do not allow paths like ../../resource - // only specified folder and resources in it - // https://lgtm.com/rules/1510366186013/ - if strings.Contains(r.URL.Path, "..") { - w.WriteHeader(http.StatusForbidden) - return - } - f, errS := os.Open(filepath.Join(s.cfg.Static.Dir, r.URL.Path)) - if errS != nil { - s.log.Warn("error opening file to calculate the Etag", "provided path", r.URL.Path) - } - - // Set etag value to the ResponseWriter - static.SetEtag(s.cfg.Static, f, w) - } - - h.ServeHTTP(w, r) - }) - } - - // handle main route - mux.HandleFunc("/", s.ServeHTTP) - if s.cfg.EnableHTTP() { if s.cfg.EnableH2C() { - s.http = &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{}), ErrorLog: s.stdLog} + s.http = &http.Server{Handler: h2c.NewHandler(s, &http2.Server{}), ErrorLog: s.stdLog} } else { - s.http = &http.Server{Handler: mux, ErrorLog: s.stdLog} + s.http = &http.Server{Handler: s, ErrorLog: s.stdLog} } } @@ -238,7 +189,7 @@ func (s *Plugin) serve(errCh chan error) { //nolint:gocognit } if s.cfg.EnableFCGI() { - s.fcgi = &http.Server{Handler: mux, ErrorLog: s.stdLog} + s.fcgi = &http.Server{Handler: s, ErrorLog: s.stdLog} } // start http, https and fcgi servers if requested in the config diff --git a/plugins/http/serve.go b/plugins/http/serve.go index 338d4339..78796322 100644 --- a/plugins/http/serve.go +++ b/plugins/http/serve.go @@ -231,12 +231,24 @@ func (s *Plugin) tlsAddr(host string, forcePort bool) string { return host } +// static plugin name +const static string = "static" + func applyMiddlewares(server *http.Server, middlewares map[string]Middleware, order []string, log logger.Logger) { for i := len(order) - 1; i >= 0; i-- { + // set static last in the row + if order[i] == static { + continue + } if mdwr, ok := middlewares[order[i]]; ok { server.Handler = mdwr.Middleware(server.Handler) } else { log.Warn("requested middleware does not exist", "requested", order[i]) } } + + // set static if exists + if mdwr, ok := middlewares[static]; ok { + server.Handler = mdwr.Middleware(server.Handler) + } } diff --git a/plugins/http/static/static.go b/plugins/http/static/static.go deleted file mode 100644 index d0278466..00000000 --- a/plugins/http/static/static.go +++ /dev/null @@ -1,88 +0,0 @@ -package static - -import ( - "io/fs" - "net/http" - "path/filepath" - "strings" - - httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" -) - -type ExtensionFilter struct { - allowed map[string]struct{} - forbidden map[string]struct{} -} - -func NewExtensionFilter(allow, forbid []string) *ExtensionFilter { - ef := &ExtensionFilter{ - allowed: make(map[string]struct{}, len(allow)), - forbidden: make(map[string]struct{}, len(forbid)), - } - - for i := 0; i < len(forbid); i++ { - // skip empty lines - if forbid[i] == "" { - continue - } - ef.forbidden[forbid[i]] = struct{}{} - } - - for i := 0; i < len(allow); i++ { - // skip empty lines - if allow[i] == "" { - continue - } - ef.allowed[allow[i]] = struct{}{} - } - - // check if any forbidden items presented in the allowed - // if presented, delete such items from allowed - for k := range ef.allowed { - if _, ok := ef.forbidden[k]; ok { - delete(ef.allowed, k) - } - } - - return ef -} - -type FileSystem struct { - ef *ExtensionFilter - // embedded - http.FileSystem -} - -// Open wrapper around http.FileSystem Open method, name here is the name of the -func (f FileSystem) Open(name string) (http.File, error) { - file, err := f.FileSystem.Open(name) - if err != nil { - return nil, err - } - - fstat, err := file.Stat() - if err != nil { - return nil, fs.ErrNotExist - } - - if fstat.IsDir() { - return nil, fs.ErrPermission - } - - ext := strings.ToLower(filepath.Ext(fstat.Name())) - if _, ok := f.ef.forbidden[ext]; ok { - return nil, fs.ErrPermission - } - - // if file extension is allowed, append it to the FileInfo slice - if _, ok := f.ef.allowed[ext]; ok { - return file, nil - } - - return nil, fs.ErrNotExist -} - -// FS is a constructor for the http.FileSystem -func FS(config *httpConfig.Static) http.FileSystem { - return FileSystem{NewExtensionFilter(config.Allow, config.Forbid), http.Dir(config.Dir)} -} diff --git a/plugins/kv/rpc.go b/plugins/kv/rpc.go index 4947dbe3..2d4babbe 100644 --- a/plugins/kv/rpc.go +++ b/plugins/kv/rpc.go @@ -1,11 +1,10 @@ package kv import ( - "unsafe" - "github.com/spiral/errors" "github.com/spiral/roadrunner/v2/plugins/kv/payload/generated" "github.com/spiral/roadrunner/v2/plugins/logger" + "github.com/spiral/roadrunner/v2/utils" ) // Wrapper for the plugin @@ -31,10 +30,10 @@ func (r *rpc) Has(in []byte, res *map[string]bool) error { if !dataRoot.Items(tmpItem, i) { continue } - keys = append(keys, strConvert(tmpItem.Key())) + keys = append(keys, utils.AsString(tmpItem.Key())) } - if st, ok := r.storages[strConvert(dataRoot.Storage())]; ok { + if st, ok := r.storages[utils.AsString(dataRoot.Storage())]; ok { ret, err := st.Has(keys...) if err != nil { return err @@ -73,7 +72,7 @@ func (r *rpc) Set(in []byte, ok *bool) error { items = append(items, itc) } - if st, exists := r.storages[strConvert(dataRoot.Storage())]; exists { + if st, exists := r.storages[utils.AsString(dataRoot.Storage())]; exists { err := st.Set(items...) if err != nil { return err @@ -104,7 +103,7 @@ func (r *rpc) MGet(in []byte, res *map[string]interface{}) error { keys = append(keys, string(tmpItem.Key())) } - if st, exists := r.storages[strConvert(dataRoot.Storage())]; exists { + if st, exists := r.storages[utils.AsString(dataRoot.Storage())]; exists { ret, err := st.MGet(keys...) if err != nil { return err @@ -143,7 +142,7 @@ func (r *rpc) MExpire(in []byte, ok *bool) error { items = append(items, itc) } - if st, exists := r.storages[strConvert(dataRoot.Storage())]; exists { + if st, exists := r.storages[utils.AsString(dataRoot.Storage())]; exists { err := st.MExpire(items...) if err != nil { return errors.E(op, err) @@ -173,7 +172,7 @@ func (r *rpc) TTL(in []byte, res *map[string]interface{}) error { keys = append(keys, string(tmpItem.Key())) } - if st, exists := r.storages[strConvert(dataRoot.Storage())]; exists { + if st, exists := r.storages[utils.AsString(dataRoot.Storage())]; exists { ret, err := st.TTL(keys...) if err != nil { return err @@ -201,7 +200,7 @@ func (r *rpc) Delete(in []byte, ok *bool) error { } keys = append(keys, string(tmpItem.Key())) } - if st, exists := r.storages[strConvert(dataRoot.Storage())]; exists { + if st, exists := r.storages[utils.AsString(dataRoot.Storage())]; exists { err := st.Delete(keys...) if err != nil { return errors.E(op, err) @@ -215,7 +214,3 @@ func (r *rpc) Delete(in []byte, ok *bool) error { *ok = false return errors.E(op, errors.Errorf("no such storage: %s", dataRoot.Storage())) } - -func strConvert(s []byte) string { - return *(*string)(unsafe.Pointer(&s)) -} diff --git a/plugins/server/config.go b/plugins/server/config.go index a4b0d91c..00ce4140 100644 --- a/plugins/server/config.go +++ b/plugins/server/config.go @@ -4,7 +4,7 @@ import ( "time" ) -// All config (.rr.yaml) +// Config All config (.rr.yaml) // For other section use pointer to distinguish between `empty` and `not present` type Config struct { // Server config section diff --git a/plugins/server/plugin.go b/plugins/server/plugin.go index 22b568d8..ef77f7ab 100644 --- a/plugins/server/plugin.go +++ b/plugins/server/plugin.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "strings" - "unsafe" "github.com/spiral/errors" "github.com/spiral/roadrunner/v2/pkg/transport" @@ -59,8 +58,7 @@ func (server *Plugin) Name() string { } // Available interface implementation -func (server *Plugin) Available() { -} +func (server *Plugin) Available() {} // Serve (Start) server plugin (just a mock here to satisfy interface) func (server *Plugin) Serve() chan error { @@ -239,10 +237,10 @@ func (server *Plugin) collectEvents(event interface{}) { case events.EventWorkerError: server.log.Error(strings.TrimRight(we.Payload.(error).Error(), " \n\t")) case events.EventWorkerLog: - server.log.Debug(strings.TrimRight(toString(we.Payload.([]byte)), " \n\t")) + server.log.Debug(strings.TrimRight(utils.AsString(we.Payload.([]byte)), " \n\t")) // stderr event is INFO level case events.EventWorkerStderr: - server.log.Info(strings.TrimRight(toString(we.Payload.([]byte)), " \n\t")) + server.log.Info(strings.TrimRight(utils.AsString(we.Payload.([]byte)), " \n\t")) } } } @@ -253,15 +251,10 @@ func (server *Plugin) collectWorkerLogs(event interface{}) { case events.EventWorkerError: server.log.Error(strings.TrimRight(we.Payload.(error).Error(), " \n\t")) case events.EventWorkerLog: - server.log.Debug(strings.TrimRight(toString(we.Payload.([]byte)), " \n\t")) + server.log.Debug(strings.TrimRight(utils.AsString(we.Payload.([]byte)), " \n\t")) // stderr event is INFO level case events.EventWorkerStderr: - server.log.Info(strings.TrimRight(toString(we.Payload.([]byte)), " \n\t")) + server.log.Info(strings.TrimRight(utils.AsString(we.Payload.([]byte)), " \n\t")) } } } - -// unsafe, but lightning fast []byte to string conversion -func toString(data []byte) string { - return *(*string)(unsafe.Pointer(&data)) -} diff --git a/plugins/service/process.go b/plugins/service/process.go index 49219eb0..cac5c41e 100644 --- a/plugins/service/process.go +++ b/plugins/service/process.go @@ -7,10 +7,10 @@ import ( "sync/atomic" "syscall" "time" - "unsafe" "github.com/spiral/errors" "github.com/spiral/roadrunner/v2/plugins/logger" + "github.com/spiral/roadrunner/v2/utils" ) // Process structure contains an information about process, restart information, log, errors, etc @@ -50,7 +50,7 @@ func NewServiceProcess(restartAfterExit bool, execTimeout time.Duration, restart // write message to the log (stderr) func (p *Process) Write(b []byte) (int, error) { - p.log.Info(toString(b)) + p.log.Info(utils.AsString(b)) return len(b), nil } @@ -145,9 +145,3 @@ func (p *Process) execHandler() { p.Unlock() } } - -// unsafe and fast []byte to string convert -//go:inline -func toString(data []byte) string { - return *(*string)(unsafe.Pointer(&data)) -} diff --git a/plugins/static/config.go b/plugins/static/config.go new file mode 100644 index 00000000..c3f9c17d --- /dev/null +++ b/plugins/static/config.go @@ -0,0 +1,55 @@ +package static + +import ( + "os" + + "github.com/spiral/errors" +) + +// Config describes file location and controls access to them. +type Config struct { + Static *struct { + // Dir contains name of directory to control access to. + // Default - "." + Dir string + + // CalculateEtag can be true/false and used to calculate etag for the static + CalculateEtag bool `mapstructure:"calculate_etag"` + + // Weak etag `W/` + Weak bool + + // forbid specifies list of file extensions which are forbidden for access. + // example: .php, .exe, .bat, .htaccess and etc. + Forbid []string + + // Allow specifies list of file extensions which are allowed for access. + // example: .php, .exe, .bat, .htaccess and etc. + Allow []string + + // Request headers to add to every static. + Request map[string]string + + // Response headers to add to every static. + Response map[string]string + } +} + +// Valid returns nil if config is valid. +func (c *Config) Valid() error { + const op = errors.Op("static_plugin_valid") + st, err := os.Stat(c.Static.Dir) + if err != nil { + if os.IsNotExist(err) { + return errors.E(op, errors.Errorf("root directory '%s' does not exists", c.Static.Dir)) + } + + return err + } + + if !st.IsDir() { + return errors.E(op, errors.Errorf("invalid root directory '%s'", c.Static.Dir)) + } + + return nil +} diff --git a/plugins/http/static/etag.go b/plugins/static/etag.go index 5d41cc53..5ee0d2f3 100644 --- a/plugins/http/static/etag.go +++ b/plugins/static/etag.go @@ -4,10 +4,8 @@ import ( "hash/crc32" "io" "net/http" - "os" - "unsafe" - httpConfig "github.com/spiral/roadrunner/v2/plugins/http/config" + "github.com/spiral/roadrunner/v2/utils" ) const etag string = "Etag" @@ -18,7 +16,22 @@ var weakPrefix = []byte(`W/`) // CRC32 table var crc32q = crc32.MakeTable(0x48D90782) -func SetEtag(cfg *httpConfig.Static, f *os.File, w http.ResponseWriter) { +// SetEtag sets etag for the file +func SetEtag(weak bool, f http.File, name string, w http.ResponseWriter) { + // preallocate + calculatedEtag := make([]byte, 0, 64) + + // write weak + if weak { + calculatedEtag = append(calculatedEtag, weakPrefix...) + calculatedEtag = append(calculatedEtag, '"') + calculatedEtag = appendUint(calculatedEtag, crc32.Checksum(utils.AsBytes(name), crc32q)) + calculatedEtag = append(calculatedEtag, '"') + + w.Header().Set(etag, utils.AsString(calculatedEtag)) + return + } + // read the file content body, err := io.ReadAll(f) if err != nil { @@ -30,21 +43,13 @@ func SetEtag(cfg *httpConfig.Static, f *os.File, w http.ResponseWriter) { return } - // preallocate - calculatedEtag := make([]byte, 0, 64) - - // write weak - if cfg.Weak { - calculatedEtag = append(calculatedEtag, weakPrefix...) - } - calculatedEtag = append(calculatedEtag, '"') calculatedEtag = appendUint(calculatedEtag, uint32(len(body))) calculatedEtag = append(calculatedEtag, '-') calculatedEtag = appendUint(calculatedEtag, crc32.Checksum(body, crc32q)) calculatedEtag = append(calculatedEtag, '"') - w.Header().Set(etag, byteToSrt(calculatedEtag)) + w.Header().Set(etag, utils.AsString(calculatedEtag)) } // appendUint appends n to dst and returns the extended dst. @@ -65,7 +70,3 @@ func appendUint(dst []byte, n uint32) []byte { dst = append(dst, buf[i:]...) return dst } - -func byteToSrt(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} diff --git a/plugins/static/plugin.go b/plugins/static/plugin.go new file mode 100644 index 00000000..f6d9a0f2 --- /dev/null +++ b/plugins/static/plugin.go @@ -0,0 +1,188 @@ +package static + +import ( + "net/http" + "path" + "strings" + + "github.com/spiral/errors" + "github.com/spiral/roadrunner/v2/plugins/config" + "github.com/spiral/roadrunner/v2/plugins/logger" +) + +// PluginName contains default service name. +const PluginName = "static" + +const RootPluginName = "http" + +// Plugin serves static files. Potentially convert into middleware? +type Plugin struct { + // server configuration (location, forbidden files and etc) + cfg *Config + + log logger.Logger + + // root is initiated http directory + root http.Dir + + // file extensions which are allowed to be served + allowedExtensions map[string]struct{} + + // file extensions which are forbidden to be served + forbiddenExtensions map[string]struct{} +} + +// Init must return configure service and return true if service hasStatus enabled. Must return error in case of +// misconfiguration. Services must not be used without proper configuration pushed first. +func (s *Plugin) Init(cfg config.Configurer, log logger.Logger) error { + const op = errors.Op("static_plugin_init") + if !cfg.Has(RootPluginName) { + return errors.E(op, errors.Disabled) + } + + err := cfg.UnmarshalKey(RootPluginName, &s.cfg) + if err != nil { + return errors.E(op, errors.Disabled, err) + } + + if s.cfg.Static == nil { + return errors.E(op, errors.Disabled) + } + + s.log = log + s.root = http.Dir(s.cfg.Static.Dir) + + err = s.cfg.Valid() + if err != nil { + return errors.E(op, err) + } + + // create 2 hashmaps with the allowed and forbidden file extensions + s.allowedExtensions = make(map[string]struct{}, len(s.cfg.Static.Allow)) + s.forbiddenExtensions = make(map[string]struct{}, len(s.cfg.Static.Forbid)) + + // init forbidden + for i := 0; i < len(s.cfg.Static.Forbid); i++ { + // skip empty lines + if s.cfg.Static.Forbid[i] == "" { + continue + } + s.forbiddenExtensions[s.cfg.Static.Forbid[i]] = struct{}{} + } + + // init allowed + for i := 0; i < len(s.cfg.Static.Allow); i++ { + // skip empty lines + if s.cfg.Static.Allow[i] == "" { + continue + } + s.allowedExtensions[s.cfg.Static.Allow[i]] = struct{}{} + } + + // check if any forbidden items presented in the allowed + // if presented, delete such items from allowed + for k := range s.forbiddenExtensions { + delete(s.allowedExtensions, k) + } + + // at this point we have distinct allowed and forbidden hashmaps, also with alwaysServed + return nil +} + +func (s *Plugin) Name() string { + return PluginName +} + +// Middleware must return true if request/response pair is handled within the middleware. +func (s *Plugin) Middleware(next http.Handler) http.Handler { + // Define the http.HandlerFunc + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // do not allow paths like ../../resource + // only specified folder and resources in it + // https://lgtm.com/rules/1510366186013/ + if strings.Contains(r.URL.Path, "..") { + w.WriteHeader(http.StatusForbidden) + return + } + + if s.cfg.Static.Request != nil { + for k, v := range s.cfg.Static.Request { + r.Header.Add(k, v) + } + } + + if s.cfg.Static.Response != nil { + for k, v := range s.cfg.Static.Response { + w.Header().Set(k, v) + } + } + + // first - create a proper file path + fPath := path.Clean(r.URL.Path) + ext := strings.ToLower(path.Ext(fPath)) + + // check that file extension in the forbidden list + if _, ok := s.forbiddenExtensions[ext]; ok { + s.log.Debug("file extension is forbidden", "ext", ext) + next.ServeHTTP(w, r) + return + } + + // if we have some allowed extensions, we should check them + // if not - all extensions allowed except forbidden + if len(s.allowedExtensions) > 0 { + // not found in allowed + if _, ok := s.allowedExtensions[ext]; !ok { + next.ServeHTTP(w, r) + return + } + + // file extension allowed + } + + // ok, file is not in the forbidden list + // Stat it and get file info + f, err := s.root.Open(fPath) + if err != nil { + // else no such file, show error in logs only in debug mode + s.log.Debug("no such file or directory", "error", err) + // pass request to the worker + next.ServeHTTP(w, r) + return + } + + // at high confidence there is should not be an error + // because we stat-ed the path previously and know, that that is file (not a dir), and it exists + finfo, err := f.Stat() + if err != nil { + // else no such file, show error in logs only in debug mode + s.log.Debug("no such file or directory", "error", err) + // pass request to the worker + next.ServeHTTP(w, r) + return + } + + defer func() { + err = f.Close() + if err != nil { + s.log.Error("file close error", "error", err) + } + }() + + // if provided path to the dir, do not serve the dir, but pass the request to the worker + if finfo.IsDir() { + s.log.Debug("possible path to dir provided") + // pass request to the worker + next.ServeHTTP(w, r) + return + } + + // set etag + if s.cfg.Static.CalculateEtag { + SetEtag(s.cfg.Static.Weak, f, finfo.Name(), w) + } + + // we passed all checks - serve the file + http.ServeContent(w, r, finfo.Name(), finfo.ModTime(), f) + }) +} diff --git a/tests/plugins/http/configs/.rr-http-static-etags.yaml b/tests/plugins/http/configs/.rr-http-static-etags.yaml index e18c50dd..b09de0f4 100644 --- a/tests/plugins/http/configs/.rr-http-static-etags.yaml +++ b/tests/plugins/http/configs/.rr-http-static-etags.yaml @@ -15,8 +15,7 @@ http: uploads: forbid: [ ".php", ".exe", ".bat" ] static: - dir: "../../../" - pattern: "/tests/" + dir: "../../../tests" forbid: [ "" ] allow: [ ".txt", ".php" ] calculate_etag: true diff --git a/tests/plugins/http/configs/.rr-http-static-files.yaml b/tests/plugins/http/configs/.rr-http-static-files.yaml index 5d8b50e8..18c6107d 100644 --- a/tests/plugins/http/configs/.rr-http-static-files.yaml +++ b/tests/plugins/http/configs/.rr-http-static-files.yaml @@ -15,8 +15,7 @@ http: uploads: forbid: [ ".php", ".exe", ".bat" ] static: - dir: "../../../" - pattern: "/tests/" + dir: "../../../tests" allow: [ ".ico" ] forbid: [ ".php", ".htaccess" ] @@ -25,6 +24,7 @@ http: max_jobs: 0 allocate_timeout: 60s destroy_timeout: 60s + logs: mode: development - level: error + level: info diff --git a/tests/plugins/http/configs/.rr-http-static-security.yaml b/tests/plugins/http/configs/.rr-http-static-security.yaml index bbec13f9..e2e3af2a 100644 --- a/tests/plugins/http/configs/.rr-http-static-security.yaml +++ b/tests/plugins/http/configs/.rr-http-static-security.yaml @@ -15,8 +15,7 @@ http: uploads: forbid: [ ".php", ".exe", ".bat" ] static: - dir: "../../../" - pattern: "/tests/" + dir: "../../../tests" forbid: [ "" ] allow: [ ".txt", ".php" ] calculate_etag: true diff --git a/tests/plugins/http/configs/.rr-http-static.yaml b/tests/plugins/http/configs/.rr-http-static.yaml index bbec13f9..e2e3af2a 100644 --- a/tests/plugins/http/configs/.rr-http-static.yaml +++ b/tests/plugins/http/configs/.rr-http-static.yaml @@ -15,8 +15,7 @@ http: uploads: forbid: [ ".php", ".exe", ".bat" ] static: - dir: "../../../" - pattern: "/tests/" + dir: "../../../tests" forbid: [ "" ] allow: [ ".txt", ".php" ] calculate_etag: true diff --git a/tests/plugins/http/http_plugin_test.go b/tests/plugins/http/http_plugin_test.go index 8f76e3ba..128eec26 100644 --- a/tests/plugins/http/http_plugin_test.go +++ b/tests/plugins/http/http_plugin_test.go @@ -30,6 +30,7 @@ import ( "github.com/spiral/roadrunner/v2/plugins/logger" "github.com/spiral/roadrunner/v2/plugins/resetter" "github.com/spiral/roadrunner/v2/plugins/server" + "github.com/spiral/roadrunner/v2/plugins/static" "github.com/spiral/roadrunner/v2/tests/mocks" "github.com/yookoala/gofast" @@ -1578,6 +1579,7 @@ func TestStaticEtagPlugin(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) @@ -1633,7 +1635,7 @@ func TestStaticEtagPlugin(t *testing.T) { func serveStaticSampleEtag(t *testing.T) { // OK 200 response - b, r, err := get("http://localhost:21603/tests/static/sample.txt") + b, r, err := get("http://localhost:21603/sample.txt") assert.NoError(t, err) assert.Equal(t, "sample\n", b) assert.Equal(t, r.StatusCode, http.StatusOK) @@ -1646,7 +1648,7 @@ func serveStaticSampleEtag(t *testing.T) { Timeout: time.Second * 5, } - parsedURL, _ := url.Parse("http://localhost:21603/tests/static/sample.txt") + parsedURL, _ := url.Parse("http://localhost:21603/sample.txt") req := &http.Request{ Method: http.MethodGet, @@ -1675,6 +1677,7 @@ func TestStaticPluginSecurity(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) @@ -1804,7 +1807,7 @@ func serveStaticSampleNotAllowedPath(t *testing.T) { _, r, err := get("http://localhost:21603/../../../../tests/../static/sample.txt") assert.NoError(t, err) - assert.Equal(t, r.StatusCode, 200) + assert.Equal(t, 403, r.StatusCode) _ = r.Body.Close() } @@ -1823,6 +1826,7 @@ func TestStaticPlugin(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) @@ -1879,7 +1883,7 @@ func TestStaticPlugin(t *testing.T) { } func staticHeaders(t *testing.T) { - req, err := http.NewRequest("GET", "http://localhost:21603/tests/client.php", nil) + req, err := http.NewRequest("GET", "http://localhost:21603/client.php", nil) if err != nil { t.Fatal(err) } @@ -1907,7 +1911,7 @@ func staticHeaders(t *testing.T) { } func staticNotForbid(t *testing.T) { - b, r, err := get("http://localhost:21603/tests/client.php") + b, r, err := get("http://localhost:21603/client.php") assert.NoError(t, err) assert.Equal(t, all("../../../tests/client.php"), b) assert.Equal(t, all("../../../tests/client.php"), b) @@ -1915,7 +1919,7 @@ func staticNotForbid(t *testing.T) { } func serveStaticSample(t *testing.T) { - b, r, err := get("http://localhost:21603/tests/static/sample.txt") + b, r, err := get("http://localhost:21603/sample.txt") assert.NoError(t, err) assert.Equal(t, "sample\n", b) _ = r.Body.Close() @@ -1936,6 +1940,7 @@ func TestStaticDisabled_Error(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) assert.Error(t, cont.Init()) @@ -1956,6 +1961,7 @@ func TestStaticFilesDisabled(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) @@ -2032,10 +2038,13 @@ func TestStaticFilesForbid(t *testing.T) { mockLogger.EXPECT().Debug("worker destructed", "pid", gomock.Any()).AnyTimes() mockLogger.EXPECT().Debug("worker constructed", "pid", gomock.Any()).AnyTimes() - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/http?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/client.XXX?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) - mockLogger.EXPECT().Debug("201 GET http://localhost:34653/tests/client.php?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/http?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/client.XXX?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) + mockLogger.EXPECT().Debug("201 GET http://localhost:34653/client.php?hello=world", "remote", "127.0.0.1", "elapsed", gomock.Any()).MinTimes(1) mockLogger.EXPECT().Error("file open error", "error", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("no such file or directory", "error", gomock.Any()).AnyTimes() + mockLogger.EXPECT().Debug("possible path to dir provided").AnyTimes() + mockLogger.EXPECT().Debug("file extension is forbidden", gomock.Any(), gomock.Any()).AnyTimes() mockLogger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() // placeholder for the workerlogerror err = cont.RegisterAll( @@ -2044,6 +2053,7 @@ func TestStaticFilesForbid(t *testing.T) { &server.Plugin{}, &httpPlugin.Plugin{}, &gzip.Plugin{}, + &static.Plugin{}, ) assert.NoError(t, err) @@ -2094,37 +2104,29 @@ func TestStaticFilesForbid(t *testing.T) { t.Run("StaticTestFilesDir", staticTestFilesDir) t.Run("StaticNotFound", staticNotFound) t.Run("StaticFilesForbid", staticFilesForbid) - t.Run("StaticFilesAlways", staticFilesAlways) stopCh <- struct{}{} wg.Wait() } func staticTestFilesDir(t *testing.T) { - b, r, err := get("http://localhost:34653/tests/http?hello=world") + b, r, err := get("http://localhost:34653/http?hello=world") assert.NoError(t, err) - assert.Equal(t, "403 Forbidden\n", b) + assert.Equal(t, "WORLD", b) _ = r.Body.Close() } func staticNotFound(t *testing.T) { - b, _, _ := get("http://localhost:34653/tests/client.XXX?hello=world") //nolint:bodyclose - assert.Equal(t, "404 page not found\n", b) -} - -func staticFilesAlways(t *testing.T) { - _, r, err := get("http://localhost:34653/tests/favicon.ico") - assert.NoError(t, err) - assert.Equal(t, 404, r.StatusCode) - _ = r.Body.Close() + b, _, _ := get("http://localhost:34653/client.XXX?hello=world") //nolint:bodyclose + assert.Equal(t, "WORLD", b) } func staticFilesForbid(t *testing.T) { - b, r, err := get("http://localhost:34653/tests/client.php?hello=world") + b, r, err := get("http://localhost:34653/client.php?hello=world") if err != nil { t.Fatal(err) } - assert.Equal(t, "403 Forbidden\n", b) + assert.Equal(t, "WORLD", b) _ = r.Body.Close() } diff --git a/tests/plugins/server/configs/.rr-no-app-section.yaml b/tests/plugins/server/configs/.rr-no-app-section.yaml index e44eeb56..d28265d5 100644 --- a/tests/plugins/server/configs/.rr-no-app-section.yaml +++ b/tests/plugins/server/configs/.rr-no-app-section.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "pipes" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/plugins/server/configs/.rr-sockets.yaml b/tests/plugins/server/configs/.rr-sockets.yaml index 0bc2d0f9..4c57f36f 100644 --- a/tests/plugins/server/configs/.rr-sockets.yaml +++ b/tests/plugins/server/configs/.rr-sockets.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "unix://unix.sock" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/plugins/server/configs/.rr-tcp.yaml b/tests/plugins/server/configs/.rr-tcp.yaml index f4580460..4582482f 100644 --- a/tests/plugins/server/configs/.rr-tcp.yaml +++ b/tests/plugins/server/configs/.rr-tcp.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "tcp://localhost:9999" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/plugins/server/configs/.rr-wrong-command.yaml b/tests/plugins/server/configs/.rr-wrong-command.yaml index c97d8b7e..9d105d90 100644 --- a/tests/plugins/server/configs/.rr-wrong-command.yaml +++ b/tests/plugins/server/configs/.rr-wrong-command.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "pipes" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/plugins/server/configs/.rr-wrong-relay.yaml b/tests/plugins/server/configs/.rr-wrong-relay.yaml index 9722a487..c4d1edb0 100644 --- a/tests/plugins/server/configs/.rr-wrong-relay.yaml +++ b/tests/plugins/server/configs/.rr-wrong-relay.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "pupes" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/plugins/server/configs/.rr.yaml b/tests/plugins/server/configs/.rr.yaml index e44eeb56..d28265d5 100644 --- a/tests/plugins/server/configs/.rr.yaml +++ b/tests/plugins/server/configs/.rr.yaml @@ -3,10 +3,10 @@ server: user: "" group: "" env: - "RR_CONFIG": "/some/place/on/the/C134" - "RR_CONFIG2": "C138" + - RR_CONFIG: "/some/place/on/the/C134" + - RR_CONFIG2: "C138" relay: "pipes" relay_timeout: "20s" logs: mode: development - level: error
\ No newline at end of file + level: error diff --git a/tests/static/sample.txt b/tests/sample.txt index d64a3d96..d64a3d96 100644 --- a/tests/static/sample.txt +++ b/tests/sample.txt diff --git a/utils/convert.go b/utils/convert.go new file mode 100644 index 00000000..8d153ce5 --- /dev/null +++ b/utils/convert.go @@ -0,0 +1,34 @@ +package utils + +import ( + "reflect" + "unsafe" +) + +// AsBytes returns a slice that refers to the data backing the string s. +func AsBytes(s string) []byte { + // get the pointer to the data of the string + p := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data) + + var b []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + hdr.Data = uintptr(p) + // we need to set the cap and len for the string to byte convert + // because string is shorter than []bytes + hdr.Cap = len(s) + hdr.Len = len(s) + + return b +} + +// AsString returns a string that refers to the data backing the slice s. +func AsString(b []byte) string { + p := unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&b)).Data) + + var s string + hdr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + hdr.Data = uintptr(p) + hdr.Len = len(b) + + return s +} |