OrgGo
/
caddy
mirror of https://github.com/caddyserver/caddy.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

filesystem: Globally declared filesystems, `fs` directive (#5833)

This commit is contained in:
a 2024-01-13 14:12:43 -06:00 committed by GitHub
parent b359ca565c
commit c839a98ff5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 450 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
caddy.go
View File

@ -39,6 +39,7 @@ import (
"github.com/google/uuid"
"go.uber.org/zap"
"github.com/caddyserver/caddy/v2/internal/filesystems"
"github.com/caddyserver/caddy/v2/notify"
)
@ -84,6 +85,9 @@ type Config struct {
storage certmagic.Storage
cancelFunc context.CancelFunc
// filesystems is a dict of filesystems that will later be loaded from and added to.
filesystems FileSystems
}
// App is a thing that Caddy runs.
@ -447,6 +451,9 @@ func run(newCfg *Config, start bool) (Context, error) {
}
}
// create the new filesystem map
newCfg.filesystems = &filesystems.FilesystemMap{}
// prepare the new config for use
newCfg.apps = make(map[string]App)

2
caddyconfig/caddyfile/dispenser_test.go
View File

@ -305,7 +305,7 @@ func TestDispenser_ArgErr_Err(t *testing.T) {
t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err)
}
var ErrBarIsFull = errors.New("bar is full")
ErrBarIsFull := errors.New("bar is full")
bookingError := d.Errf("unable to reserve: %w", ErrBarIsFull)
if !errors.Is(bookingError, ErrBarIsFull) {
t.Errorf("Errf(): should be able to unwrap the error chain")

14
caddyconfig/caddyfile/parse_test.go
View File

@ -22,7 +22,7 @@ import (
)
func TestParseVariadic(t *testing.T) {
var args = make([]string, 10)
args := make([]string, 10)
for i, tc := range []struct {
input string
result bool
@ -111,7 +111,6 @@ func TestAllTokens(t *testing.T) {
input := []byte("a b c\nd e")
expected := []string{"a", "b", "c", "d", "e"}
tokens, err := allTokens("TestAllTokens", input)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
@ -149,7 +148,8 @@ func TestParseOneAndImport(t *testing.T) {
"localhost",
}, []int{1}},
{`localhost:1234
{
`localhost:1234
dir1 foo bar`, false, []string{
"localhost:1234",
}, []int{3},
@ -407,13 +407,13 @@ func TestRecursiveImport(t *testing.T) {
err = os.WriteFile(recursiveFile1, []byte(
`localhost
dir1
import recursive_import_test2`), 0644)
import recursive_import_test2`), 0o644)
if err != nil {
t.Fatal(err)
}
defer os.Remove(recursiveFile1)
err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0644)
err = os.WriteFile(recursiveFile2, []byte("dir2 1"), 0o644)
if err != nil {
t.Fatal(err)
}
@ -441,7 +441,7 @@ func TestRecursiveImport(t *testing.T) {
err = os.WriteFile(recursiveFile1, []byte(
`localhost
dir1
import `+recursiveFile2), 0644)
import `+recursiveFile2), 0o644)
if err != nil {
t.Fatal(err)
}
@ -495,7 +495,7 @@ func TestDirectiveImport(t *testing.T) {
}
err = os.WriteFile(directiveFile, []byte(`prop1 1
prop2 2`), 0644)
prop2 2`), 0o644)
if err != nil {
t.Fatal(err)
}

18
caddyconfig/httpcaddyfile/builtins.go
View File

@ -40,6 +40,7 @@ import (
func init() {
RegisterDirective("bind", parseBind)
RegisterDirective("tls", parseTLS)
RegisterHandlerDirective("fs", parseFilesystem)
RegisterHandlerDirective("root", parseRoot)
RegisterHandlerDirective("vars", parseVars)
RegisterHandlerDirective("redir", parseRedir)
@ -658,6 +659,23 @@ func parseRoot(h Helper) (caddyhttp.MiddlewareHandler, error) {
return caddyhttp.VarsMiddleware{"root": root}, nil
}
// parseFilesystem parses the fs directive. Syntax:
//
// fs <filesystem>
func parseFilesystem(h Helper) (caddyhttp.MiddlewareHandler, error) {
var name string
for h.Next() {
if !h.NextArg() {
return nil, h.ArgErr()
}
name = h.Val()
if h.NextArg() {
return nil, h.ArgErr()
}
}
return caddyhttp.VarsMiddleware{"fs": name}, nil
}
// parseVars parses the vars directive. See its UnmarshalCaddyfile method for syntax.
func parseVars(h Helper) (caddyhttp.MiddlewareHandler, error) {
v := new(caddyhttp.VarsMiddleware)

1
caddyconfig/httpcaddyfile/directives.go
View File

@ -41,6 +41,7 @@ var directiveOrder = []string{
"map",
"vars",
"fs",
"root",
"skip_log",

9
caddyconfig/httpcaddyfile/directives_test.go
View File

@ -31,20 +31,23 @@ func TestHostsFromKeys(t *testing.T) {
[]Address{
{Original: ":2015", Port: "2015"},
},
[]string{}, []string{},
[]string{},
[]string{},
},
{
[]Address{
{Original: ":443", Port: "443"},
},
[]string{}, []string{},
[]string{},
[]string{},
},
{
[]Address{
{Original: "foo", Host: "foo"},
{Original: ":2015", Port: "2015"},
},
[]string{}, []string{"foo"},
[]string{},
[]string{"foo"},
},
{
[]Address{

7
caddyconfig/httpcaddyfile/httptype.go
View File

@ -271,6 +271,12 @@ func (st ServerType) Setup(
if !reflect.DeepEqual(pkiApp, &caddypki.PKI{CAs: make(map[string]*caddypki.CA)}) {
cfg.AppsRaw["pki"] = caddyconfig.JSON(pkiApp, &warnings)
}
if filesystems, ok := options["filesystem"].(caddy.Module); ok {
cfg.AppsRaw["caddy.filesystems"] = caddyconfig.JSON(
filesystems,
&warnings)
}
if storageCvtr, ok := options["storage"].(caddy.StorageConverter); ok {
cfg.StorageRaw = caddyconfig.JSONModuleObject(storageCvtr,
"module",
@ -280,7 +286,6 @@ func (st ServerType) Setup(
if adminConfig, ok := options["admin"].(*caddy.AdminConfig); ok && adminConfig != nil {
cfg.Admin = adminConfig
}
if pc, ok := options["persist_config"].(string); ok && pc == "off" {
if cfg.Admin == nil {
cfg.Admin = new(caddy.AdminConfig)

5
caddytest/integration/caddyfile_test.go
View File

@ -9,7 +9,6 @@ import (
)
func TestRespond(t *testing.T) {
// arrange
tester := caddytest.NewTester(t)
tester.InitServer(`
@ -32,7 +31,6 @@ func TestRespond(t *testing.T) {
}
func TestRedirect(t *testing.T) {
// arrange
tester := caddytest.NewTester(t)
tester.InitServer(`
@ -61,7 +59,6 @@ func TestRedirect(t *testing.T) {
}
func TestDuplicateHosts(t *testing.T) {
// act and assert
caddytest.AssertLoadError(t,
`
@ -76,7 +73,6 @@ func TestDuplicateHosts(t *testing.T) {
}
func TestReadCookie(t *testing.T) {
localhost, _ := url.Parse("http://localhost")
cookie := http.Cookie{
Name: "clientname",
@ -110,7 +106,6 @@ func TestReadCookie(t *testing.T) {
}
func TestReplIndex(t *testing.T) {
tester := caddytest.NewTester(t)
tester.InitServer(`
{

1
caddytest/integration/reverseproxy_test.go
View File

@ -57,7 +57,6 @@ func TestSRVReverseProxy(t *testing.T) {
}
func TestDialWithPlaceholderUnix(t *testing.T) {
if runtime.GOOS == "windows" {
t.SkipNow()
}

2
caddytest/integration/sni_test.go
View File

@ -7,7 +7,6 @@ import (
)
func TestDefaultSNI(t *testing.T) {
// arrange
tester := caddytest.NewTester(t)
tester.InitServer(`{
@ -107,7 +106,6 @@ func TestDefaultSNI(t *testing.T) {
}
func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
// arrange
tester := caddytest.NewTester(t)
tester.InitServer(`

1
caddytest/integration/stream_test.go
View File

@ -360,7 +360,6 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
func testH2ToH1ChunkedResponseServeH1(t *testing.T) *http.Server {
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Host != "127.0.0.1:9443" {
t.Errorf("r.Host doesn't match, %v!", r.Host)
w.WriteHeader(http.StatusNotFound)

12
context.go
View File

@ -23,6 +23,8 @@ import (
"github.com/caddyserver/certmagic"
"go.uber.org/zap"
"github.com/caddyserver/caddy/v2/internal/filesystems"
)
// Context is a type which defines the lifetime of modules that
@ -37,6 +39,7 @@ import (
// not actually need to do this).
type Context struct {
context.Context
moduleInstances map[string][]Module
cfg *Config
cleanupFuncs []func()
@ -81,6 +84,15 @@ func (ctx *Context) OnCancel(f func()) {
ctx.cleanupFuncs = append(ctx.cleanupFuncs, f)
}
// Filesystems returns a ref to the FilesystemMap
func (ctx *Context) Filesystems() FileSystems {
// if no config is loaded, we use a default filesystemmap, which includes the osfs
if ctx.cfg == nil {
return &filesystems.FilesystemMap{}
}
return ctx.cfg.filesystems
}
// LoadModule loads the Caddy module(s) from the specified field of the parent struct
// pointer and returns the loaded module(s). The struct pointer and its field name as
// a string are necessary so that reflection can be used to read the struct tag on the

10
filesystem.go Normal file
View File

@ -0,0 +1,10 @@
package caddy
import "io/fs"
type FileSystems interface {
Register(k string, v fs.FS)
Unregister(k string)
Get(k string) (v fs.FS, ok bool)
Default() fs.FS
}

77
internal/filesystems/map.go Normal file
View File

@ -0,0 +1,77 @@
package filesystems
import (
"io/fs"
"strings"
"sync"
)
const (
DefaultFilesystemKey = "default"
)
var DefaultFilesystem = &wrapperFs{key: DefaultFilesystemKey, FS: OsFS{}}
// wrapperFs exists so can easily add to wrapperFs down the line
type wrapperFs struct {
key string
fs.FS
}
// FilesystemMap stores a map of filesystems
// the empty key will be overwritten to be the default key
// it includes a default filesystem, based off the os fs
type FilesystemMap struct {
m sync.Map
}
// note that the first invocation of key cannot be called in a racy context.
func (f *FilesystemMap) key(k string) string {
if k == "" {
k = DefaultFilesystemKey
}
return k
}
// Register will add the filesystem with key to later be retrieved
// A call with a nil fs will call unregister, ensuring that a call to Default() will never be nil
func (f *FilesystemMap) Register(k string, v fs.FS) {
k = f.key(k)
if v == nil {
f.Unregister(k)
return
}
f.m.Store(k, &wrapperFs{key: k, FS: v})
}
// Unregister will remove the filesystem with key from the filesystem map
// if the key is the default key, it will set the default to the osFS instead of deleting it
// modules should call this on cleanup to be safe
func (f *FilesystemMap) Unregister(k string) {
k = f.key(k)
if k == DefaultFilesystemKey {
f.m.Store(k, DefaultFilesystem)
} else {
f.m.Delete(k)
}
}
// Get will get a filesystem with a given key
func (f *FilesystemMap) Get(k string) (v fs.FS, ok bool) {
k = f.key(k)
c, ok := f.m.Load(strings.TrimSpace(k))
if !ok {
if k == DefaultFilesystemKey {
f.m.Store(k, DefaultFilesystem)
return DefaultFilesystem, true
}
return nil, ok
}
return c.(fs.FS), true
}
// Default will get the default filesystem in the filesystem map
func (f *FilesystemMap) Default() fs.FS {
val, _ := f.Get(DefaultFilesystemKey)
return val
}

29
internal/filesystems/os.go Normal file
View File

@ -0,0 +1,29 @@
package filesystems
import (
"io/fs"
"os"
"path/filepath"
)
// OsFS is a simple fs.FS implementation that uses the local
// file system. (We do not use os.DirFS because we do our own
// rooting or path prefixing without being constrained to a single
// root folder. The standard os.DirFS implementation is problematic
// since roots can be dynamic in our application.)
//
// OsFS also implements fs.StatFS, fs.GlobFS, fs.ReadDirFS, and fs.ReadFileFS.
type OsFS struct{}
func (OsFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (OsFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }
func (OsFS) Glob(pattern string) ([]string, error) { return filepath.Glob(pattern) }
func (OsFS) ReadDir(name string) ([]fs.DirEntry, error) { return os.ReadDir(name) }
func (OsFS) ReadFile(name string) ([]byte, error) { return os.ReadFile(name) }
var (
_ fs.StatFS = (*OsFS)(nil)
_ fs.GlobFS = (*OsFS)(nil)
_ fs.ReadDirFS = (*OsFS)(nil)
_ fs.ReadFileFS = (*OsFS)(nil)
)

112
modules/caddyfs/filesystem.go Normal file
View File

@ -0,0 +1,112 @@
package caddyfs
import (
"encoding/json"
"fmt"
"io/fs"
"go.uber.org/zap"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
)
func init() {
caddy.RegisterModule(Filesystems{})
httpcaddyfile.RegisterGlobalOption("filesystem", parseFilesystems)
}
type moduleEntry struct {
Key string `json:"name,omitempty"`
FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
fileSystem fs.FS
}
// Filesystems loads caddy.fs modules into the global filesystem map
type Filesystems struct {
Filesystems []*moduleEntry `json:"filesystems"`
defers []func()
}
func parseFilesystems(d *caddyfile.Dispenser, existingVal any) (any, error) {
p := &Filesystems{}
current, ok := existingVal.(*Filesystems)
if ok {
p = current
}
x := &moduleEntry{}
err := x.UnmarshalCaddyfile(d)
if err != nil {
return nil, err
}
p.Filesystems = append(p.Filesystems, x)
return p, nil
}
// CaddyModule returns the Caddy module information.
func (Filesystems) CaddyModule() caddy.ModuleInfo {
return caddy.ModuleInfo{
ID: "caddy.filesystems",
New: func() caddy.Module { return new(Filesystems) },
}
}
func (xs *Filesystems) Start() error { return nil }
func (xs *Filesystems) Stop() error { return nil }
func (xs *Filesystems) Provision(ctx caddy.Context) error {
// load the filesystem module
for _, f := range xs.Filesystems {
if len(f.FileSystemRaw) > 0 {
mod, err := ctx.LoadModule(f, "FileSystemRaw")
if err != nil {
return fmt.Errorf("loading file system module: %v", err)
}
f.fileSystem = mod.(fs.FS)
}
// register that module
ctx.Logger().Debug("registering fs", zap.String("fs", f.Key))
ctx.Filesystems().Register(f.Key, f.fileSystem)
// remember to unregister the module when we are done
xs.defers = append(xs.defers, func() {
ctx.Logger().Debug("registering fs", zap.String("fs", f.Key))
ctx.Filesystems().Unregister(f.Key)
})
}
return nil
}
func (f *Filesystems) Cleanup() error {
for _, v := range f.defers {
v()
}
return nil
}
func (f *moduleEntry) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
for d.Next() {
// key required for now
if !d.Args(&f.Key) {
return d.ArgErr()
}
// get the module json
if !d.NextArg() {
return d.ArgErr()
}
name := d.Val()
modID := "caddy.fs." + name
unm, err := caddyfile.UnmarshalModule(d, modID)
if err != nil {
return err
}
fsys, ok := unm.(fs.FS)
if !ok {
return d.Errf("module %s (%T) is not a supported file system implementation (requires fs.FS)", modID, unm)
}
f.FileSystemRaw = caddyconfig.JSONModuleObject(fsys, "backend", name, nil)
}
return nil
}

2
modules/caddyhttp/encode/encode_test.go
View File

@ -105,7 +105,6 @@ func TestPreferOrder(t *testing.T) {
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
if test.accept == "" {
r.Header.Del("Accept-Encoding")
} else {
@ -258,7 +257,6 @@ func TestValidate(t *testing.T) {
t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
}
})
}
}

14
modules/caddyhttp/fileserver/browse.go
View File

@ -52,7 +52,7 @@ type Browse struct {
TemplateFile string `json:"template_file,omitempty"`
}
func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
fsrv.logger.Debug("browse enabled; listing directory contents",
zap.String("path", dirPath),
zap.String("root", root))
@ -82,7 +82,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
}
}
dir, err := fsrv.openFile(dirPath, w)
dir, err := fsrv.openFile(fileSystem, dirPath, w)
if err != nil {
return err
}
@ -91,7 +91,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
// TODO: not entirely sure if path.Clean() is necessary here but seems like a safe plan (i.e. /%2e%2e%2f) - someone could verify this
listing, err := fsrv.loadDirectoryContents(r.Context(), dir.(fs.ReadDirFile), root, path.Clean(r.URL.EscapedPath()), repl)
listing, err := fsrv.loadDirectoryContents(r.Context(), fileSystem, dir.(fs.ReadDirFile), root, path.Clean(r.URL.EscapedPath()), repl)
switch {
case errors.Is(err, fs.ErrPermission):
return caddyhttp.Error(http.StatusForbidden, err)
@ -145,7 +145,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
return nil
}
func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (*browseTemplateContext, error) {
func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, fileSystem fs.FS, dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (*browseTemplateContext, error) {
files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
if err != nil && err != io.EOF {
return nil, err
@ -154,7 +154,7 @@ func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, dir fs.ReadDi
// user can presumably browse "up" to parent folder if path is longer than "/"
canGoUp := len(urlPath) > 1
return fsrv.directoryListing(ctx, files, canGoUp, root, urlPath, repl), nil
return fsrv.directoryListing(ctx, fileSystem, files, canGoUp, root, urlPath, repl), nil
}
// browseApplyQueryParams applies query parameters to the listing.
@ -223,12 +223,12 @@ func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.T
// isSymlinkTargetDir returns true if f's symbolic link target
// is a directory.
func (fsrv *FileServer) isSymlinkTargetDir(f fs.FileInfo, root, urlPath string) bool {
func (fsrv *FileServer) isSymlinkTargetDir(fileSystem fs.FS, f fs.FileInfo, root, urlPath string) bool {
if !isSymlink(f) {
return false
}
target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
targetInfo, err := fs.Stat(fsrv.fileSystem, target)
targetInfo, err := fs.Stat(fileSystem, target)
if err != nil {
return false
}

6
modules/caddyhttp/fileserver/browsetplcontext.go
View File

@ -32,7 +32,7 @@ import (
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)
func (fsrv *FileServer) directoryListing(ctx context.Context, entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) *browseTemplateContext {
func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) *browseTemplateContext {
filesToHide := fsrv.transformHidePaths(repl)
name, _ := url.PathUnescape(urlPath)
@ -62,7 +62,7 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, entries []fs.DirEn
continue
}
isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(info, root, urlPath)
isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(fileSystem, info, root, urlPath)
// add the slash after the escape of path to avoid escaping the slash as well
if isDir {
@ -76,7 +76,7 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, entries []fs.DirEn
fileIsSymlink := isSymlink(info)
if fileIsSymlink {
path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
fileInfo, err := fs.Stat(fsrv.fileSystem, path)
fileInfo, err := fs.Stat(fileSystem, path)
if err == nil {
size = fileInfo.Size()
}

21
modules/caddyhttp/fileserver/caddyfile.go
View File

@ -15,13 +15,11 @@
package fileserver
import (
"io/fs"
"path/filepath"
"strings"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
@ -37,7 +35,7 @@ func init() {
// server and configures it with this syntax:
//
// file_server [<matcher>] [browse] {
// fs <backend...>
// fs <filesystem>
// root <path>
// hide <files...>
// index <files...>
@ -68,21 +66,10 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
if !h.NextArg() {
return nil, h.ArgErr()
}
if fsrv.FileSystemRaw != nil {
return nil, h.Err("file system module already specified")
if fsrv.FileSystem != "" {
return nil, h.Err("file system already specified")
}
name := h.Val()
modID := "caddy.fs." + name
unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
if err != nil {
return nil, err
}
fsys, ok := unm.(fs.FS)
if !ok {
return nil, h.Errf("module %s (%T) is not a supported file system implementation (requires fs.FS)", modID, unm)
}
fsrv.FileSystemRaw = caddyconfig.JSONModuleObject(fsys, "backend", name, nil)
fsrv.FileSystem = h.Val()
case "hide":
fsrv.Hide = h.RemainingArgs()
if len(fsrv.Hide) == 0 {

49
modules/caddyhttp/fileserver/matcher.go
View File

@ -15,7 +15,6 @@
package fileserver
import (
"encoding/json"
"fmt"
"io/fs"
"net/http"
@ -64,8 +63,7 @@ func init() {
type MatchFile struct {
// The file system implementation to use. By default, the
// local disk file system will be used.
FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
fileSystem fs.FS
FileSystem string `json:"fs,omitempty"`
// The root directory, used for creating absolute
// file paths, and required when working with
@ -108,6 +106,8 @@ type MatchFile struct {
// component in order to be used as a split delimiter.
SplitPath []string `json:"split_path,omitempty"`
fsmap caddy.FileSystems
logger *zap.Logger
}
@ -181,6 +181,11 @@ func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
root = values["root"][0]
}
var fsName string
if len(values["fs"]) > 0 {
fsName = values["fs"][0]
}
var try_policy string
if len(values["try_policy"]) > 0 {
root = values["try_policy"][0]
@ -191,6 +196,7 @@ func (MatchFile) CELLibrary(ctx caddy.Context) (cel.Library, error) {
TryFiles: values["try_files"],
TryPolicy: try_policy,
SplitPath: values["split_path"],
FileSystem: fsName,
}
err = m.Provision(ctx)
@ -264,22 +270,16 @@ func celFileMatcherMacroExpander() parser.MacroExpander {
func (m *MatchFile) Provision(ctx caddy.Context) error {
m.logger = ctx.Logger()
// establish the file system to use
if len(m.FileSystemRaw) > 0 {
mod, err := ctx.LoadModule(m, "FileSystemRaw")
if err != nil {
return fmt.Errorf("loading file system module: %v", err)
}
m.fileSystem = mod.(fs.FS)
}
if m.fileSystem == nil {
m.fileSystem = osFS{}
}
m.fsmap = ctx.Filesystems()
if m.Root == "" {
m.Root = "{http.vars.root}"
}
if m.FileSystem == "" {
m.FileSystem = "{http.vars.fs}"
}
// if list of files to try was omitted entirely, assume URL path
// (use placeholder instead of r.URL.Path; see issue #4146)
if m.TryFiles == nil {
@ -320,6 +320,13 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
root := filepath.Clean(repl.ReplaceAll(m.Root, "."))
fsName := repl.ReplaceAll(m.FileSystem, "")
fileSystem, ok := m.fsmap.Get(fsName)
if !ok {
m.logger.Error("use of unregistered filesystem", zap.String("fs", fsName))
return false
}
type matchCandidate struct {
fullpath, relative, splitRemainder string
}
@ -368,7 +375,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
if runtime.GOOS == "windows" {
globResults = []string{fullPattern} // precious Windows
} else {
globResults, err = fs.Glob(m.fileSystem, fullPattern)
globResults, err = fs.Glob(fileSystem, fullPattern)
if err != nil {
m.logger.Error("expanding glob", zap.Error(err))
}
@ -410,7 +417,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
}
candidates := makeCandidates(pattern)
for _, c := range candidates {
if info, exists := m.strictFileExists(c.fullpath); exists {
if info, exists := m.strictFileExists(fileSystem, c.fullpath); exists {
setPlaceholders(c, info)
return true
}
@ -424,7 +431,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
for _, pattern := range m.TryFiles {
candidates := makeCandidates(pattern)
for _, c := range candidates {
info, err := fs.Stat(m.fileSystem, c.fullpath)
info, err := fs.Stat(fileSystem, c.fullpath)
if err == nil && info.Size() > largestSize {
largestSize = info.Size()
largest = c
@ -445,7 +452,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
for _, pattern := range m.TryFiles {
candidates := makeCandidates(pattern)
for _, c := range candidates {
info, err := fs.Stat(m.fileSystem, c.fullpath)
info, err := fs.Stat(fileSystem, c.fullpath)
if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
smallestSize = info.Size()
smallest = c
@ -465,7 +472,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
for _, pattern := range m.TryFiles {
candidates := makeCandidates(pattern)
for _, c := range candidates {
info, err := fs.Stat(m.fileSystem, c.fullpath)
info, err := fs.Stat(fileSystem, c.fullpath)
if err == nil &&
(recentInfo == nil || info.ModTime().After(recentInfo.ModTime())) {
recent = c
@ -503,8 +510,8 @@ func parseErrorCode(input string) error {
// the file must also be a directory; if it does
// NOT end in a forward slash, the file must NOT
// be a directory.
func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
info, err := fs.Stat(m.fileSystem, file)
func (m MatchFile) strictFileExists(fileSystem fs.FS, file string) (os.FileInfo, bool) {
info, err := fs.Stat(fileSystem, file)
if err != nil {
// in reality, this can be any error
// such as permission or even obscure

18
modules/caddyhttp/fileserver/matcher_test.go
View File

@ -24,6 +24,7 @@ import (
"testing"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/internal/filesystems"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
)
@ -116,7 +117,7 @@ func TestFileMatcher(t *testing.T) {
},
} {
m := &MatchFile{
fileSystem: osFS{},
fsmap: &filesystems.FilesystemMap{},
Root: "./testdata",
TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
}
@ -225,7 +226,7 @@ func TestPHPFileMatcher(t *testing.T) {
},
} {
m := &MatchFile{
fileSystem: osFS{},
fsmap: &filesystems.FilesystemMap{},
Root: "./testdata",
TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
SplitPath: []string{".php"},
@ -264,7 +265,10 @@ func TestPHPFileMatcher(t *testing.T) {
}
func TestFirstSplit(t *testing.T) {
m := MatchFile{SplitPath: []string{".php"}}
m := MatchFile{
SplitPath: []string{".php"},
fsmap: &filesystems.FilesystemMap{},
}
actual, remainder := m.firstSplit("index.PHP/somewhere")
expected := "index.PHP"
expectedRemainder := "/somewhere"
@ -276,8 +280,7 @@ func TestFirstSplit(t *testing.T) {
}
}
var (
expressionTests = []struct {
var expressionTests = []struct {
name string
expression *caddyhttp.MatchExpression
urlTarget string
@ -286,7 +289,7 @@ var (
wantErr bool
wantResult bool
clientCertificate []byte
}{
}{
{
name: "file error no args (MatchFile)",
expression: &caddyhttp.MatchExpression{
@ -351,8 +354,7 @@ var (
urlTarget: "https://example.com/nopenope.txt",
wantResult: false,
},
}
)
}
func TestMatchExpressionMatch(t *testing.T) {
for _, tst := range expressionTests {

79
modules/caddyhttp/fileserver/staticfiles.go
View File

@ -15,7 +15,6 @@
package fileserver
import (
"encoding/json"
"errors"
"fmt"
"io"
@ -97,15 +96,8 @@ type FileServer struct {
// The file system implementation to use. By default, Caddy uses the local
// disk file system.
//
// File system modules used here must adhere to the following requirements:
// - Implement fs.FS interface.
// - Support seeking on opened files; i.e.returned fs.File values must
// implement the io.Seeker interface. This is required for determining
// Content-Length and satisfying Range requests.
// - fs.File values that represent directories must implement the
// fs.ReadDirFile interface so that directory listings can be procured.
FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
fileSystem fs.FS
// if a non default filesystem is used, it must be first be registered in the globals section.
FileSystem string `json:"fs,omitempty"`
// The path to the root of the site. Default is `{http.vars.root}` if set,
// or current working directory otherwise. This should be a trusted value.
@ -169,6 +161,8 @@ type FileServer struct {
PrecompressedOrder []string `json:"precompressed_order,omitempty"`
precompressors map[string]encode.Precompressed
fsmap caddy.FileSystems
logger *zap.Logger
}
@ -184,16 +178,10 @@ func (FileServer) CaddyModule() caddy.ModuleInfo {
func (fsrv *FileServer) Provision(ctx caddy.Context) error {
fsrv.logger = ctx.Logger()
// establish which file system (possibly a virtual one) we'll be using
if len(fsrv.FileSystemRaw) > 0 {
mod, err := ctx.LoadModule(fsrv, "FileSystemRaw")
if err != nil {
return fmt.Errorf("loading file system module: %v", err)
}
fsrv.fileSystem = mod.(fs.FS)
}
if fsrv.fileSystem == nil {
fsrv.fileSystem = osFS{}
fsrv.fsmap = ctx.Filesystems()
if fsrv.FileSystem == "" {
fsrv.FileSystem = "{http.vars.fs}"
}
if fsrv.Root == "" {
@ -263,19 +251,26 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
filesToHide := fsrv.transformHidePaths(repl)
root := repl.ReplaceAll(fsrv.Root, ".")
fsName := repl.ReplaceAll(fsrv.FileSystem, "")
fileSystem, ok := fsrv.fsmap.Get(fsName)
if !ok {
return caddyhttp.Error(http.StatusNotFound, fmt.Errorf("filesystem not found"))
}
// remove any trailing `/` as it breaks fs.ValidPath() in the stdlib
filename := strings.TrimSuffix(caddyhttp.SanitizedPathJoin(root, r.URL.Path), "/")
fsrv.logger.Debug("sanitized path join",
zap.String("site_root", root),
zap.String("fs", fsName),
zap.String("request_path", r.URL.Path),
zap.String("result", filename))
// get information about the file
info, err := fs.Stat(fsrv.fileSystem, filename)
info, err := fs.Stat(fileSystem, filename)
if err != nil {
err = fsrv.mapDirOpenError(err, filename)
err = fsrv.mapDirOpenError(fileSystem, err, filename)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) {
return fsrv.notFound(w, r, next)
} else if errors.Is(err, fs.ErrPermission) {
@ -299,7 +294,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
continue
}
indexInfo, err := fs.Stat(fsrv.fileSystem, indexPath)
indexInfo, err := fs.Stat(fileSystem, indexPath)
if err != nil {
continue
}
@ -327,7 +322,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
zap.String("path", filename),
zap.Strings("index_filenames", fsrv.IndexNames))
if fsrv.Browse != nil && !fileHidden(filename, filesToHide) {
return fsrv.serveBrowse(root, filename, w, r, next)
return fsrv.serveBrowse(fileSystem, root, filename, w, r, next)
}
return fsrv.notFound(w, r, next)
}
@ -381,13 +376,13 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
continue
}
compressedFilename := filename + precompress.Suffix()
compressedInfo, err := fs.Stat(fsrv.fileSystem, compressedFilename)
compressedInfo, err := fs.Stat(fileSystem, compressedFilename)
if err != nil || compressedInfo.IsDir() {
fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
continue
}
fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err))
file, err = fsrv.openFile(compressedFilename, w)
file, err = fsrv.openFile(fileSystem, compressedFilename, w)
if err != nil {
fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err))
if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
@ -416,7 +411,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
fsrv.logger.Debug("opening file", zap.String("filename", filename))
// open the file
file, err = fsrv.openFile(filename, w)
file, err = fsrv.openFile(fileSystem, filename, w)
if err != nil {
if herr, ok := err.(caddyhttp.HandlerError); ok &&
herr.StatusCode == http.StatusNotFound {
@ -502,10 +497,10 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
// the response is configured to inform the client how to best handle it
// and a well-described handler error is returned (do not wrap the
// returned error value).
func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (fs.File, error) {
file, err := fsrv.fileSystem.Open(filename)
func (fsrv *FileServer) openFile(fileSystem fs.FS, filename string, w http.ResponseWriter) (fs.File, error) {
file, err := fileSystem.Open(filename)
if err != nil {
err = fsrv.mapDirOpenError(err, filename)
err = fsrv.mapDirOpenError(fileSystem, err, filename)
if errors.Is(err, fs.ErrNotExist) {
fsrv.logger.Debug("file not found", zap.String("filename", filename), zap.Error(err))
return nil, caddyhttp.Error(http.StatusNotFound, err)
@ -530,7 +525,7 @@ func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (fs.Fil
// Adapted from the Go standard library; originally written by Nathaniel Caza.
// https://go-review.googlesource.com/c/go/+/36635/
// https://go-review.googlesource.com/c/go/+/36804/
func (fsrv *FileServer) mapDirOpenError(originalErr error, name string) error {
func (fsrv *FileServer) mapDirOpenError(fileSystem fs.FS, originalErr error, name string) error {
if errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) {
return originalErr
}
@ -540,7 +535,7 @@ func (fsrv *FileServer) mapDirOpenError(originalErr error, name string) error {
if parts[i] == "" {
continue
}
fi, err := fs.Stat(fsrv.fileSystem, strings.Join(parts[:i+1], separator))
fi, err := fs.Stat(fileSystem, strings.Join(parts[:i+1], separator))
if err != nil {
return originalErr
}
@ -673,21 +668,6 @@ func (wr statusOverrideResponseWriter) Unwrap() http.ResponseWriter {
return wr.ResponseWriter
}
// osFS is a simple fs.FS implementation that uses the local
// file system. (We do not use os.DirFS because we do our own
// rooting or path prefixing without being constrained to a single
// root folder. The standard os.DirFS implementation is problematic
// since roots can be dynamic in our application.)
//
// osFS also implements fs.StatFS, fs.GlobFS, fs.ReadDirFS, and fs.ReadFileFS.
type osFS struct{}
func (osFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }
func (osFS) Glob(pattern string) ([]string, error) { return filepath.Glob(pattern) }
func (osFS) ReadDir(name string) ([]fs.DirEntry, error) { return os.ReadDir(name) }
func (osFS) ReadFile(name string) ([]byte, error) { return os.ReadFile(name) }
var defaultIndexNames = []string{"index.html", "index.txt"}
const (
@ -699,9 +679,4 @@ const (
var (
_ caddy.Provisioner = (*FileServer)(nil)
_ caddyhttp.MiddlewareHandler = (*FileServer)(nil)
_ fs.StatFS = (*osFS)(nil)
_ fs.GlobFS = (*osFS)(nil)
_ fs.ReadDirFS = (*osFS)(nil)
_ fs.ReadFileFS = (*osFS)(nil)
)

2
modules/caddyhttp/matchers_test.go
View File

@ -862,7 +862,6 @@ func TestHeaderREMatcher(t *testing.T) {
}
func BenchmarkHeaderREMatcher(b *testing.B) {
i := 0
match := MatchHeaderRE{"Field": &MatchRegexp{Pattern: "^foo(.*)$", Name: "name"}}
input := http.Header{"Field": []string{"foobar"}}
@ -1086,6 +1085,7 @@ func TestNotMatcher(t *testing.T) {
}
}
}
func BenchmarkLargeHostMatcher(b *testing.B) {
// this benchmark simulates a large host matcher (thousands of entries) where each
// value is an exact hostname (not a placeholder or wildcard) - compare the results

4
modules/caddyhttp/reverseproxy/ascii_test.go
View File

@ -26,7 +26,7 @@ package reverseproxy
import "testing"
func TestEqualFold(t *testing.T) {
var tests = []struct {
tests := []struct {
name string
a, b string
want bool
@ -64,7 +64,7 @@ func TestEqualFold(t *testing.T) {
}
func TestIsPrint(t *testing.T) {
var tests = []struct {
tests := []struct {
name string
in string
want bool

12
modules/caddyhttp/reverseproxy/fastcgi/client_test.go
View File

@ -48,7 +48,7 @@ import (
// and output "FAILED" in response
const (
scriptFile = "/tank/www/fcgic_test.php"
//ipPort = "remote-php-serv:59000"
// ipPort = "remote-php-serv:59000"
ipPort = "127.0.0.1:59000"
)
@ -57,7 +57,6 @@ var globalt *testing.T
type FastCGIServer struct{}
func (s FastCGIServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if err := req.ParseMultipartForm(100000000); err != nil {
log.Printf("[ERROR] failed to parse: %v", err)
}
@ -84,7 +83,7 @@ func (s FastCGIServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if req.MultipartForm != nil {
fileNum = len(req.MultipartForm.File)
for kn, fns := range req.MultipartForm.File {
//fmt.Fprintln(resp, "server:filekey ", kn )
// fmt.Fprintln(resp, "server:filekey ", kn )
length += len(kn)
for _, f := range fns {
fd, err := f.Open()
@ -101,13 +100,13 @@ func (s FastCGIServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
length += int(l0)
defer fd.Close()
md5 := fmt.Sprintf("%x", h.Sum(nil))
//fmt.Fprintln(resp, "server:filemd5 ", md5 )
// fmt.Fprintln(resp, "server:filemd5 ", md5 )
if kn != md5 {
fmt.Fprintln(resp, "server:err ", md5, kn)
stat = "FAILED"
}
//fmt.Fprintln(resp, "server:filename ", f.Filename )
// fmt.Fprintln(resp, "server:filename ", f.Filename )
}
}
}
@ -181,7 +180,6 @@ func sendFcgi(reqType int, fcgiParams map[string]string, data []byte, posts map[
}
func generateRandFile(size int) (p string, m string) {
p = filepath.Join(os.TempDir(), "fcgict"+strconv.Itoa(rand.Int()))
// open output file
@ -236,7 +234,7 @@ func DisabledTest(t *testing.T) {
fcgiParams := make(map[string]string)
fcgiParams["REQUEST_METHOD"] = "GET"
fcgiParams["SERVER_PROTOCOL"] = "HTTP/1.1"
//fcgi_params["GATEWAY_INTERFACE"] = "CGI/1.1"
// fcgi_params["GATEWAY_INTERFACE"] = "CGI/1.1"
fcgiParams["SCRIPT_FILENAME"] = scriptFile
// simple GET

1
modules/caddyhttp/reverseproxy/selectionpolicies_test.go
View File

@ -629,7 +629,6 @@ func TestRandomChoicePolicy(t *testing.T) {
if h == pool[0] {
t.Error("RandomChoicePolicy should not choose pool[0]")
}
}
func TestCookieHashPolicy(t *testing.T) {

1
modules/caddyhttp/tracing/tracerprovider_test.go
View File

@ -28,7 +28,6 @@ func Test_tracersProvider_cleanupTracerProvider(t *testing.T) {
tp.getTracerProvider()
err := tp.cleanupTracerProvider(zap.NewNop())
if err != nil {
t.Errorf("There should be no error: %v", err)
}

1
modules/standard/imports.go
View File

@ -5,6 +5,7 @@ import (
_ "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
_ "github.com/caddyserver/caddy/v2/modules/caddyevents"
_ "github.com/caddyserver/caddy/v2/modules/caddyevents/eventsconfig"
_ "github.com/caddyserver/caddy/v2/modules/caddyfs"
_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/standard"
_ "github.com/caddyserver/caddy/v2/modules/caddypki"
_ "github.com/caddyserver/caddy/v2/modules/caddypki/acmeserver"