diff --git a/caddytest/integration/caddyfile_adapt/file_server_sort.caddyfiletest b/caddytest/integration/caddyfile_adapt/file_server_sort.caddyfiletest
index 62bfd0cbab5..7f07cba8051 100644
--- a/caddytest/integration/caddyfile_adapt/file_server_sort.caddyfiletest
+++ b/caddytest/integration/caddyfile_adapt/file_server_sort.caddyfiletest
@@ -1,7 +1,9 @@
 :80
 
-file_server browse {
-	sort size desc
+file_server {
+	browse {
+		sort size desc
+	}
 }
 ----------
 {
@@ -16,14 +18,15 @@ file_server browse {
 						{
 							"handle": [
 								{
-									"browse": {},
+									"browse": {
+										"sort": [
+											"size",
+											"desc"
+										]
+									},
 									"handler": "file_server",
 									"hide": [
 										"./Caddyfile"
-									],
-									"sort": [
-										"size",
-										"desc"
 									]
 								}
 							]
diff --git a/modules/caddyhttp/fileserver/browse.go b/modules/caddyhttp/fileserver/browse.go
index bd02c584fec..f5cc16b42ab 100644
--- a/modules/caddyhttp/fileserver/browse.go
+++ b/modules/caddyhttp/fileserver/browse.go
@@ -52,8 +52,19 @@ var BrowseTemplate string
 type Browse struct {
 	// Filename of the template to use instead of the embedded browse template.
 	TemplateFile string `json:"template_file,omitempty"`
+
 	// Determines whether or not targets of symlinks should be revealed.
 	RevealSymlinks bool `json:"reveal_symlinks,omitempty"`
+
+	// Override the default sort.
+	// It includes the following options:
+	//   - sort_by: name(default), namedirfirst, size, time
+	//   - order: asc(default), desc
+	// eg.:
+	//   - `sort time desc` will sort by time in descending order
+	//   - `sort size` will sort by size in ascending order
+	// The first option must be `sort_by` and the second option must be `order` (if exists).
+	SortOptions []string `json:"sort,omitempty"`
 }
 
 func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
@@ -210,7 +221,7 @@ func (fsrv *FileServer) browseApplyQueryParams(w http.ResponseWriter, r *http.Re
 
 	// The configs in Caddyfile have lower priority than Query params,
 	// so put it at first.
-	for idx, item := range fsrv.SortOptions {
+	for idx, item := range fsrv.Browse.SortOptions {
 		// Only `sort` & `order`, 2 params are allowed
 		if idx >= 2 {
 			break
diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go
index 603a828c968..71b7638e477 100644
--- a/modules/caddyhttp/fileserver/caddyfile.go
+++ b/modules/caddyhttp/fileserver/caddyfile.go
@@ -119,6 +119,16 @@ func (fsrv *FileServer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 						return d.Err("Symlinks path reveal is already enabled")
 					}
 					fsrv.Browse.RevealSymlinks = true
+				case "sort":
+					for d.NextArg() {
+						dVal := d.Val()
+						switch dVal {
+						case sortByName, sortByNameDirFirst, sortBySize, sortByTime, sortOrderAsc, sortOrderDesc:
+							fsrv.Browse.SortOptions = append(fsrv.Browse.SortOptions, dVal)
+						default:
+							return d.Errf("unknown sort option '%s'", dVal)
+						}
+					}
 				default:
 					return d.Errf("unknown subdirective '%s'", d.Val())
 				}
@@ -171,17 +181,6 @@ func (fsrv *FileServer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			}
 			fsrv.EtagFileExtensions = etagFileExtensions
 
-		case "sort":
-			for d.NextArg() {
-				dVal := d.Val()
-				switch dVal {
-				case sortByName, sortBySize, sortByTime, sortOrderAsc, sortOrderDesc:
-					fsrv.SortOptions = append(fsrv.SortOptions, dVal)
-				default:
-					return d.Errf("unknown sort option '%s'", dVal)
-				}
-			}
-
 		default:
 			return d.Errf("unknown subdirective '%s'", d.Val())
 		}
diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go
index 48812cfec3d..2dd237ae7dc 100644
--- a/modules/caddyhttp/fileserver/staticfiles.go
+++ b/modules/caddyhttp/fileserver/staticfiles.go
@@ -153,16 +153,6 @@ type FileServer struct {
 	// a 404 error. By default, this is false (disabled).
 	PassThru bool `json:"pass_thru,omitempty"`
 
-	// Override the default sort.
-	// It includes the following options:
-	//   - sort_by: name(default), namedirfirst, size, time
-	//   - order: asc(default), desc
-	// eg.:
-	//   - `sort time desc` will sort by time in descending order
-	//   - `sort size` will sort by size in ascending order
-	// The first option must be `sort_by` and the second option must be `order` (if exists).
-	SortOptions []string `json:"sort,omitempty"`
-
 	// Selection of encoders to use to check for precompressed files.
 	PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"`
 
@@ -246,19 +236,21 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		fsrv.precompressors[ae] = p
 	}
 
-	// check sort options
-	for idx, sortOption := range fsrv.SortOptions {
-		switch idx {
-		case 0:
-			if sortOption != sortByName && sortOption != sortByNameDirFirst && sortOption != sortBySize && sortOption != sortByTime {
-				return fmt.Errorf("the first option must be one of the following: %s, %s, %s, %s, but got %s", sortByName, sortByNameDirFirst, sortBySize, sortByTime, sortOption)
-			}
-		case 1:
-			if sortOption != sortOrderAsc && sortOption != sortOrderDesc {
-				return fmt.Errorf("the second option must be one of the following: %s, %s, but got %s", sortOrderAsc, sortOrderDesc, sortOption)
+	if fsrv.Browse != nil {
+		// check sort options
+		for idx, sortOption := range fsrv.Browse.SortOptions {
+			switch idx {
+			case 0:
+				if sortOption != sortByName && sortOption != sortByNameDirFirst && sortOption != sortBySize && sortOption != sortByTime {
+					return fmt.Errorf("the first option must be one of the following: %s, %s, %s, %s, but got %s", sortByName, sortByNameDirFirst, sortBySize, sortByTime, sortOption)
+				}
+			case 1:
+				if sortOption != sortOrderAsc && sortOption != sortOrderDesc {
+					return fmt.Errorf("the second option must be one of the following: %s, %s, but got %s", sortOrderAsc, sortOrderDesc, sortOption)
+				}
+			default:
+				return fmt.Errorf("only max 2 sort options are allowed, but got %d", idx+1)
 			}
-		default:
-			return fmt.Errorf("only max 2 sort options are allowed, but got %d", idx+1)
 		}
 	}