feat: allow disabling file detections by reading header (#1175)

This commit is contained in:
WeidiDeng 2021-01-07 18:30:17 +08:00 committed by GitHub
parent 43e0d4a856
commit 6914063853
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 104 additions and 85 deletions

View File

@ -66,6 +66,7 @@ func addServerFlags(flags *pflag.FlagSet) {
flags.Bool("disable-thumbnails", false, "disable image thumbnails") flags.Bool("disable-thumbnails", false, "disable image thumbnails")
flags.Bool("disable-preview-resize", false, "disable resize of image previews") flags.Bool("disable-preview-resize", false, "disable resize of image previews")
flags.Bool("disable-exec", false, "disables Command Runner feature") flags.Bool("disable-exec", false, "disables Command Runner feature")
flags.Bool("disable-type-detection-by-header", false, "disables type detection by reading file headers")
} }
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
@ -243,6 +244,9 @@ func getRunParams(flags *pflag.FlagSet, st *storage.Storage) *settings.Server {
_, disablePreviewResize := getParamB(flags, "disable-preview-resize") _, disablePreviewResize := getParamB(flags, "disable-preview-resize")
server.ResizePreview = !disablePreviewResize server.ResizePreview = !disablePreviewResize
_, disableTypeDetectionByHeader := getParamB(flags, "disable-type-detection-by-header")
server.TypeDetectionByHeader = !disableTypeDetectionByHeader
_, disableExec := getParamB(flags, "disable-exec") _, disableExec := getParamB(flags, "disable-exec")
server.EnableExec = !disableExec server.EnableExec = !disableExec

View File

@ -42,11 +42,12 @@ type FileInfo struct {
// FileOptions are the options when getting a file info. // FileOptions are the options when getting a file info.
type FileOptions struct { type FileOptions struct {
Fs afero.Fs Fs afero.Fs
Path string Path string
Modify bool Modify bool
Expand bool Expand bool
Checker rules.Checker ReadHeader bool
Checker rules.Checker
} }
// NewFileInfo creates a File object from a path and a given user. This File // NewFileInfo creates a File object from a path and a given user. This File
@ -75,13 +76,13 @@ func NewFileInfo(opts FileOptions) (*FileInfo, error) {
if opts.Expand { if opts.Expand {
if file.IsDir { if file.IsDir {
if err := file.readListing(opts.Checker); err != nil { //nolint:shadow if err := file.readListing(opts.Checker, opts.ReadHeader); err != nil { //nolint:shadow
return nil, err return nil, err
} }
return file, nil return file, nil
} }
err = file.detectType(opts.Modify, true) err = file.detectType(opts.Modify, true, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -134,7 +135,7 @@ func (i *FileInfo) Checksum(algo string) error {
//nolint:goconst //nolint:goconst
//TODO: use constants //TODO: use constants
func (i *FileInfo) detectType(modify, saveContent bool) error { func (i *FileInfo) detectType(modify, saveContent, readHeader bool) error {
if IsNamedPipe(i.Mode) { if IsNamedPipe(i.Mode) {
i.Type = "blob" i.Type = "blob"
return nil return nil
@ -143,6 +144,51 @@ func (i *FileInfo) detectType(modify, saveContent bool) error {
// imagine the situation where a file in a dir with thousands // imagine the situation where a file in a dir with thousands
// of files couldn't be opened: we'd have immediately // of files couldn't be opened: we'd have immediately
// a 500 even though it doesn't matter. So we just log it. // a 500 even though it doesn't matter. So we just log it.
var buffer []byte
mimetype := mime.TypeByExtension(i.Extension)
if mimetype == "" && readHeader {
buffer = i.readFirstBytes()
mimetype = http.DetectContentType(buffer)
}
switch {
case strings.HasPrefix(mimetype, "video"):
i.Type = "video"
i.detectSubtitles()
return nil
case strings.HasPrefix(mimetype, "audio"):
i.Type = "audio"
return nil
case strings.HasPrefix(mimetype, "image"):
i.Type = "image"
return nil
case (strings.HasPrefix(mimetype, "text") || (len(buffer) > 0 && !isBinary(buffer))) && i.Size <= 10*1024*1024: // 10 MB
i.Type = "text"
if !modify {
i.Type = "textImmutable"
}
if saveContent {
afs := &afero.Afero{Fs: i.Fs}
content, err := afs.ReadFile(i.Path)
if err != nil {
return err
}
i.Content = string(content)
}
return nil
default:
i.Type = "blob"
}
return nil
}
func (i *FileInfo) readFirstBytes() []byte {
reader, err := i.Fs.Open(i.Path) reader, err := i.Fs.Open(i.Path)
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -159,44 +205,7 @@ func (i *FileInfo) detectType(modify, saveContent bool) error {
return nil return nil
} }
mimetype := mime.TypeByExtension(i.Extension) return buffer[:n]
if mimetype == "" {
mimetype = http.DetectContentType(buffer[:n])
}
switch {
case strings.HasPrefix(mimetype, "video"):
i.Type = "video"
i.detectSubtitles()
return nil
case strings.HasPrefix(mimetype, "audio"):
i.Type = "audio"
return nil
case strings.HasPrefix(mimetype, "image"):
i.Type = "image"
return nil
case isBinary(buffer[:n], n) || i.Size > 10*1024*1024: // 10 MB
i.Type = "blob"
return nil
default:
i.Type = "text"
if !modify {
i.Type = "textImmutable"
}
if saveContent {
afs := &afero.Afero{Fs: i.Fs}
content, err := afs.ReadFile(i.Path)
if err != nil {
return err
}
i.Content = string(content)
}
}
return nil
} }
func (i *FileInfo) detectSubtitles() { func (i *FileInfo) detectSubtitles() {
@ -215,7 +224,7 @@ func (i *FileInfo) detectSubtitles() {
} }
} }
func (i *FileInfo) readListing(checker rules.Checker) error { func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error {
afs := &afero.Afero{Fs: i.Fs} afs := &afero.Afero{Fs: i.Fs}
dir, err := afs.ReadDir(i.Path) dir, err := afs.ReadDir(i.Path)
if err != nil { if err != nil {
@ -261,7 +270,7 @@ func (i *FileInfo) readListing(checker rules.Checker) error {
} else { } else {
listing.NumFiles++ listing.NumFiles++
err := file.detectType(true, false) err := file.detectType(true, false, readHeader)
if err != nil { if err != nil {
return err return err
} }

View File

@ -5,7 +5,7 @@ import (
"unicode/utf8" "unicode/utf8"
) )
func isBinary(content []byte, _ int) bool { func isBinary(content []byte) bool {
maybeStr := string(content) maybeStr := string(content)
runeCnt := utf8.RuneCount(content) runeCnt := utf8.RuneCount(content)
runeIndex := 0 runeIndex := 0

View File

@ -46,11 +46,12 @@ func previewHandler(imgSvc ImgService, fileCache FileCache, enableThumbnails, re
} }
file, err := files.NewFileInfo(files.FileOptions{ file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs, Fs: d.user.Fs,
Path: "/" + vars["path"], Path: "/" + vars["path"],
Modify: d.user.Perm.Modify, Modify: d.user.Perm.Modify,
Expand: true, Expand: true,
Checker: d, ReadHeader: d.server.TypeDetectionByHeader,
Checker: d,
}) })
if err != nil { if err != nil {
return errToStatus(err), err return errToStatus(err), err

View File

@ -27,11 +27,12 @@ var withHashFile = func(fn handleFunc) handleFunc {
d.user = user d.user = user
file, err := files.NewFileInfo(files.FileOptions{ file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs, Fs: d.user.Fs,
Path: link.Path, Path: link.Path,
Modify: d.user.Perm.Modify, Modify: d.user.Perm.Modify,
Expand: true, Expand: true,
Checker: d, ReadHeader: d.server.TypeDetectionByHeader,
Checker: d,
}) })
if err != nil { if err != nil {
return errToStatus(err), err return errToStatus(err), err

View File

@ -84,11 +84,12 @@ var rawHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data)
} }
file, err := files.NewFileInfo(files.FileOptions{ file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs, Fs: d.user.Fs,
Path: r.URL.Path, Path: r.URL.Path,
Modify: d.user.Perm.Modify, Modify: d.user.Perm.Modify,
Expand: false, Expand: false,
Checker: d, ReadHeader: d.server.TypeDetectionByHeader,
Checker: d,
}) })
if err != nil { if err != nil {
return errToStatus(err), err return errToStatus(err), err

View File

@ -20,11 +20,12 @@ import (
var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) {
file, err := files.NewFileInfo(files.FileOptions{ file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs, Fs: d.user.Fs,
Path: r.URL.Path, Path: r.URL.Path,
Modify: d.user.Perm.Modify, Modify: d.user.Perm.Modify,
Expand: true, Expand: true,
Checker: d, ReadHeader: d.server.TypeDetectionByHeader,
Checker: d,
}) })
if err != nil { if err != nil {
return errToStatus(err), err return errToStatus(err), err
@ -58,11 +59,12 @@ func resourceDeleteHandler(fileCache FileCache) handleFunc {
} }
file, err := files.NewFileInfo(files.FileOptions{ file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs, Fs: d.user.Fs,
Path: r.URL.Path, Path: r.URL.Path,
Modify: d.user.Perm.Modify, Modify: d.user.Perm.Modify,
Expand: true, Expand: true,
Checker: d, ReadHeader: d.server.TypeDetectionByHeader,
Checker: d,
}) })
if err != nil { if err != nil {
return errToStatus(err), err return errToStatus(err), err

View File

@ -30,17 +30,18 @@ func (s *Settings) GetRules() []rules.Rule {
// Server specific settings. // Server specific settings.
type Server struct { type Server struct {
Root string `json:"root"` Root string `json:"root"`
BaseURL string `json:"baseURL"` BaseURL string `json:"baseURL"`
Socket string `json:"socket"` Socket string `json:"socket"`
TLSKey string `json:"tlsKey"` TLSKey string `json:"tlsKey"`
TLSCert string `json:"tlsCert"` TLSCert string `json:"tlsCert"`
Port string `json:"port"` Port string `json:"port"`
Address string `json:"address"` Address string `json:"address"`
Log string `json:"log"` Log string `json:"log"`
EnableThumbnails bool `json:"enableThumbnails"` EnableThumbnails bool `json:"enableThumbnails"`
ResizePreview bool `json:"resizePreview"` ResizePreview bool `json:"resizePreview"`
EnableExec bool `json:"enableExec"` EnableExec bool `json:"enableExec"`
TypeDetectionByHeader bool `json:"typeDetectionByHeader"`
} }
// Clean cleans any variables that might need cleaning. // Clean cleans any variables that might need cleaning.