From 6914063853a8a3f3cecfa4b21f223820c2a0b7df Mon Sep 17 00:00:00 2001 From: WeidiDeng Date: Thu, 7 Jan 2021 18:30:17 +0800 Subject: [PATCH] feat: allow disabling file detections by reading header (#1175) --- cmd/root.go | 4 ++ files/file.go | 105 +++++++++++++++++++++++-------------------- files/utils.go | 2 +- http/preview.go | 11 ++--- http/public.go | 11 ++--- http/raw.go | 11 ++--- http/resource.go | 22 ++++----- settings/settings.go | 23 +++++----- 8 files changed, 104 insertions(+), 85 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a8a484f0..75506f78 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -66,6 +66,7 @@ func addServerFlags(flags *pflag.FlagSet) { flags.Bool("disable-thumbnails", false, "disable image thumbnails") flags.Bool("disable-preview-resize", false, "disable resize of image previews") flags.Bool("disable-exec", false, "disables Command Runner feature") + flags.Bool("disable-type-detection-by-header", false, "disables type detection by reading file headers") } var rootCmd = &cobra.Command{ @@ -243,6 +244,9 @@ func getRunParams(flags *pflag.FlagSet, st *storage.Storage) *settings.Server { _, disablePreviewResize := getParamB(flags, "disable-preview-resize") server.ResizePreview = !disablePreviewResize + _, disableTypeDetectionByHeader := getParamB(flags, "disable-type-detection-by-header") + server.TypeDetectionByHeader = !disableTypeDetectionByHeader + _, disableExec := getParamB(flags, "disable-exec") server.EnableExec = !disableExec diff --git a/files/file.go b/files/file.go index d58b2a34..fa102049 100644 --- a/files/file.go +++ b/files/file.go @@ -42,11 +42,12 @@ type FileInfo struct { // FileOptions are the options when getting a file info. type FileOptions struct { - Fs afero.Fs - Path string - Modify bool - Expand bool - Checker rules.Checker + Fs afero.Fs + Path string + Modify bool + Expand bool + ReadHeader bool + Checker rules.Checker } // NewFileInfo creates a File object from a path and a given user. This File @@ -75,13 +76,13 @@ func NewFileInfo(opts FileOptions) (*FileInfo, error) { if opts.Expand { if file.IsDir { - if err := file.readListing(opts.Checker); err != nil { //nolint:shadow + if err := file.readListing(opts.Checker, opts.ReadHeader); err != nil { //nolint:shadow return nil, err } return file, nil } - err = file.detectType(opts.Modify, true) + err = file.detectType(opts.Modify, true, true) if err != nil { return nil, err } @@ -134,7 +135,7 @@ func (i *FileInfo) Checksum(algo string) error { //nolint:goconst //TODO: use constants -func (i *FileInfo) detectType(modify, saveContent bool) error { +func (i *FileInfo) detectType(modify, saveContent, readHeader bool) error { if IsNamedPipe(i.Mode) { i.Type = "blob" return nil @@ -143,6 +144,51 @@ func (i *FileInfo) detectType(modify, saveContent bool) error { // imagine the situation where a file in a dir with thousands // of files couldn't be opened: we'd have immediately // a 500 even though it doesn't matter. So we just log it. + + var buffer []byte + + mimetype := mime.TypeByExtension(i.Extension) + if mimetype == "" && readHeader { + buffer = i.readFirstBytes() + mimetype = http.DetectContentType(buffer) + } + + switch { + case strings.HasPrefix(mimetype, "video"): + i.Type = "video" + i.detectSubtitles() + return nil + case strings.HasPrefix(mimetype, "audio"): + i.Type = "audio" + return nil + case strings.HasPrefix(mimetype, "image"): + i.Type = "image" + return nil + case (strings.HasPrefix(mimetype, "text") || (len(buffer) > 0 && !isBinary(buffer))) && i.Size <= 10*1024*1024: // 10 MB + i.Type = "text" + + if !modify { + i.Type = "textImmutable" + } + + if saveContent { + afs := &afero.Afero{Fs: i.Fs} + content, err := afs.ReadFile(i.Path) + if err != nil { + return err + } + + i.Content = string(content) + } + return nil + default: + i.Type = "blob" + } + + return nil +} + +func (i *FileInfo) readFirstBytes() []byte { reader, err := i.Fs.Open(i.Path) if err != nil { log.Print(err) @@ -159,44 +205,7 @@ func (i *FileInfo) detectType(modify, saveContent bool) error { return nil } - mimetype := mime.TypeByExtension(i.Extension) - if mimetype == "" { - mimetype = http.DetectContentType(buffer[:n]) - } - - switch { - case strings.HasPrefix(mimetype, "video"): - i.Type = "video" - i.detectSubtitles() - return nil - case strings.HasPrefix(mimetype, "audio"): - i.Type = "audio" - return nil - case strings.HasPrefix(mimetype, "image"): - i.Type = "image" - return nil - case isBinary(buffer[:n], n) || i.Size > 10*1024*1024: // 10 MB - i.Type = "blob" - return nil - default: - i.Type = "text" - - if !modify { - i.Type = "textImmutable" - } - - if saveContent { - afs := &afero.Afero{Fs: i.Fs} - content, err := afs.ReadFile(i.Path) - if err != nil { - return err - } - - i.Content = string(content) - } - } - - return nil + return buffer[:n] } func (i *FileInfo) detectSubtitles() { @@ -215,7 +224,7 @@ func (i *FileInfo) detectSubtitles() { } } -func (i *FileInfo) readListing(checker rules.Checker) error { +func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { afs := &afero.Afero{Fs: i.Fs} dir, err := afs.ReadDir(i.Path) if err != nil { @@ -261,7 +270,7 @@ func (i *FileInfo) readListing(checker rules.Checker) error { } else { listing.NumFiles++ - err := file.detectType(true, false) + err := file.detectType(true, false, readHeader) if err != nil { return err } diff --git a/files/utils.go b/files/utils.go index bcaa13f1..f4b0365d 100644 --- a/files/utils.go +++ b/files/utils.go @@ -5,7 +5,7 @@ import ( "unicode/utf8" ) -func isBinary(content []byte, _ int) bool { +func isBinary(content []byte) bool { maybeStr := string(content) runeCnt := utf8.RuneCount(content) runeIndex := 0 diff --git a/http/preview.go b/http/preview.go index b6f8c256..0d956a51 100644 --- a/http/preview.go +++ b/http/preview.go @@ -46,11 +46,12 @@ func previewHandler(imgSvc ImgService, fileCache FileCache, enableThumbnails, re } file, err := files.NewFileInfo(files.FileOptions{ - Fs: d.user.Fs, - Path: "/" + vars["path"], - Modify: d.user.Perm.Modify, - Expand: true, - Checker: d, + Fs: d.user.Fs, + Path: "/" + vars["path"], + Modify: d.user.Perm.Modify, + Expand: true, + ReadHeader: d.server.TypeDetectionByHeader, + Checker: d, }) if err != nil { return errToStatus(err), err diff --git a/http/public.go b/http/public.go index 0ab5d952..d34b97e3 100644 --- a/http/public.go +++ b/http/public.go @@ -27,11 +27,12 @@ var withHashFile = func(fn handleFunc) handleFunc { d.user = user file, err := files.NewFileInfo(files.FileOptions{ - Fs: d.user.Fs, - Path: link.Path, - Modify: d.user.Perm.Modify, - Expand: true, - Checker: d, + Fs: d.user.Fs, + Path: link.Path, + Modify: d.user.Perm.Modify, + Expand: true, + ReadHeader: d.server.TypeDetectionByHeader, + Checker: d, }) if err != nil { return errToStatus(err), err diff --git a/http/raw.go b/http/raw.go index 1f3c19ef..c65e1d61 100644 --- a/http/raw.go +++ b/http/raw.go @@ -84,11 +84,12 @@ var rawHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) } file, err := files.NewFileInfo(files.FileOptions{ - Fs: d.user.Fs, - Path: r.URL.Path, - Modify: d.user.Perm.Modify, - Expand: false, - Checker: d, + Fs: d.user.Fs, + Path: r.URL.Path, + Modify: d.user.Perm.Modify, + Expand: false, + ReadHeader: d.server.TypeDetectionByHeader, + Checker: d, }) if err != nil { return errToStatus(err), err diff --git a/http/resource.go b/http/resource.go index 08e885f8..9d036dca 100644 --- a/http/resource.go +++ b/http/resource.go @@ -20,11 +20,12 @@ import ( var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { file, err := files.NewFileInfo(files.FileOptions{ - Fs: d.user.Fs, - Path: r.URL.Path, - Modify: d.user.Perm.Modify, - Expand: true, - Checker: d, + Fs: d.user.Fs, + Path: r.URL.Path, + Modify: d.user.Perm.Modify, + Expand: true, + ReadHeader: d.server.TypeDetectionByHeader, + Checker: d, }) if err != nil { return errToStatus(err), err @@ -58,11 +59,12 @@ func resourceDeleteHandler(fileCache FileCache) handleFunc { } file, err := files.NewFileInfo(files.FileOptions{ - Fs: d.user.Fs, - Path: r.URL.Path, - Modify: d.user.Perm.Modify, - Expand: true, - Checker: d, + Fs: d.user.Fs, + Path: r.URL.Path, + Modify: d.user.Perm.Modify, + Expand: true, + ReadHeader: d.server.TypeDetectionByHeader, + Checker: d, }) if err != nil { return errToStatus(err), err diff --git a/settings/settings.go b/settings/settings.go index 6bf4c4db..0f8616d2 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -30,17 +30,18 @@ func (s *Settings) GetRules() []rules.Rule { // Server specific settings. type Server struct { - Root string `json:"root"` - BaseURL string `json:"baseURL"` - Socket string `json:"socket"` - TLSKey string `json:"tlsKey"` - TLSCert string `json:"tlsCert"` - Port string `json:"port"` - Address string `json:"address"` - Log string `json:"log"` - EnableThumbnails bool `json:"enableThumbnails"` - ResizePreview bool `json:"resizePreview"` - EnableExec bool `json:"enableExec"` + Root string `json:"root"` + BaseURL string `json:"baseURL"` + Socket string `json:"socket"` + TLSKey string `json:"tlsKey"` + TLSCert string `json:"tlsCert"` + Port string `json:"port"` + Address string `json:"address"` + Log string `json:"log"` + EnableThumbnails bool `json:"enableThumbnails"` + ResizePreview bool `json:"resizePreview"` + EnableExec bool `json:"enableExec"` + TypeDetectionByHeader bool `json:"typeDetectionByHeader"` } // Clean cleans any variables that might need cleaning.