Mirror
/
filebrowser
mirror of https://github.com/filebrowser/filebrowser.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: limit image resize workers

This commit is contained in:
Oleg Lobanov 2020-07-23 02:41:19 +02:00
parent 14e2f84ceb
commit 94ef59602f
No known key found for this signature in database
GPG Key ID: 7CC64E41212621B0
7 changed files with 510 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/root.go
View File

@ -13,6 +13,8 @@ import (
"strings"
"syscall"
"github.com/filebrowser/filebrowser/v2/img"
homedir "github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -56,6 +58,7 @@ func addServerFlags(flags *pflag.FlagSet) {
flags.StringP("root", "r", ".", "root to prepend to relative paths")
flags.String("socket", "", "socket to listen to (cannot be used with address, port, cert nor key flags)")
flags.StringP("baseurl", "b", "", "base url")
flags.Int("img-processors", 4, "image processors count")
}
var rootCmd = &cobra.Command{
@ -103,6 +106,14 @@ user created with the credentials from options "username" and "password".`,
quickSetup(cmd.Flags(), d)
}
// build img service
workersCount, err := cmd.Flags().GetInt("img-processors")
checkErr(err)
if workersCount < 1 {
log.Fatal("Image resize workers count could not be < 1")
}
imgSvc := img.New(workersCount)
server := getRunParams(cmd.Flags(), d.store)
setupLog(server.Log)
@ -132,7 +143,7 @@ user created with the credentials from options "username" and "password".`,
signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
go cleanupHandler(listener, sigc)
handler, err := fbhttp.NewHandler(d.store, server)
handler, err := fbhttp.NewHandler(imgSvc, d.store, server)
checkErr(err)
defer listener.Close()

1
go.mod
View File

@ -14,6 +14,7 @@ require (
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
github.com/maruel/natural v0.0.0-20180416170133-dbcb3e2e8cf1
github.com/marusama/semaphore/v2 v2.4.1
github.com/mholt/archiver v3.1.1+incompatible
github.com/mitchellh/go-homedir v1.1.0
github.com/nwaples/rardecode v1.0.0 // indirect

2
go.sum
View File

@ -127,6 +127,8 @@ github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNA
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/maruel/natural v0.0.0-20180416170133-dbcb3e2e8cf1 h1:PEhRT94KBTY4E0KdCYmhvDGWjSFBxc68j2M6PMRix8U=
github.com/maruel/natural v0.0.0-20180416170133-dbcb3e2e8cf1/go.mod h1:wI697HNhDFM/vBruYM3ckbszQ2+DOIeH9qdBKMdf288=
github.com/marusama/semaphore/v2 v2.4.1 h1:Y29DhhFMvreVgoqF9EtaSJAF9t2E7Sk7i5VW81sqB8I=
github.com/marusama/semaphore/v2 v2.4.1/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=

4
http/http.go
View File

@ -14,7 +14,7 @@ type modifyRequest struct {
Which []string `json:"which"` // Answer to: which fields?
}
func NewHandler(store *storage.Storage, server *settings.Server) (http.Handler, error) {
func NewHandler(imgSvc ImgService, store *storage.Storage, server *settings.Server) (http.Handler, error) {
server.Clean()
r := mux.NewRouter()
@ -59,7 +59,7 @@ func NewHandler(store *storage.Storage, server *settings.Server) (http.Handler,
api.Handle("/settings", monkey(settingsPutHandler, "")).Methods("PUT")
api.PathPrefix("/raw").Handler(monkey(rawHandler, "/api/raw")).Methods("GET")
api.PathPrefix("/preview/{size}/{path:.*}").Handler(monkey(previewHandler, "/api/preview")).Methods("GET")
api.PathPrefix("/preview/{size}/{path:.*}").Handler(monkey(previewHandler(imgSvc), "/api/preview")).Methods("GET")
api.PathPrefix("/command").Handler(monkey(commandsHandler, "/api/command")).Methods("GET")
api.PathPrefix("/search").Handler(monkey(searchHandler, "/api/search")).Methods("GET")

115
http/preview.go
View File

@ -1,14 +1,17 @@
package http
import (
"context"
"errors"
"fmt"
"image"
"io"
"net/http"
"github.com/disintegration/imaging"
"github.com/gorilla/mux"
"github.com/spf13/afero"
"github.com/filebrowser/filebrowser/v2/files"
"github.com/filebrowser/filebrowser/v2/img"
)
const (
@ -16,44 +19,49 @@ const (
sizeBig = "big"
)
type imageProcessor func(src image.Image) (image.Image, error)
type ImgService interface {
FormatFromExtension(ext string) (img.Format, error)
Resize(ctx context.Context, file afero.File, width, height int, out io.Writer, options ...img.Option) error
}
var previewHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) {
if !d.user.Perm.Download {
return http.StatusAccepted, nil
}
vars := mux.Vars(r)
size := vars["size"]
if size != sizeBig && size != sizeThumb {
return http.StatusNotImplemented, nil
}
func previewHandler(imgSvc ImgService) handleFunc {
return withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) {
if !d.user.Perm.Download {
return http.StatusAccepted, nil
}
vars := mux.Vars(r)
size := vars["size"]
if size != sizeBig && size != sizeThumb {
return http.StatusNotImplemented, nil
}
file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs,
Path: "/" + vars["path"],
Modify: d.user.Perm.Modify,
Expand: true,
Checker: d,
file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs,
Path: "/" + vars["path"],
Modify: d.user.Perm.Modify,
Expand: true,
Checker: d,
})
if err != nil {
return errToStatus(err), err
}
setContentDisposition(w, r, file)
switch file.Type {
case "image":
return handleImagePreview(imgSvc, w, r, file, size)
default:
return http.StatusNotImplemented, fmt.Errorf("can't create preview for %s type", file.Type)
}
})
if err != nil {
return errToStatus(err), err
}
}
setContentDisposition(w, r, file)
switch file.Type {
case "image":
return handleImagePreview(w, r, file, size)
default:
return http.StatusNotImplemented, fmt.Errorf("can't create preview for %s type", file.Type)
}
})
func handleImagePreview(w http.ResponseWriter, r *http.Request, file *files.FileInfo, size string) (int, error) {
format, err := imaging.FormatFromExtension(file.Extension)
func handleImagePreview(imgSvc ImgService, w http.ResponseWriter, r *http.Request, file *files.FileInfo, size string) (int, error) {
format, err := imgSvc.FormatFromExtension(file.Extension)
if err != nil {
// Unsupported extensions directly return the raw data
if err == imaging.ErrUnsupportedFormat {
if err == img.ErrUnsupportedFormat {
return rawFileHandler(w, r, file)
}
return errToStatus(err), err
@ -65,37 +73,38 @@ func handleImagePreview(w http.ResponseWriter, r *http.Request, file *files.File
}
defer fd.Close()
if format == imaging.GIF && size == sizeBig {
if _, err := rawFileHandler(w, r, file); err != nil { //nolint: govet
if format == img.FormatGif && size == sizeBig {
if _, err := rawFileHandler(w, r, file); err != nil {
return errToStatus(err), err
}
return 0, nil
}
var imgProcessor imageProcessor
var (
width int
height int
options []img.Option
)
switch size {
case sizeBig:
imgProcessor = func(img image.Image) (image.Image, error) {
return imaging.Fit(img, 1080, 1080, imaging.Lanczos), nil
}
width = 1080
height = 1080
options = append(options, img.WithHighPriority())
case sizeThumb:
imgProcessor = func(img image.Image) (image.Image, error) {
return imaging.Thumbnail(img, 128, 128, imaging.Box), nil
}
width = 128
height = 128
options = append(options, img.WithMode(img.ResizeModeFill), img.WithQuality(img.QualityLow))
default:
return http.StatusBadRequest, fmt.Errorf("unsupported preview size %s", size)
}
img, err := imaging.Decode(fd, imaging.AutoOrientation(true))
if err != nil {
return errToStatus(err), err
}
img, err = imgProcessor(img)
if err != nil {
return errToStatus(err), err
}
if imaging.Encode(w, img, format) != nil {
return errToStatus(err), err
if err := imgSvc.Resize(r.Context(), fd, width, height, w, options...); err != nil {
switch {
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
default:
return 0, err
}
}
return 0, nil
}

172
img/service.go Normal file
View File

@ -0,0 +1,172 @@
//go:generate go-enum --sql --marshal --file $GOFILE
package img
import (
"context"
"errors"
"io"
"path/filepath"
"github.com/disintegration/imaging"
"github.com/marusama/semaphore/v2"
"github.com/spf13/afero"
)
// ErrUnsupportedFormat means the given image format is not supported.
var ErrUnsupportedFormat = errors.New("unsupported image format")
// Service
type Service struct {
lowPrioritySem semaphore.Semaphore
highPrioritySem semaphore.Semaphore
}
func New(workers int) *Service {
return &Service{
lowPrioritySem: semaphore.New(workers),
highPrioritySem: semaphore.New(workers),
}
}
// Format is an image file format.
/*
ENUM(
jpeg
png
gif
tiff
bmp
)
*/
type Format int
func (x Format) toImaging() imaging.Format {
switch x {
case FormatJpeg:
return imaging.JPEG
case FormatPng:
return imaging.PNG
case FormatGif:
return imaging.GIF
case FormatTiff:
return imaging.TIFF
case FormatBmp:
return imaging.BMP
default:
return imaging.JPEG
}
}
/*
ENUM(
high
medium
low
)
*/
type Quality int
func (x Quality) resampleFilter() imaging.ResampleFilter {
switch x {
case QualityHigh:
return imaging.Lanczos
case QualityMedium:
return imaging.Box
case QualityLow:
return imaging.NearestNeighbor
default:
return imaging.Linear
}
}
/*
ENUM(
fit
fill
)
*/
type ResizeMode int
func (s *Service) FormatFromExtension(ext string) (Format, error) {
format, err := imaging.FormatFromExtension(ext)
if err != nil {
return -1, ErrUnsupportedFormat
}
switch format {
case imaging.JPEG:
return FormatJpeg, nil
case imaging.PNG:
return FormatPng, nil
case imaging.GIF:
return FormatGif, nil
case imaging.TIFF:
return FormatTiff, nil
case imaging.BMP:
return FormatBmp, nil
}
return -1, ErrUnsupportedFormat
}
type resizeConfig struct {
prioritized bool
resizeMode ResizeMode
quality Quality
}
type Option func(*resizeConfig)
func WithMode(mode ResizeMode) Option {
return func(config *resizeConfig) {
config.resizeMode = mode
}
}
func WithQuality(quality Quality) Option {
return func(config *resizeConfig) {
config.quality = quality
}
}
func WithHighPriority() Option {
return func(config *resizeConfig) {
config.prioritized = true
}
}
func (s *Service) Resize(ctx context.Context, file afero.File, width, height int, out io.Writer, options ...Option) error {
config := resizeConfig{
resizeMode: ResizeModeFit,
quality: QualityMedium,
}
for _, option := range options {
option(&config)
}
sem := s.lowPrioritySem
if config.prioritized {
sem = s.highPrioritySem
}
if err := sem.Acquire(ctx, 1); err != nil {
return err
}
defer sem.Release(1)
format, err := s.FormatFromExtension(filepath.Ext(file.Name()))
if err != nil {
return ErrUnsupportedFormat
}
img, err := imaging.Decode(file, imaging.AutoOrientation(true))
if err != nil {
return err
}
switch config.resizeMode {
case ResizeModeFill:
img = imaging.Fill(img, width, height, imaging.Center, config.quality.resampleFilter())
default:
img = imaging.Fit(img, width, height, config.quality.resampleFilter())
}
return imaging.Encode(out, img, format.toImaging())
}

259
img/service_enum.go Normal file
View File

@ -0,0 +1,259 @@
// Code generated by go-enum
// DO NOT EDIT!
package img
import (
"database/sql/driver"
"fmt"
)
const (
// FormatJpeg is a Format of type Jpeg
FormatJpeg Format = iota
// FormatPng is a Format of type Png
FormatPng
// FormatGif is a Format of type Gif
FormatGif
// FormatTiff is a Format of type Tiff
FormatTiff
// FormatBmp is a Format of type Bmp
FormatBmp
)
const _FormatName = "jpegpnggiftiffbmp"
var _FormatMap = map[Format]string{
0: _FormatName[0:4],
1: _FormatName[4:7],
2: _FormatName[7:10],
3: _FormatName[10:14],
4: _FormatName[14:17],
}
// String implements the Stringer interface.
func (x Format) String() string {
if str, ok := _FormatMap[x]; ok {
return str
}
return fmt.Sprintf("Format(%d)", x)
}
var _FormatValue = map[string]Format{
_FormatName[0:4]: 0,
_FormatName[4:7]: 1,
_FormatName[7:10]: 2,
_FormatName[10:14]: 3,
_FormatName[14:17]: 4,
}
// ParseFormat attempts to convert a string to a Format
func ParseFormat(name string) (Format, error) {
if x, ok := _FormatValue[name]; ok {
return x, nil
}
return Format(0), fmt.Errorf("%s is not a valid Format", name)
}
// MarshalText implements the text marshaller method
func (x Format) MarshalText() ([]byte, error) {
return []byte(x.String()), nil
}
// UnmarshalText implements the text unmarshaller method
func (x *Format) UnmarshalText(text []byte) error {
name := string(text)
tmp, err := ParseFormat(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Scan implements the Scanner interface.
func (x *Format) Scan(value interface{}) error {
var name string
switch v := value.(type) {
case string:
name = v
case []byte:
name = string(v)
case nil:
*x = Format(0)
return nil
}
tmp, err := ParseFormat(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Value implements the driver Valuer interface.
func (x Format) Value() (driver.Value, error) {
return x.String(), nil
}
const (
// QualityHigh is a Quality of type High
QualityHigh Quality = iota
// QualityMedium is a Quality of type Medium
QualityMedium
// QualityLow is a Quality of type Low
QualityLow
)
const _QualityName = "highmediumlow"
var _QualityMap = map[Quality]string{
0: _QualityName[0:4],
1: _QualityName[4:10],
2: _QualityName[10:13],
}
// String implements the Stringer interface.
func (x Quality) String() string {
if str, ok := _QualityMap[x]; ok {
return str
}
return fmt.Sprintf("Quality(%d)", x)
}
var _QualityValue = map[string]Quality{
_QualityName[0:4]: 0,
_QualityName[4:10]: 1,
_QualityName[10:13]: 2,
}
// ParseQuality attempts to convert a string to a Quality
func ParseQuality(name string) (Quality, error) {
if x, ok := _QualityValue[name]; ok {
return x, nil
}
return Quality(0), fmt.Errorf("%s is not a valid Quality", name)
}
// MarshalText implements the text marshaller method
func (x Quality) MarshalText() ([]byte, error) {
return []byte(x.String()), nil
}
// UnmarshalText implements the text unmarshaller method
func (x *Quality) UnmarshalText(text []byte) error {
name := string(text)
tmp, err := ParseQuality(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Scan implements the Scanner interface.
func (x *Quality) Scan(value interface{}) error {
var name string
switch v := value.(type) {
case string:
name = v
case []byte:
name = string(v)
case nil:
*x = Quality(0)
return nil
}
tmp, err := ParseQuality(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Value implements the driver Valuer interface.
func (x Quality) Value() (driver.Value, error) {
return x.String(), nil
}
const (
// ResizeModeFit is a ResizeMode of type Fit
ResizeModeFit ResizeMode = iota
// ResizeModeFill is a ResizeMode of type Fill
ResizeModeFill
)
const _ResizeModeName = "fitfill"
var _ResizeModeMap = map[ResizeMode]string{
0: _ResizeModeName[0:3],
1: _ResizeModeName[3:7],
}
// String implements the Stringer interface.
func (x ResizeMode) String() string {
if str, ok := _ResizeModeMap[x]; ok {
return str
}
return fmt.Sprintf("ResizeMode(%d)", x)
}
var _ResizeModeValue = map[string]ResizeMode{
_ResizeModeName[0:3]: 0,
_ResizeModeName[3:7]: 1,
}
// ParseResizeMode attempts to convert a string to a ResizeMode
func ParseResizeMode(name string) (ResizeMode, error) {
if x, ok := _ResizeModeValue[name]; ok {
return x, nil
}
return ResizeMode(0), fmt.Errorf("%s is not a valid ResizeMode", name)
}
// MarshalText implements the text marshaller method
func (x ResizeMode) MarshalText() ([]byte, error) {
return []byte(x.String()), nil
}
// UnmarshalText implements the text unmarshaller method
func (x *ResizeMode) UnmarshalText(text []byte) error {
name := string(text)
tmp, err := ParseResizeMode(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Scan implements the Scanner interface.
func (x *ResizeMode) Scan(value interface{}) error {
var name string
switch v := value.(type) {
case string:
name = v
case []byte:
name = string(v)
case nil:
*x = ResizeMode(0)
return nil
}
tmp, err := ParseResizeMode(name)
if err != nil {
return err
}
*x = tmp
return nil
}
// Value implements the driver Valuer interface.
func (x ResizeMode) Value() (driver.Value, error) {
return x.String(), nil
}