feat: support alist as a storage

pkg/storage/alist.go

@@ -0,0 +1,72 @@
+package storage
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"polaris/pkg/alist"
+
+	"github.com/gabriel-vasile/mimetype"
+)
+
+func NewAlist(cfg *alist.Config, dir string) (*Alist, error) {
+	cl := alist.New(cfg)
+	_, err := cl.Login()
+	if err != nil {
+		return nil, err
+	}
+	return &Alist{baseDir: dir, cfg: cfg, client: cl}, nil
+}
+
+type Alist struct {
+	baseDir    string
+	cfg        *alist.Config
+	client     *alist.Client
+	progresser func() float64
+}
+
+func (a *Alist) Move(src, dest string) error {
+	if err := a.Copy(src, dest); err != nil {
+		return err
+	}
+	return os.RemoveAll(src)
+}
+
+func (a *Alist) Copy(src, dest string) error {
+	b, err := NewBase(src)
+	if err != nil {
+		return err
+	}
+	a.progresser = b.Progress
+
+	uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error {
+		_, err := a.client.UploadStream(srcReader, destPath)
+		return err
+	}
+	mkdirFunc := func(dir string) error {
+		return a.client.Mkdir(dir)
+	}
+
+	baseDest := filepath.Join(a.baseDir, dest)
+	return b.Upload(baseDest, false, false, false, uploadFunc, mkdirFunc)
+}
+
+func (a *Alist) ReadDir(dir string) ([]fs.FileInfo, error) {
+	return nil, nil
+}
+
+func (a *Alist) ReadFile(s string) ([]byte, error) {
+	return nil, nil
+}
+
+func (a *Alist) WriteFile(s string, bytes []byte) error {
+	return nil
+}
+
+func (a *Alist) UploadProgress() float64 {
+	if a.progresser == nil {
+		return 0
+	}
+	return a.progresser()
+}

pkg/storage/base.go

@@ -0,0 +1,132 @@
+package storage
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"polaris/log"
+	"polaris/pkg/utils"
+
+	"github.com/gabriel-vasile/mimetype"
+	"github.com/pkg/errors"
+)
+type Storage interface {
+	Move(src, dest string) error
+	Copy(src, dest string) error
+	ReadDir(dir string) ([]fs.FileInfo, error)
+	ReadFile(string) ([]byte, error)
+	WriteFile(string, []byte) error
+	UploadProgress() float64
+}
+
+
+type uploadFunc func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error
+
+type Base struct {
+	src          string
+	totalSize    int64
+	uploadedSize int64
+}
+
+func NewBase(src string) (*Base, error) {
+	b := &Base{src: src}
+	err := b.calculateSize()
+	return b, err
+}
+
+func (b *Base) Upload(destDir string, tryLink, detectMime, changeMediaHash bool, upload uploadFunc, mkdir func(string) error) error {
+	os.MkdirAll(destDir, os.ModePerm)
+
+	targetBase := filepath.Join(destDir, filepath.Base(b.src)) //文件的场景，要加上文件名, move filename ./dir/
+	info, err := os.Stat(b.src)
+	if err != nil {
+		return errors.Wrap(err, "read source dir")
+	}
+	if info.IsDir() { //如果是路径，则只移动路径里面的文件，不管当前路径, 行为类似 move dirname/* target_dir/
+		targetBase = destDir
+	}
+	log.Debugf("local storage target base dir is: %v", targetBase)
+
+	err = filepath.Walk(b.src, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(b.src, path)
+		if err != nil {
+			return errors.Wrapf(err, "relation between %s and %s", b.src, path)
+		}
+		destName := filepath.Join(targetBase, rel)
+
+		if info.IsDir() {
+			mkdir(destName)
+		} else { //is file
+			if tryLink {
+				if err := os.Link(path, destName); err == nil {
+					return nil //link success
+				}
+				log.Warnf("hard link file error: %v, will try copy file, source: %s, dest: %s", err, path, destName)
+			}
+			if changeMediaHash {
+				if err := utils.ChangeFileHash(path); err != nil {
+					log.Errorf("change file %v hash error: %v", path, err)
+				}
+			}
+
+			if f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm); err != nil {
+				return errors.Wrapf(err, "read file %v", path)
+			} else { //open success
+				defer f.Close()
+				var mtype *mimetype.MIME
+				if detectMime {
+					mtype, err = mimetype.DetectFile(path)
+					if err != nil {
+						return errors.Wrap(err, "mime type error")
+					}
+				}
+				return upload(destName, info, &progressReader{R: f, Add: func(i int) {
+					b.uploadedSize += int64(i)
+				}}, mtype)
+			}
+
+		}
+		log.Infof("file copy complete: %v", destName)
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "move file error")
+	}
+	return nil
+
+}
+
+func (b *Base) calculateSize() error {
+	var size int64
+	err := filepath.Walk(b.src, func(_ string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	b.totalSize = size
+	return err
+}
+
+func (b *Base) Progress() float64 {
+	return float64(b.uploadedSize)/float64(b.totalSize)
+}
+
+
+type progressReader struct {
+	R io.Reader
+	Add func(int)
+}
+
+func (pr *progressReader) Read(p []byte) (int, error) {
+	n, err := pr.R.Read(p)
+	pr.Add(n)
+	return n, err
+}

pkg/storage/local.go

@@ -6,22 +6,14 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"polaris/log"
 
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/pkg/errors"
 )
 
-type Storage interface {
-	Move(src, dest string) error
-	Copy(src, dest string) error
-	ReadDir(dir string) ([]fs.FileInfo, error)
-	ReadFile(string) ([]byte, error)
-	WriteFile(string, []byte) error
-}
 
 func NewLocalStorage(dir string) (*LocalStorage, error) {
 	os.MkdirAll(dir, 0655)
-
 	return &LocalStorage{dir: dir}, nil
 }
 
@@ -30,57 +22,28 @@ type LocalStorage struct {
 }
 
 func (l *LocalStorage) Copy(src, destDir string) error {
-	os.MkdirAll(filepath.Join(l.dir, destDir), os.ModePerm)
-
-	targetBase := filepath.Join(l.dir, destDir, filepath.Base(src)) //文件的场景，要加上文件名, move filename ./dir/
-	info, err := os.Stat(src)
+	b, err := NewBase(src)
 	if err != nil {
-		return errors.Wrap(err, "read source dir")
+		return err
 	}
-	if info.IsDir() { //如果是路径，则只移动路径里面的文件，不管当前路径, 行为类似 move dirname/* target_dir/
-		targetBase = filepath.Join(l.dir, destDir)
-	}
-	log.Debugf("local storage target base dir is: %v", targetBase)
-
-	err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		rel, err := filepath.Rel(src, path)
-		if err != nil {
-			return errors.Wrapf(err, "relation between %s and %s", src, path)
-		}
-		destName := filepath.Join(targetBase, rel)
-
-		if info.IsDir() {
-			os.Mkdir(destName, os.ModePerm)
-		} else { //is file
-			if err := os.Link(path, destName); err != nil {
-				log.Warnf("hard link file error: %v, will try copy file, source: %s, dest: %s", err, path, destName)
-				if writer, err := os.OpenFile(destName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil {
-					return errors.Wrapf(err, "create file %s", destName)
-				} else {
-					defer writer.Close()
-					if f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm); err != nil {
-						return errors.Wrapf(err, "read file %v", path)
-					} else { //open success
-						defer f.Close()
-						_, err := io.Copy(writer, f)
-						if err != nil {
-							return errors.Wrap(err, "transmitting data error")
-						}
-					}
-				}
 
+	baseDest := filepath.Join(l.dir, destDir)
+	uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error {
+		if writer, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil {
+			return errors.Wrapf(err, "create file %s", destPath)
+		} else {
+			defer writer.Close()
+			_, err := io.Copy(writer, srcReader)
+			if err != nil {
+				return errors.Wrap(err, "transmitting data error")
 			}
+
 		}
-		log.Infof("file copy complete: %v", destName)
 		return nil
-	})
-	if err != nil {
-		return errors.Wrap(err, "move file error")
 	}
-	return nil
+	return b.Upload(baseDest, true, false, false, uploadFunc, func(s string) error {
+		return os.Mkdir(s, os.ModePerm)
+	})
 }
 
 func (l *LocalStorage) Move(src, destDir string) error {
@@ -103,3 +66,7 @@ func (l *LocalStorage) WriteFile(name string, data []byte) error {
 	os.MkdirAll(filepath.Dir(path), os.ModePerm)
 	return os.WriteFile(path, data, os.ModePerm)
 }
+
+func (l *LocalStorage) UploadProgress() float64 {
+	return 0
+}

pkg/storage/webdav.go

@@ -1,13 +1,12 @@
 package storage
 
 import (
+	"io"
 	"io/fs"
 	"net/http"
 	"os"
 	"path/filepath"
-	"polaris/log"
 	"polaris/pkg/gowebdav"
-	"polaris/pkg/utils"
 
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/pkg/errors"
@@ -17,6 +16,7 @@ type WebdavStorage struct {
 	fs              *gowebdav.Client
 	dir             string
 	changeMediaHash bool
+	progresser func() float64
 }
 
 func NewWebdavStorage(url, user, password, path string, changeMediaHash bool) (*WebdavStorage, error) {
@@ -31,67 +31,29 @@ func NewWebdavStorage(url, user, password, path string, changeMediaHash bool) (*
 }
 
 func (w *WebdavStorage) Copy(local, remoteDir string) error {
-	remoteBase := filepath.Join(w.dir, remoteDir, filepath.Base(local))
-	info, err := os.Stat(local)
+	b, err := NewBase(local)
 	if err != nil {
-		return errors.Wrap(err, "read source dir")
-	}
-	if info.IsDir() { //如果是路径，则只移动路径里面的文件，不管当前路径, 行为类似 move dirname/* target_dir/
-		remoteBase = filepath.Join(w.dir, remoteDir)
+		return err
 	}
 
-	//log.Infof("remove all content in %s", remoteBase)
-	//w.fs.RemoveAll(remoteBase)
-	err = filepath.Walk(local, func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return errors.Wrapf(err, "read file %v", path)
+	w.progresser = b.Progress
+
+	uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mtype *mimetype.MIME) error {
+		callback := func(r *http.Request) {
+			r.Header.Set("Content-Type", mtype.String())
+			r.ContentLength = destInfo.Size()
 		}
 
-		rel, err := filepath.Rel(local, path)
-		if err != nil {
-			return errors.Wrap(err, "path relation")
+		if err := w.fs.WriteStream(destPath, srcReader, 0666, callback); err != nil {
+			return errors.Wrap(err, "transmitting data error")
 		}
-		remoteName := filepath.Join(remoteBase, rel)
+		return nil
 
-		if info.IsDir() {
-			log.Infof("skip dir %v, webdav will mkdir automatically", info.Name())
+	}
 
-			// if err := w.fs.Mkdir(remoteName, 0666); err != nil {
-			// 	return errors.Wrapf(err, "mkdir %v", remoteName)
-			// }
-
-		} else { //is file
-			if w.changeMediaHash {
-				if err := utils.ChangeFileHash(path); err != nil {
-					log.Errorf("change file %v hash error: %v", path, err)
-				}
-			}
-			if f, err := os.OpenFile(path, os.O_RDONLY, 0666); err != nil {
-				return errors.Wrapf(err, "read file %v", path)
-			} else { //open success
-				defer f.Close()
-				mtype, err := mimetype.DetectFile(path)
-				if err != nil {
-					return errors.Wrap(err, "mime type error")
-				}
-
-				callback := func(r *http.Request) {
-					r.Header.Set("Content-Type", mtype.String())
-					r.ContentLength = info.Size()
-				}
-
-				if err := w.fs.WriteStream(remoteName, f, 0666, callback); err != nil {
-					return errors.Wrap(err, "transmitting data error")
-				}
-			}
-		}
-		log.Infof("file copy complete: %v", remoteName)
+	return b.Upload(filepath.Join(w.dir, remoteDir), false, true, w.changeMediaHash, uploadFunc, func(s string) error {
 		return nil
 	})
-	if err != nil {
-		return errors.Wrap(err, "move file error")
-	}
-	return nil
 }
 
 func (w *WebdavStorage) Move(local, remoteDir string) error {
@@ -112,3 +74,10 @@ func (w *WebdavStorage) ReadFile(name string) ([]byte, error) {
 func (w *WebdavStorage) WriteFile(name string, data []byte) error {
 	return w.fs.Write(filepath.Join(w.dir, name), data, os.ModePerm)
 }
+
+func (w *WebdavStorage) UploadProgress() float64 {
+	if w.progresser == nil {
+		return 0
+	}
+	return w.progresser()
+}