feat: support alist as a storage

This commit is contained in:
Simon Ding
2024-11-17 21:21:21 +08:00
parent b136b9167f
commit 7d5ce8ba97
14 changed files with 536 additions and 121 deletions

View File

@@ -180,7 +180,7 @@ var (
StoragesColumns = []*schema.Column{
{Name: "id", Type: field.TypeInt, Increment: true},
{Name: "name", Type: field.TypeString, Unique: true},
{Name: "implementation", Type: field.TypeEnum, Enums: []string{"webdav", "local"}},
{Name: "implementation", Type: field.TypeEnum, Enums: []string{"webdav", "local", "alist"}},
{Name: "tv_path", Type: field.TypeString, Nullable: true},
{Name: "movie_path", Type: field.TypeString, Nullable: true},
{Name: "settings", Type: field.TypeString, Nullable: true},

View File

@@ -14,7 +14,7 @@ type Storage struct {
func (Storage) Fields() []ent.Field {
return []ent.Field{
field.String("name").Unique(),
field.Enum("implementation").Values("webdav", "local"),
field.Enum("implementation").Values("webdav", "local", "alist"),
field.String("tv_path").Optional(),
field.String("movie_path").Optional(),
field.String("settings").Optional(),

View File

@@ -67,6 +67,7 @@ type Implementation string
const (
ImplementationWebdav Implementation = "webdav"
ImplementationLocal Implementation = "local"
ImplementationAlist Implementation = "alist"
)
func (i Implementation) String() string {
@@ -76,7 +77,7 @@ func (i Implementation) String() string {
// ImplementationValidator is a validator for the "implementation" field enum values. It is called by the builders before save.
func ImplementationValidator(i Implementation) error {
switch i {
case ImplementationWebdav, ImplementationLocal:
case ImplementationWebdav, ImplementationLocal, ImplementationAlist:
return nil
default:
return fmt.Errorf("storage: invalid enum value for implementation field: %q", i)

194
pkg/alist/alist.go Normal file
View File

@@ -0,0 +1,194 @@
package alist
import (
"bytes"
"encoding/json"
"io"
"net/http"
"strings"
"time"
"github.com/pkg/errors"
)
type Resposne[T any] struct {
Code int `json:"code"`
Message string `json:"message"`
Data T `json:"data"`
}
type Config struct {
Username string
Password string
URL string
}
func New(cfg *Config) *Client {
cfg.URL = strings.Trim(cfg.URL, "/")
return &Client{
cfg: cfg,
http: http.DefaultClient,
}
}
type Client struct {
cfg *Config
http *http.Client
token string
}
func (c *Client) Login() (string, error) {
p := map[string]string{
"username": c.cfg.Username,
"password": c.cfg.Password,
}
data, _ := json.Marshal(p)
resp, err := c.http.Post(c.cfg.URL+loginUrl, "application/json", bytes.NewBuffer(data))
if err != nil {
return "", errors.Wrap(err, "login")
}
defer resp.Body.Close()
d1, err := io.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "read body")
}
var rp Resposne[map[string]string]
err = json.Unmarshal(d1, &rp)
if err != nil {
return "", errors.Wrap(err, "json")
}
if rp.Code != 200 {
return "", errors.Errorf("alist error: code %d, %s", rp.Code, rp.Message)
}
c.token = rp.Data["token"]
return c.token, nil
}
type LsInfo struct {
Content []struct {
Name string `json:"name"`
Size int `json:"size"`
IsDir bool `json:"is_dir"`
Modified time.Time `json:"modified"`
Created time.Time `json:"created"`
Sign string `json:"sign"`
Thumb string `json:"thumb"`
Type int `json:"type"`
Hashinfo string `json:"hashinfo"`
HashInfo any `json:"hash_info"`
} `json:"content"`
Total int `json:"total"`
Readme string `json:"readme"`
Header string `json:"header"`
Write bool `json:"write"`
Provider string `json:"provider"`
}
func (c *Client) Ls(dir string) (*LsInfo, error) {
in := map[string]string{
"path": dir,
}
resp, err := c.post(c.cfg.URL+lsUrl, in)
if err != nil {
return nil, errors.Wrap(err, "http")
}
var out Resposne[LsInfo]
err = json.Unmarshal(resp, &out)
if err != nil {
return nil, err
}
if out.Code != 200 {
return nil, errors.Errorf("alist error: code %d, %s", out.Code, out.Message)
}
return &out.Data, nil
}
func (c *Client) Mkdir(dir string) error {
in := map[string]string{
"path": dir,
}
resp, err := c.post(c.cfg.URL+mkdirUrl, in)
if err != nil {
return errors.Wrap(err, "http")
}
var out Resposne[any]
err = json.Unmarshal(resp, &out)
if err != nil {
return err
}
if out.Code != 200 {
return errors.Errorf("alist error: code %d, %s", out.Code, out.Message)
}
return nil
}
func (c *Client) post(url string, body interface{}) ([]byte, error) {
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return nil, errors.Wrap(err, "new request")
}
req.Header.Add("Authorization", c.token)
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return nil, errors.Wrap(err, "http")
}
defer resp.Body.Close()
d1, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "read body")
}
return d1, nil
}
type UploadStreamResponse struct {
Task struct {
ID string `json:"id"`
Name string `json:"name"`
State int `json:"state"`
Status string `json:"status"`
Progress int `json:"progress"`
Error string `json:"error"`
} `json:"task"`
}
func (c *Client) UploadStream(reader io.Reader, toDir string) (*UploadStreamResponse, error) {
req, err := http.NewRequest(http.MethodPost, c.cfg.URL+streamUploadUrl, reader)
if err != nil {
return nil, err
}
req.Header.Add("Authorization", "{{alist_token}}")
req.Header.Add("File-Path", toDir)
req.Header.Add("As-Task", "true")
req.Header.Add("Content-Length", "")
req.Header.Add("Content-Type", "application/octet-stream")
res, err := c.http.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
d1, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
var out Resposne[UploadStreamResponse]
err = json.Unmarshal(d1, &out)
if err != nil {
return nil, err
}
if out.Code != 200 {
return nil, errors.Errorf("alist error: code %d, %s", out.Code, out.Message)
}
return &out.Data, nil
}

30
pkg/alist/alist_test.go Normal file
View File

@@ -0,0 +1,30 @@
package alist
import (
"polaris/log"
"testing"
)
func TestLogin(t *testing.T) {
c := New(&Config{
URL: "http://10.0.0.8:5244/",
Username: "",
Password: "",
})
cre, err := c.Login()
if err != nil {
log.Errorf("login fail: %v", err)
t.Fail()
} else {
log.Errorf("login success: %s", cre)
}
info, err := c.Ls("/aliyun")
if err != nil {
log.Errorf("ls fail: %v", err)
t.Fail()
} else {
log.Infof("ls results: %+v", info)
}
err = c.Mkdir("/aliyun/test1")
log.Errorf("mkdir: %v", err)
}

8
pkg/alist/url.go Normal file
View File

@@ -0,0 +1,8 @@
package alist
const (
loginUrl = "/api/auth/login"
lsUrl = "/api/fs/list"
mkdirUrl = "/api/fs/mkdir"
streamUploadUrl = "/api/fs/put"
)

72
pkg/storage/alist.go Normal file
View File

@@ -0,0 +1,72 @@
package storage
import (
"io"
"io/fs"
"os"
"path/filepath"
"polaris/pkg/alist"
"github.com/gabriel-vasile/mimetype"
)
func NewAlist(cfg *alist.Config, dir string) (*Alist, error) {
cl := alist.New(cfg)
_, err := cl.Login()
if err != nil {
return nil, err
}
return &Alist{baseDir: dir, cfg: cfg, client: cl}, nil
}
type Alist struct {
baseDir string
cfg *alist.Config
client *alist.Client
progresser func() float64
}
func (a *Alist) Move(src, dest string) error {
if err := a.Copy(src, dest); err != nil {
return err
}
return os.RemoveAll(src)
}
func (a *Alist) Copy(src, dest string) error {
b, err := NewBase(src)
if err != nil {
return err
}
a.progresser = b.Progress
uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error {
_, err := a.client.UploadStream(srcReader, destPath)
return err
}
mkdirFunc := func(dir string) error {
return a.client.Mkdir(dir)
}
baseDest := filepath.Join(a.baseDir, dest)
return b.Upload(baseDest, false, false, false, uploadFunc, mkdirFunc)
}
func (a *Alist) ReadDir(dir string) ([]fs.FileInfo, error) {
return nil, nil
}
func (a *Alist) ReadFile(s string) ([]byte, error) {
return nil, nil
}
func (a *Alist) WriteFile(s string, bytes []byte) error {
return nil
}
func (a *Alist) UploadProgress() float64 {
if a.progresser == nil {
return 0
}
return a.progresser()
}

132
pkg/storage/base.go Normal file
View File

@@ -0,0 +1,132 @@
package storage
import (
"io"
"io/fs"
"os"
"path/filepath"
"polaris/log"
"polaris/pkg/utils"
"github.com/gabriel-vasile/mimetype"
"github.com/pkg/errors"
)
type Storage interface {
Move(src, dest string) error
Copy(src, dest string) error
ReadDir(dir string) ([]fs.FileInfo, error)
ReadFile(string) ([]byte, error)
WriteFile(string, []byte) error
UploadProgress() float64
}
type uploadFunc func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error
type Base struct {
src string
totalSize int64
uploadedSize int64
}
func NewBase(src string) (*Base, error) {
b := &Base{src: src}
err := b.calculateSize()
return b, err
}
func (b *Base) Upload(destDir string, tryLink, detectMime, changeMediaHash bool, upload uploadFunc, mkdir func(string) error) error {
os.MkdirAll(destDir, os.ModePerm)
targetBase := filepath.Join(destDir, filepath.Base(b.src)) //文件的场景,要加上文件名, move filename ./dir/
info, err := os.Stat(b.src)
if err != nil {
return errors.Wrap(err, "read source dir")
}
if info.IsDir() { //如果是路径,则只移动路径里面的文件,不管当前路径, 行为类似 move dirname/* target_dir/
targetBase = destDir
}
log.Debugf("local storage target base dir is: %v", targetBase)
err = filepath.Walk(b.src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(b.src, path)
if err != nil {
return errors.Wrapf(err, "relation between %s and %s", b.src, path)
}
destName := filepath.Join(targetBase, rel)
if info.IsDir() {
mkdir(destName)
} else { //is file
if tryLink {
if err := os.Link(path, destName); err == nil {
return nil //link success
}
log.Warnf("hard link file error: %v, will try copy file, source: %s, dest: %s", err, path, destName)
}
if changeMediaHash {
if err := utils.ChangeFileHash(path); err != nil {
log.Errorf("change file %v hash error: %v", path, err)
}
}
if f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm); err != nil {
return errors.Wrapf(err, "read file %v", path)
} else { //open success
defer f.Close()
var mtype *mimetype.MIME
if detectMime {
mtype, err = mimetype.DetectFile(path)
if err != nil {
return errors.Wrap(err, "mime type error")
}
}
return upload(destName, info, &progressReader{R: f, Add: func(i int) {
b.uploadedSize += int64(i)
}}, mtype)
}
}
log.Infof("file copy complete: %v", destName)
return nil
})
if err != nil {
return errors.Wrap(err, "move file error")
}
return nil
}
func (b *Base) calculateSize() error {
var size int64
err := filepath.Walk(b.src, func(_ string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
size += info.Size()
}
return err
})
b.totalSize = size
return err
}
func (b *Base) Progress() float64 {
return float64(b.uploadedSize)/float64(b.totalSize)
}
type progressReader struct {
R io.Reader
Add func(int)
}
func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.R.Read(p)
pr.Add(n)
return n, err
}

View File

@@ -6,22 +6,14 @@ import (
"io/ioutil"
"os"
"path/filepath"
"polaris/log"
"github.com/gabriel-vasile/mimetype"
"github.com/pkg/errors"
)
type Storage interface {
Move(src, dest string) error
Copy(src, dest string) error
ReadDir(dir string) ([]fs.FileInfo, error)
ReadFile(string) ([]byte, error)
WriteFile(string, []byte) error
}
func NewLocalStorage(dir string) (*LocalStorage, error) {
os.MkdirAll(dir, 0655)
return &LocalStorage{dir: dir}, nil
}
@@ -30,57 +22,28 @@ type LocalStorage struct {
}
func (l *LocalStorage) Copy(src, destDir string) error {
os.MkdirAll(filepath.Join(l.dir, destDir), os.ModePerm)
targetBase := filepath.Join(l.dir, destDir, filepath.Base(src)) //文件的场景,要加上文件名, move filename ./dir/
info, err := os.Stat(src)
b, err := NewBase(src)
if err != nil {
return errors.Wrap(err, "read source dir")
return err
}
if info.IsDir() { //如果是路径,则只移动路径里面的文件,不管当前路径, 行为类似 move dirname/* target_dir/
targetBase = filepath.Join(l.dir, destDir)
}
log.Debugf("local storage target base dir is: %v", targetBase)
err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(src, path)
if err != nil {
return errors.Wrapf(err, "relation between %s and %s", src, path)
}
destName := filepath.Join(targetBase, rel)
if info.IsDir() {
os.Mkdir(destName, os.ModePerm)
} else { //is file
if err := os.Link(path, destName); err != nil {
log.Warnf("hard link file error: %v, will try copy file, source: %s, dest: %s", err, path, destName)
if writer, err := os.OpenFile(destName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil {
return errors.Wrapf(err, "create file %s", destName)
} else {
defer writer.Close()
if f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm); err != nil {
return errors.Wrapf(err, "read file %v", path)
} else { //open success
defer f.Close()
_, err := io.Copy(writer, f)
if err != nil {
return errors.Wrap(err, "transmitting data error")
}
}
}
baseDest := filepath.Join(l.dir, destDir)
uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mimeType *mimetype.MIME) error {
if writer, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm); err != nil {
return errors.Wrapf(err, "create file %s", destPath)
} else {
defer writer.Close()
_, err := io.Copy(writer, srcReader)
if err != nil {
return errors.Wrap(err, "transmitting data error")
}
}
log.Infof("file copy complete: %v", destName)
return nil
})
if err != nil {
return errors.Wrap(err, "move file error")
}
return nil
return b.Upload(baseDest, true, false, false, uploadFunc, func(s string) error {
return os.Mkdir(s, os.ModePerm)
})
}
func (l *LocalStorage) Move(src, destDir string) error {
@@ -103,3 +66,7 @@ func (l *LocalStorage) WriteFile(name string, data []byte) error {
os.MkdirAll(filepath.Dir(path), os.ModePerm)
return os.WriteFile(path, data, os.ModePerm)
}
func (l *LocalStorage) UploadProgress() float64 {
return 0
}

View File

@@ -1,13 +1,12 @@
package storage
import (
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"polaris/log"
"polaris/pkg/gowebdav"
"polaris/pkg/utils"
"github.com/gabriel-vasile/mimetype"
"github.com/pkg/errors"
@@ -17,6 +16,7 @@ type WebdavStorage struct {
fs *gowebdav.Client
dir string
changeMediaHash bool
progresser func() float64
}
func NewWebdavStorage(url, user, password, path string, changeMediaHash bool) (*WebdavStorage, error) {
@@ -31,67 +31,29 @@ func NewWebdavStorage(url, user, password, path string, changeMediaHash bool) (*
}
func (w *WebdavStorage) Copy(local, remoteDir string) error {
remoteBase := filepath.Join(w.dir, remoteDir, filepath.Base(local))
info, err := os.Stat(local)
b, err := NewBase(local)
if err != nil {
return errors.Wrap(err, "read source dir")
}
if info.IsDir() { //如果是路径,则只移动路径里面的文件,不管当前路径, 行为类似 move dirname/* target_dir/
remoteBase = filepath.Join(w.dir, remoteDir)
return err
}
//log.Infof("remove all content in %s", remoteBase)
//w.fs.RemoveAll(remoteBase)
err = filepath.Walk(local, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return errors.Wrapf(err, "read file %v", path)
w.progresser = b.Progress
uploadFunc := func(destPath string, destInfo fs.FileInfo, srcReader io.Reader, mtype *mimetype.MIME) error {
callback := func(r *http.Request) {
r.Header.Set("Content-Type", mtype.String())
r.ContentLength = destInfo.Size()
}
rel, err := filepath.Rel(local, path)
if err != nil {
return errors.Wrap(err, "path relation")
if err := w.fs.WriteStream(destPath, srcReader, 0666, callback); err != nil {
return errors.Wrap(err, "transmitting data error")
}
remoteName := filepath.Join(remoteBase, rel)
return nil
if info.IsDir() {
log.Infof("skip dir %v, webdav will mkdir automatically", info.Name())
}
// if err := w.fs.Mkdir(remoteName, 0666); err != nil {
// return errors.Wrapf(err, "mkdir %v", remoteName)
// }
} else { //is file
if w.changeMediaHash {
if err := utils.ChangeFileHash(path); err != nil {
log.Errorf("change file %v hash error: %v", path, err)
}
}
if f, err := os.OpenFile(path, os.O_RDONLY, 0666); err != nil {
return errors.Wrapf(err, "read file %v", path)
} else { //open success
defer f.Close()
mtype, err := mimetype.DetectFile(path)
if err != nil {
return errors.Wrap(err, "mime type error")
}
callback := func(r *http.Request) {
r.Header.Set("Content-Type", mtype.String())
r.ContentLength = info.Size()
}
if err := w.fs.WriteStream(remoteName, f, 0666, callback); err != nil {
return errors.Wrap(err, "transmitting data error")
}
}
}
log.Infof("file copy complete: %v", remoteName)
return b.Upload(filepath.Join(w.dir, remoteDir), false, true, w.changeMediaHash, uploadFunc, func(s string) error {
return nil
})
if err != nil {
return errors.Wrap(err, "move file error")
}
return nil
}
func (w *WebdavStorage) Move(local, remoteDir string) error {
@@ -112,3 +74,10 @@ func (w *WebdavStorage) ReadFile(name string) ([]byte, error) {
func (w *WebdavStorage) WriteFile(name string, data []byte) error {
return w.fs.Write(filepath.Join(w.dir, name), data, os.ModePerm)
}
func (w *WebdavStorage) UploadProgress() float64 {
if w.progresser == nil {
return 0
}
return w.progresser()
}

View File

@@ -4,13 +4,13 @@ import (
"bytes"
"encoding/xml"
"fmt"
"github.com/pkg/errors"
"os"
"path/filepath"
"polaris/db"
"polaris/ent/media"
storage1 "polaris/ent/storage"
"polaris/log"
"polaris/pkg/alist"
"polaris/pkg/metadata"
"polaris/pkg/notifier"
"polaris/pkg/storage"
@@ -18,6 +18,8 @@ import (
"slices"
"strconv"
"strings"
"github.com/pkg/errors"
)
func (c *Client) writeNfoFile(historyId int) error {
@@ -218,6 +220,13 @@ func (c *Client) getStorage(storageId int, mediaType media.MediaType) (storage.S
return nil, errors.Wrap(err, "new webdav")
}
return storageImpl1, nil
case storage1.ImplementationAlist:
cfg := st.ToWebDavSetting()
storageImpl1, err := storage.NewAlist(&alist.Config{URL: cfg.URL, Username: cfg.User, Password: cfg.Password}, targetPath)
if err != nil {
return nil, errors.Wrap(err, "alist")
}
return storageImpl1, nil
}
return nil, errors.New("no storage found")
}

View File

@@ -71,7 +71,6 @@ func (s *Server) SetSetting(c *gin.Context) (interface{}, error) {
if _, err := template.New("test").Parse(in.MovieNamingFormat); err != nil {
return nil, errors.Wrap(err, "movie format")
}
s.db.SetSetting(db.SettingMovieNamingFormat, in.MovieNamingFormat)
} else {
s.db.SetSetting(db.SettingMovieNamingFormat, "")

View File

@@ -2,9 +2,11 @@ package server
import (
"fmt"
"os"
"polaris/db"
"polaris/log"
"polaris/pkg/alist"
"polaris/pkg/storage"
"polaris/pkg/utils"
"strconv"
@@ -39,6 +41,21 @@ func (s *Server) AddStorage(c *gin.Context) (interface{}, error) {
for _, f := range fs {
log.Infof("file name: %v", f.Name())
}
} else if in.Implementation == "alist" {
cfg := in.ToWebDavSetting()
_, err := storage.NewAlist(&alist.Config{URL: cfg.URL, Username: cfg.User, Password: cfg.Password}, in.TvPath)
if err != nil {
return nil, errors.Wrap(err, "alist")
}
} else if in.Implementation == "local" {
_, err := os.Stat(in.TvPath)
if err != nil {
return nil, err
}
_, err = os.Stat(in.MoviePath)
if err != nil {
return nil, err
}
}
log.Infof("received add storage input: %v", in)
err := s.db.AddStorage(&in)

View File

@@ -70,15 +70,16 @@ class _StorageState extends ConsumerState<StorageSettings> {
decoration: const InputDecoration(labelText: "名称"),
validator: FormBuilderValidators.required(),
),
s.implementation != "local"
s.implementation == "webdav" || s.implementation == "alist"
? Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
FormBuilderTextField(
name: "url",
autovalidateMode: AutovalidateMode.onUserInteraction,
decoration:
const InputDecoration(labelText: "Webdav地"),
decoration: const InputDecoration(
labelText: "",
hintText: "https://abc.somewebsite.com/"),
validator: FormBuilderValidators.required(),
),
FormBuilderTextField(
@@ -92,13 +93,15 @@ class _StorageState extends ConsumerState<StorageSettings> {
decoration: const InputDecoration(labelText: "密码"),
obscureText: true,
),
FormBuilderCheckbox(
name: "change_file_hash",
title: const Text(
"上传时更改文件哈希",
style: TextStyle(fontSize: 14),
),
),
s.implementation == "webdav"
? FormBuilderCheckbox(
name: "change_file_hash",
title: const Text(
"上传时更改文件哈希",
style: TextStyle(fontSize: 14),
),
)
: Container(),
],
)
: Container(),
@@ -149,6 +152,8 @@ class _StorageState extends ConsumerState<StorageSettings> {
title = "本地存储";
} else if (s.implementation == "webdav") {
title = "webdav 存储";
} else if (s.implementation == "alist") {
title = "Alist 存储";
}
return showSettingDialog(
@@ -189,6 +194,18 @@ class _StorageState extends ConsumerState<StorageSettings> {
Storage(implementation: "webdav", name: "webdav1"));
},
),
),
SettingsCard(
child: InkWell(
child: const Center(
child: Text("Alist"),
),
onTap: () {
Navigator.of(context).pop();
showStorageDetails(
Storage(implementation: "alist", name: "Alist1"));
},
),
)
],
),