diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 6a45bb7d4e1..569b3fba5c7 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -2,7 +2,11 @@ package url_tree import ( "context" + "errors" + "github.com/alist-org/alist/v3/internal/op" stdpath "path" + "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -14,7 +18,8 @@ import ( type Urls struct { model.Storage Addition - root *Node + root *Node + mutex sync.RWMutex } func (d *Urls) Config() driver.Config { @@ -40,11 +45,15 @@ func (d *Urls) Drop(ctx context.Context) error { } func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) } func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, dir.GetPath()) log.Debugf("path: %s, node: %+v", dir.GetPath(), node) if node == nil { @@ -59,6 +68,8 @@ func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, file.GetPath()) log.Debugf("path: %s, node: %+v", file.GetPath(), node) if node == nil { @@ -72,6 +83,174 @@ func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.NotFile } +func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + node := GetNodeFromRootByPath(d.root, parentDir.GetPath()) + if node == nil { + return nil, errs.ObjectNotFound + } + if node.isFile() { + return nil, errs.NotFolder + } + dir := &Node{ + Name: dirName, + Level: node.Level + 1, + } + node.Children = append(node.Children, dir) + d.updateStorage() + return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName)) +} + +func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot move parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcDir, srcName := stdpath.Split(srcObj.GetPath()) + srcParentNode := GetNodeFromRootByPath(d.root, srcDir) + if srcParentNode == nil { + return nil, errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(srcParentNode.Children)) + var srcNode *Node + for _, child := range srcParentNode.Children { + if child.Name == srcName { + srcNode = child + } else { + newChildren = append(newChildren, child) + } + } + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcParentNode.Children = newChildren + srcNode.setLevel(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, srcNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName)) +} + +func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcNode.Name = newName + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) +} + +func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot copy parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + newNode := srcNode.deepCopy(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, newNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath()))) +} + +func (d *Urls) Remove(ctx context.Context, obj model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + objDir, objName := stdpath.Split(obj.GetPath()) + nodeParent := GetNodeFromRootByPath(d.root, objDir) + if nodeParent == nil { + return errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(nodeParent.Children)) + var deletedObj *Node + for _, child := range nodeParent.Children { + if child.Name != objName { + newChildren = append(newChildren, child) + } else { + deletedObj = child + } + } + if deletedObj == nil { + return errs.ObjectNotFound + } + nodeParent.Children = newChildren + if deletedObj.Size > 0 { + d.root.calSize() + } + d.updateStorage() + return nil +} + +func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dirNode == nil || dirNode.isFile() { + return nil, errs.NotFolder + } + newNode := &Node{ + Name: name, + Level: dirNode.Level + 1, + Url: url, + } + dirNode.Children = append(dirNode.Children, newNode) + if d.HeadSize { + size, err := getSizeFromUrl(url) + if err != nil { + log.Errorf("get size from url error: %s", err) + } else { + newNode.Size = size + d.root.calSize() + } + } + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name)) +} + +func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return errs.UploadNotSupported +} + +func (d *Urls) updateStorage() { + d.UrlStructure = StringifyTree(d.root) + op.MustSaveDriverStorage(d) +} + //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go index b3ae33dc059..c40414f58c3 100644 --- a/drivers/url_tree/meta.go +++ b/drivers/url_tree/meta.go @@ -12,6 +12,7 @@ type Addition struct { // define other UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` + Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ @@ -20,7 +21,7 @@ var config = driver.Config{ OnlyLocal: false, OnlyProxy: false, NoCache: true, - NoUpload: true, + NoUpload: false, NeedMs: false, DefaultRoot: "", CheckStatus: true, diff --git a/drivers/url_tree/types.go b/drivers/url_tree/types.go index 7e8ca3d93ae..cf62d29d65a 100644 --- a/drivers/url_tree/types.go +++ b/drivers/url_tree/types.go @@ -1,5 +1,7 @@ package url_tree +import "github.com/alist-org/alist/v3/pkg/utils" + // Node is a node in the folder tree type Node struct { Url string @@ -44,3 +46,19 @@ func (node *Node) calSize() int64 { node.Size = size return size } + +func (node *Node) setLevel(level int) { + node.Level = level + for _, child := range node.Children { + child.setLevel(level + 1) + } +} + +func (node *Node) deepCopy(level int) *Node { + ret := *node + ret.Level = level + ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) { + return child.deepCopy(level + 1), nil + }) + return &ret +} diff --git a/drivers/url_tree/util.go b/drivers/url_tree/util.go index 4065218fcc1..61a3fde2c1f 100644 --- a/drivers/url_tree/util.go +++ b/drivers/url_tree/util.go @@ -153,6 +153,9 @@ func splitPath(path string) []string { if path == "/" { return []string{"root"} } + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } parts := strings.Split(path, "/") parts[0] = "root" return parts @@ -190,3 +193,46 @@ func getSizeFromUrl(url string) (int64, error) { } return size, nil } + +func StringifyTree(node *Node) string { + sb := strings.Builder{} + if node.Level == -1 { + for i, child := range node.Children { + sb.WriteString(StringifyTree(child)) + if i < len(node.Children)-1 { + sb.WriteString("\n") + } + } + return sb.String() + } + for i := 0; i < node.Level; i++ { + sb.WriteString(" ") + } + if node.Url == "" { + sb.WriteString(node.Name) + sb.WriteString(":") + for _, child := range node.Children { + sb.WriteString("\n") + sb.WriteString(StringifyTree(child)) + } + } else if node.Size == 0 && node.Modified == 0 { + if stdpath.Base(node.Url) == node.Name { + sb.WriteString(node.Url) + } else { + sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url)) + } + } else { + sb.WriteString(node.Name) + sb.WriteString(":") + if node.Size != 0 || node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Size, 10)) + sb.WriteString(":") + } + if node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Modified, 10)) + sb.WriteString(":") + } + sb.WriteString(node.Url) + } + return sb.String() +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 781e85325ee..6fd5e8d6f90 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -80,6 +80,13 @@ type Put interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error } +type PutURL interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) error +} + //type WriteResult interface { // MkdirResult // MoveResult @@ -109,6 +116,13 @@ type PutResult interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) } +type PutURLResult interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) +} + type UpdateProgress func(percentage float64) type Progress struct { diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 4158051a8f0..405f96cbe2d 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,8 +2,11 @@ package tool import ( "context" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/task" + "net/url" + "path" "path/filepath" "github.com/alist-org/alist/v3/internal/conf" @@ -30,18 +33,6 @@ type AddURLArgs struct { } func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) { - // get tool - tool, err := Tools.Get(args.Tool) - if err != nil { - return nil, errors.Wrapf(err, "failed get tool") - } - // check tool is ready - if !tool.IsReady() { - // try to init tool - if _, err := tool.Init(); err != nil { - return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) - } - } // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) if err != nil { @@ -63,6 +54,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro return nil, errors.WithStack(errs.NotFolder) } } + // try putting url + if args.Tool == "SimpleHttp" && tryPutUrl(ctx, storage, dstDirActualPath, args.URL) { + return nil, nil + } + + // get tool + tool, err := Tools.Get(args.Tool) + if err != nil { + return nil, errors.Wrapf(err, "failed get tool") + } + // check tool is ready + if !tool.IsReady() { + // try to init tool + if _, err := tool.Init(); err != nil { + return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) + } + } uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) @@ -98,3 +106,18 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro DownloadTaskManager.Add(t) return t, nil } + +func tryPutUrl(ctx context.Context, storage driver.Driver, dstDirActualPath, urlStr string) bool { + _, ok := storage.(driver.PutURL) + _, okResult := storage.(driver.PutURLResult) + if !ok && !okResult { + return false + } + u, err := url.Parse(urlStr) + if err != nil { + return false + } + dstName := path.Base(u.Path) + err = op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) + return err == nil +} diff --git a/internal/op/fs.go b/internal/op/fs.go index e49c941a62f..01727e7598c 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -586,3 +586,43 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } return errors.WithStack(err) } + +func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + _, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName)) + if err == nil { + return errors.New("obj already exists") + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + switch s := storage.(type) { + case driver.PutURLResult: + var newObj model.Obj + newObj, err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.PutURL: + err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errs.NotImplement + } + log.Debugf("put url [%s](%s) done", dstName, url) + return errors.WithStack(err) +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 9e26030a04d..c7b7af76c10 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -145,7 +145,9 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 500) return } - tasks = append(tasks, t) + if t != nil { + tasks = append(tasks, t) + } } common.SuccessResp(c, gin.H{ "tasks": getTaskInfos(tasks),