Golang上传整个dir concurent返回许多打开的文件

Question

我正在尝试将整个目录上传到服务器.它适用于小目录,但100 + +图片它返回错误"到许多打开文件".我从中读取后立即关闭文件.知道如何解决这个问题吗？

这是我的代码

    func uploadDir(path string) error {
    dir, err := os.Open(path)
    if err != nil {
        return err
    }

    files, err := dir.Readdirnames(-1)
    if err != nil {
        return err
    }
    dir.Close()

    errChan := make(chan error)
    resChan := make(chan *client.PutResult)
    remaining := len(files)
    for _, file := range files {
        file := file
        go func() {
            file, err := os.Open(path + "/" + file)
            if err != nil {
                errChan <- err
            }
            c := client.NewClient(os.Getenv("DROPS_SERVER"))
            res, err := c.Upload(client.NewUploadHandleFromReader(file))
            file.Close()
            if err != nil {
                errChan <- err
            }
            resChan <- res
        }()
    }

    for {
        select {
        case res := <-resChan:
            log.Println(res)
            remaining--
        case err := <-errChan:
            if err != nil {
                return err
            }
        }
        if remaining == 0 {
            break
        }
    }
    return nil
}

Answer 1

原始代码不限制活动go例程的数量,因此不限制打开文件描述符的数量.多个操作系统对打开文件描述符的数量有限制.解决方法是创建固定数量的worker go例程.

func uploadDir(path string) error {

    // Read directory and close.

    dir, err := os.Open(path)
    if err != nil {
        return err
    }
    names, err := dir.Readdirnames(-1)
    if err != nil {
        return err
    }
    dir.Close()

    // Copy names to a channel for workers to consume. Close the
    // channel so that workers stop when all work is complete.

    namesChan := make(chan string, len(names))
    for _, name := range names {
        namesChan <- name
    }
    close(namesChan)

    // Create a maximum of 8 workers

    workers := 8
    if len(names) < workers {
        workers = len(names)
    }

    errChan := make(chan error, 1)
    resChan := make(chan *client.PutResult, len(names))

    // Run workers

    for i := 0; i < workers; i++ {
        go func() {
            // Consume work from namesChan. Loop will end when no more work.
            for name := range namesChan {
                file, err := os.Open(filepath.Join(path, name))
                if err != nil {
                    select {
                    case errChan <- err:
                        // will break parent goroutine out of loop
                    default:
                       // don't care, first error wins
                    }
                    return
                }
                c := client.NewClient(os.Getenv("DROPS_SERVER"))
                res, err := c.Upload(client.NewUploadHandleFromReader(file))
                file.Close()
                if err != nil {
                    select {
                    case errChan <- err:
                        // will break parent goroutine out of loop
                    default:
                       // don't care, first error wins
                    }
                    return
                }
                resChan <- res
            }
        }()
    }

    // Collect results from workers 

    for i := 0; i < len(names); i++ {
        select {
        case res := <-resChan:
            log.Println(res)
        case err := <-errChan:
            return err
        }
    }
    return nil
}

作为奖励,我修改了通道大小并发送操作,以便在出现错误时不会卡住goroutine.