如何使用多线程扫描目录

Question

我有一个单线程的目录扫描程序.扫描文件时,我必须读取属性信息并将其插入数据库.

我有两个问题.为了提高性能:

1. 如何用多线程扫描？(扫描Android手机的SD卡)
2. 如何优化批量插入到DB？

以下是代码清单:

void scan() {
    File file = new File("/mnt/sdcard");
    fun(file);
}

void fun(File file) {
    if (!file.exists()) {
        return;
    }
    if (!file.isDirectory()) {
        // read attribute information and insert to db
        return;
    } else {
        File[] arr = file.listFiles();
        for (int i = 0; i < arr.length; i++) {
            fun(arr[i]);
        }
    }
}

Answer 1

我认为使用多线程不会在这里有所帮助。扫描目录受IO限制。即使您使用多线程，它们都将等待IO操作在工作线程中完成。因此，在任何时候都只有一个线程扫描。

除非可以并行执行目录上的IO操作（例如，多个磁盘），否则将很有帮助。

Answer 2

是的，你可以使用多线程来提高性能，一个是进行磁盘 I/O，另一个是进行网络 I/O，我将写一个小例子。

这是示例，最好在睡觉前阅读:) 使用该类的构造函数ReadThenAll(5);创建 5 个线程来探索文件夹和子文件夹。玩得开心 ！！

package foo;

import java.io.File;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;

public class ReadThenAll {

    // subfolders to explore
    private final Queue exploreList = new ConcurrentLinkedQueue();

    private long counter = 0;

    public void count() {
        counter++;
    }

    public static void main(String[] args) {

        ReadThenAll me = new ReadThenAll(5);
        me.scan("/tmp");

    }

    int[] threads;

    public ReadThenAll(int numberOfThreads) {
        threads = new int[numberOfThreads];

        for (int i = 0; i < threads.length; i++) {
            threads[i] = -1;
        }
    }

    void scan(String fileName) {

        final long start = System.currentTimeMillis();

        // add the first one to the list
        File file = new File(fileName);
        exploreList.add(file);

        for (int i = 0; i < threads.length; i++) {
            FileExplorer explorer = new FileExplorer(i, this);
            Thread t = new Thread(explorer);
            t.start();
        }

        Thread waitToFinish = new Thread(new Runnable() {

            @Override
            public void run() {

                boolean working = true;
                while (working) {
                    working = false;

                    for (int i = 0; i < threads.length; i++) {
                        if (threads[i] == -1) {
                            working = true;
                            break;
                        }
                    }

                    try {
                        Thread.sleep(2);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }

                long elapsed = System.currentTimeMillis() - start;
                System.out.println("total time (ms) : " + elapsed);

            }
        });

        waitToFinish.start();
    }

    public void done(int id, int counter) {
        threads[id] = counter;
    }

    class FileExplorer implements Runnable {

        public int counter = 0;
        public ReadThenAll owner;
        private int id;

        public FileExplorer(int id, ReadThenAll owner) {
            this.id = id;
            this.owner = owner;
        }

        @Override
        public void run() {
            while (!owner.exploreList.isEmpty()) {

                // get the first from the list
                try {
                    File file = (File) owner.exploreList.remove();

                    if (file.exists()) {

                        if (!file.isDirectory()) {
                            doThemagic(file);
                        } else {

                            // add the files to the queue
                            File[] arr = file.listFiles();
                            if (arr != null) {
                                for (int i = 0; i < arr.length; i++) {
                                    owner.exploreList.add(arr[i]);
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    // silent kill :)
                }

                try {
                    Thread.sleep(1);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }

            owner.done(id, counter);
            System.out.println("total of files : " + counter);
        }

        private void doThemagic(File file) {
            System.out.println(file.toString());
            counter++;
        }
    }

}