url harvester concurrency issue, ConcurrentModificationException

Question

Hi I'm trying to do a recursive .pdf url harvest and I'm getting a ConcurrentModificationException.. I don't understand how this is happening, and I don't know much about concurrency; I would greatly appreciate some insight towards how this is occurring and how it can be fixed.

public class urlHarvester {
    private URL rootURL;
    private String fileExt;
    private int depth;
    private HashSet<String> targets;
    private HashMap<Integer, LinkedList<String>> toVisit;

public urlHarvester(URL rootURL, String fileExt, int depth) {
    this.rootURL = rootURL;
    this.fileExt = fileExt;
    this.depth = depth;
    targets = new HashSet<String>();
    toVisit = new HashMap<Integer, LinkedList<String>>();
    for (int i = 1; i < depth + 1; i++) {
        toVisit.put(i, new LinkedList<String>());
    }
    doHarvest();
}

private void doHarvest() {
    try {
        harvest(rootURL, depth);
        while (depth > 0) {
            for (String s : toVisit.get(depth)) {
                toVisit.get(depth).remove(s);
                harvest(new URL(s),depth-1);
            }
            depth--;
        }   
    } catch (Exception e) {
        System.err.println(e);
        e.printStackTrace();
    }   
    for (String s : targets) {
        System.out.println(s);
    }

}

private void harvest(URL url, int depth) {
    try {
        URLConnection urlConnection = url.openConnection();
        InputStream inputStream = urlConnection.getInputStream();
        Scanner scanner = new Scanner(new BufferedInputStream(inputStream));
        java.lang.String source = "";
        while (scanner.hasNext()) {
            source = source + scanner.next();
        }   
        inputStream.close();
        scanner.close();

        Matcher matcher = Pattern.compile("ahref=\"(.+?)\"").matcher(source);
        while(matcher.find()) {
            java.lang.String matched = matcher.group(1);
            if (!matched.startsWith("http")) {
                if (matched.startsWith("/") && url.toString().endsWith("/")) {
                    matched = url.toString() + matched.substring(1);
                } else if ((matched.startsWith("/") && !url.toString().endsWith("/"))
                        || (!matched.startsWith("/") && url.toString().endsWith("/"))) {
                    matched = url.toString() + matched;
                } else if (!matched.startsWith("/") && !url.toString().endsWith("/")) {
                    matched = url.toString() + "/" + matched;
                }
            }
            if (matched.endsWith(".pdf") && !targets.contains(matched)) {
                targets.add(matched);System.out.println("ADDED");
            }
            if (!toVisit.get(depth).contains(matched)) {
                toVisit.get(depth).add(matched);
            }
        }
    } catch (Exception e) {
        System.err.println(e);
    }
}

class with main calls:

urlHarvester harvester = new urlHarvester(new URL("http://anyasdf.com"), ".pdf", 5);

Answer 1

该错误可能与并发无关,但是由此循环引起:

for (String s : toVisit.get(depth)) {
    toVisit.get(depth).remove(s);
    harvest(new URL(s),depth-1);
}

要在迭代时从集合中删除项,您需要使用remove迭代器中的方法:

List<String> list = toVisit.get(depth); //I assume list is not null
for (Iterator<String> it = list.iterator(); it.hasNext();) {
    String s = it.next();
    it.remove();
    harvest(new URL(s),depth-1);
}