StandardTokenizer行为

Question

鉴于代码在Lucene 3.0.1下运行

import java.io.*;
import org.apache.lucene.analysis.*;
import org.apache.lucene.util.Version;

public class MyAnalyzer extends Analyzer {

   public TokenStream tokenStream(String fieldName, Reader reader) {
       return
               new StopFilter(
                       true,
                       new StandardTokenizer(Version.LUCENE_30, reader),
                       StopAnalyzer.ENGLISH_STOP_WORDS_SET
               );
   }

   private static void printTokens(String string) throws IOException {
       TokenStream ts = new MyAnalyzer().tokenStream("default", new
StringReader(string));
       TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
       while(ts.incrementToken()) {
           System.out.print(termAtt.term());
           System.out.print(" ");
       }
       System.out.println();
   }

   public static void main(String[] args) throws IOException {
       printTokens("one_two_three");           // prints "one two three"
       printTokens("four4_five5_six6");        // prints "four4_five5_six6"
       printTokens("seven7_eight_nine");       // prints "seven7_eight nine"
       printTokens("ten_eleven11_twelve");     // prints "ten_eleven11_twelve"
   }
}

我可以理解为什么one_two_three和four4_five5_six6它们被标记化,因为这在StandardTokenizer类头Javadoc中有解释.但另外两个案例更加微妙,我不太清楚我是否明白了.

Q1:如果7after的外观seven使它与联合令牌eight 分开nine,为什么ten粘在上面eleven11？

Q2:是否有任何标准和/或简单的方法可以StandardTokenizer总是在下划线上分开？

Answer 1

这是一个有趣的发现。我不太确定如何解释为什么第一季度这样做。不过，我可以提供代码来拆分 Q2 的剩余下划线：

public class MyAnalyzer extends Analyzer {
    public TokenStream tokenStream(String fieldName, Reader reader) {
        StandardTokenizer tokenizer = new StandardTokenizer(
                Version.LUCENE_30, reader);
        TokenStream tokenStream = new StandardFilter(tokenizer);
        tokenStream = new MyTokenFilter(tokenStream);
        tokenStream = new StopFilter(true, tokenStream,
                StopAnalyzer.ENGLISH_STOP_WORDS_SET);
        return tokenStream;
    }
}

public class MyTokenFilter extends TokenFilter {
    private final TermAttribute termAttr;
    private String[] terms;
    private int pos;

    public MyTokenFilter(TokenStream tokenStream) {
        super(tokenStream);
        this.termAttr = input.addAttribute(TermAttribute.class);
    }

    public boolean incrementToken() throws IOException {
        if (terms == null) {
            if (!input.incrementToken()) {
                return false;
            }
            terms = termAttr.term().split("_");
        }

        termAttr.setTermBuffer(terms[pos++]);
        if (pos == terms.length) {
            terms = null;
            pos = 0;
        }
        return true;
    }
}