Lucene代码分析14

import org.apache.lucene.analysis.Token;

import org.apache.lucene.analysis.tokenattributes.TermAttribute;

%% //以上是用户代码部分，以下是选项及词法声明

%class StandardTokenizerImpl //类名

%unicode

%integer //下面函数的返回值

%function getNextToken //进行词法及语法分析的函数

%pack

%char

%{ //此之间的代码之间拷贝到生成的java文件中

public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;

public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;

public static final int ACRONYM           = StandardTokenizer.ACRONYM;

public static final int COMPANY           = StandardTokenizer.COMPANY;

public static final int EMAIL             = StandardTokenizer.EMAIL;

public static final int HOST              = StandardTokenizer.HOST;

public static final int NUM               = StandardTokenizer.NUM;

public static final int CJ                = StandardTokenizer.CJ;

public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;

public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;

public final int yychar()

{

    return yychar;

}

final void getText(Token t) {

  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);

}

final void getText(TermAttribu