/*******************************************************************************
 * Copyright (c) 2004 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/
/* The following code was generated by JFlex 1.2.2 on 9/23/04 1:31 AM */

/*nlsXXX*/
package org.eclipse.wst.xml.core.internal.parser;

import java.io.CharArrayReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.eclipse.wst.sse.core.parser.BlockMarker;
import org.eclipse.wst.sse.core.parser.BlockTokenizer;
import org.eclipse.wst.sse.core.text.ITextRegion;
import org.eclipse.wst.sse.core.util.Debug;
import org.eclipse.wst.sse.core.util.StringUtils;
import org.eclipse.wst.xml.core.internal.Logger;
import org.eclipse.wst.xml.core.internal.parser.regions.XMLParserRegionFactory;
import org.eclipse.wst.xml.core.parser.XMLRegionContext;


/**
 * This class is a scanner generated by <a
 * href="http://www.informatik.tu-muenchen.de/~kleing/jflex/">JFlex </a> 1.2.2
 * on 9/23/04 1:31 AM from the specification file
 * <tt>file:/D:/eclipse.rad/workspace.rad/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLTokenizer.jflex</tt>
 */
public class XMLTokenizer implements BlockTokenizer, XMLRegionContext {

	/** this character denotes the end of file */
	final public static int YYEOF = -1;

	/** lexical states */
	final public static int ST_XML_DOCTYPE_EXTERNAL_ID = 23;
	final public static int ST_XML_ELEMENT_DECLARATION_CONTENT = 27;
	final public static int ST_DHTML_ATTRIBUTE_NAME = 12;
	final public static int ST_XML_PI_TAG_CLOSE = 11;
	final public static int ST_XML_DECLARATION_CLOSE = 21;
	final public static int ST_XML_PI_ATTRIBUTE_VALUE = 10;
	final public static int ST_DHTML_EQUALS = 13;
	final public static int ST_XML_TAG_NAME = 16;
	final public static int ST_XML_ATTRIBUTE_VALUE = 19;
	final public static int ST_DHTML_ATTRIBUTE_VALUE = 14;
	final public static int ST_XML_DOCTYPE_ID_SYSTEM = 25;
	final public static int ST_XML_ATTRIBUTE_NAME = 17;
	final public static int ST_XML_ELEMENT_DECLARATION = 26;
	final public static int ST_XML_DOCTYPE_DECLARATION = 22;
	final public static int ST_XML_ATTLIST_DECLARATION = 28;
	final public static int ST_XML_COMMENT_END = 4;
	final public static int ST_CDATA_TEXT = 1;
	final public static int ST_DHTML_TAG_CLOSE = 15;
	final public static int ST_XML_COMMENT = 3;
	final public static int ST_PI_CONTENT = 7;
	final public static int ST_PI_WS = 6;
	final public static int ST_CDATA_END = 2;
	final public static int ST_XML_ATTLIST_DECLARATION_CONTENT = 29;
	final public static int ST_BLOCK_TAG_SCAN = 30;
	final public static int ST_XML_PI_EQUALS = 9;
	final public static int ST_XML_DECLARATION = 20;
	final public static int YYINITIAL = 0;
	final public static int ST_XML_DOCTYPE_ID_PUBLIC = 24;
	final public static int ST_XML_EQUALS = 18;
	final public static int ST_PI = 5;
	final public static int ST_XML_PI_ATTRIBUTE_NAME = 8;

	/**
	 * Translates characters to character classes
	 */
	final private static String yycmap_packed = "\11\0\1\5\1\22\2\0\1\14\22\0\1\14\1\21\1\11\1\51" + "\1\16\1\17\1\12\1\13\1\16\1\16\1\16\1\16\1\16\1\7" + "\1\6\1\3\12\15\1\10\1\54\1\1\1\43\1\2\1\4\1\16" + "\1\32\1\55\1\30\1\31\1\35\1\52\1\34\1\34\1\40\1\34" + "\1\34\1\26\1\25\1\42\1\41\1\45\1\34\1\36\1\37\1\33" + "\1\53\2\34\1\23\1\44\1\34\1\27\1\0\1\20\1\0\1\10" + "\1\0\1\47\1\55\1\56\1\50\1\35\1\52\1\34\1\34\1\40" + "\2\34\1\26\1\25\1\42\1\41\1\45\1\34\1\36\1\37\1\46" + "\1\53\1\34\1\34\1\24\1\44\1\34\1\0\1\0\72\0\1\60" + "\10\0\27\57\1\0\37\57\1\0\72\57\2\0\13\57\2\0\10\57" + "\1\0\65\57\1\0\104\57\11\0\44\57\3\0\2\57\4\0\36\57" + "\70\0\131\57\22\0\7\57\16\0\2\60\56\0\106\60\32\0\2\60" + "\44\0\1\57\1\60\3\57\1\0\1\57\1\0\24\57\1\0\54\57" + "\1\0\7\57\3\0\1\57\1\0\1\57\1\0\1\57\1\0\1\57" + "\1\0\22\57\15\0\14\57\1\0\102\57\1\0\14\57\1\0\44\57" + "\1\0\4\60\11\0\65\57\2\0\2\57\2\0\2\57\3\0\34\57" + "\2\0\10\57\2\0\2\57\67\0\46\57\2\0\1\57\7\0\46\57"
				+ "\12\0\21\60\1\0\27\60\1\0\3\60\1\0\1\60\1\0\2\60" + "\1\0\1\60\13\0\33\57\5\0\3\57\56\0\32\57\5\0\1\60" + "\12\57\10\60\15\0\12\60\6\0\1\60\107\57\2\0\5\57\1\0" + "\17\57\1\0\4\57\1\0\1\57\17\60\2\57\2\60\1\0\4\60" + "\2\0\12\60\u0207\0\3\60\1\0\65\57\2\0\1\60\1\57\20\60" + "\3\0\4\60\3\0\12\57\2\60\2\0\12\60\21\0\3\60\1\0" + "\10\57\2\0\2\57\2\0\26\57\1\0\7\57\1\0\1\57\3\0" + "\4\57\2\0\1\60\1\0\7\60\2\0\2\60\2\0\3\60\11\0" + "\1\60\4\0\2\57\1\0\3\57\2\60\2\0\12\60\2\57\20\0" + "\1\60\2\0\6\57\4\0\2\57\2\0\26\57\1\0\7\57\1\0" + "\2\57\1\0\2\57\1\0\2\57\2\0\1\60\1\0\5\60\4\0" + "\2\60\2\0\3\60\13\0\4\57\1\0\1\57\7\0\12\60\2\60" + "\3\57\14\0\3\60\1\0\7\57\1\0\1\57\1\0\3\57\1\0" + "\26\57\1\0\7\57\1\0\2\57\1\0\5\57\2\0\1\60\1\57" + "\10\60\1\0\3\60\1\0\3\60\22\0\1\57\5\0\12\60\21\0" + "\3\60\1\0\10\57\2\0\2\57\2\0\26\57\1\0\7\57\1\0" + "\2\57\2\0\4\57\2\0\1\60\1\57\6\60\3\0\2\60\2\0" + "\3\60\10\0\2\60\4\0\2\57\1\0\3\57\4\0\12\60\22\0"
				+ "\2\60\1\0\6\57\3\0\3\57\1\0\4\57\3\0\2\57\1\0" + "\1\57\1\0\2\57\3\0\2\57\3\0\3\57\3\0\10\57\1\0" + "\3\57\4\0\5\60\3\0\3\60\1\0\4\60\11\0\1\60\17\0" + "\11\60\21\0\3\60\1\0\10\57\1\0\3\57\1\0\27\57\1\0" + "\12\57\1\0\5\57\4\0\7\60\1\0\3\60\1\0\4\60\7\0" + "\2\60\11\0\2\57\4\0\12\60\22\0\2\60\1\0\10\57\1\0" + "\3\57\1\0\27\57\1\0\12\57\1\0\5\57\4\0\7\60\1\0" + "\3\60\1\0\4\60\7\0\2\60\7\0\1\57\1\0\2\57\4\0" + "\12\60\22\0\2\60\1\0\10\57\1\0\3\57\1\0\27\57\1\0" + "\20\57\4\0\6\60\2\0\3\60\1\0\4\60\11\0\1\60\10\0" + "\2\57\4\0\12\60\221\0\56\57\1\0\1\57\1\60\2\57\7\60" + "\5\0\6\57\1\60\10\60\1\0\12\60\47\0\2\57\1\0\1\57" + "\2\0\2\57\1\0\1\57\2\0\1\57\6\0\4\57\1\0\7\57" + "\1\0\3\57\1\0\1\57\1\0\1\57\2\0\2\57\1\0\2\57" + "\1\0\1\57\1\60\2\57\6\60\1\0\2\60\1\57\2\0\5\57" + "\1\0\1\60\1\0\6\60\2\0\12\60\76\0\2\60\6\0\12\60" + "\13\0\1\60\1\0\1\60\1\0\1\60\4\0\2\60\10\57\1\0" + "\41\57\7\0\24\60\1\0\6\60\4\0\6\60\1\0\1\60\1\0"
				+ "\25\60\3\0\7\60\1\0\1\60\346\0\46\57\12\0\47\57\11\0" + "\1\57\1\0\2\57\1\0\3\57\1\0\1\57\1\0\2\57\1\0" + "\5\57\51\0\1\57\1\0\1\57\1\0\1\57\13\0\1\57\1\0" + "\1\57\1\0\1\57\3\0\2\57\3\0\1\57\5\0\3\57\1\0" + "\1\57\1\0\1\57\1\0\1\57\1\0\1\57\3\0\2\57\3\0" + "\2\57\1\0\1\57\50\0\1\57\11\0\1\57\2\0\1\57\2\0" + "\2\57\7\0\2\57\1\0\1\57\1\0\7\57\50\0\1\57\4\0" + "\1\57\10\0\1\57\u0c06\0\234\57\4\0\132\57\6\0\26\57\2\0" + "\6\57\2\0\46\57\2\0\6\57\2\0\10\57\1\0\1\57\1\0" + "\1\57\1\0\1\57\1\0\37\57\2\0\65\57\1\0\7\57\1\0" + "\1\57\3\0\3\57\1\0\7\57\3\0\4\57\2\0\6\57\4\0" + "\15\57\5\0\3\57\1\0\7\57\323\0\15\60\4\0\1\60\104\0" + "\1\57\3\0\2\57\2\0\1\57\121\0\3\57\u0e82\0\1\60\1\0" + "\1\57\31\0\11\57\6\60\1\0\5\60\13\0\124\57\4\0\2\60" + "\2\0\2\60\2\0\132\57\1\0\3\60\6\0\50\57\u1cd3\0\u51a6\57" + "\u0c5a\0\u2ba4\57\134\0\u0800\0\u1ffe\0\2\0";

	/**
	 * Translates characters to character classes
	 */
	final private static char[] yycmap = yy_unpack_cmap(yycmap_packed);

	/**
	 * Translates a state to a row index in the transition table
	 */
	final private static int yy_rowMap[] = {0, 49, 98, 147, 196, 245, 294, 343, 392, 441, 490, 539, 588, 637, 686, 735, 784, 833, 882, 931, 980, 1029, 1078, 1127, 1176, 1225, 1274, 1323, 1372, 1421, 1470, 1519, 1568, 1617, 1666, 1715, 1764, 1715, 1764, 1813, 1715, 1715, 1764, 1862, 1911, 1960, 2009, 2058, 2107, 2156, 1715, 1764, 2205, 2254, 2303, 1715, 2352, 2352, 2401, 2450, 2499, 2205, 1715, 2548, 2597, 1715, 2646, 2695, 2744, 2793, 2842, 2891, 1715, 2940, 2989, 3038, 3087, 1715, 3136, 3185, 3234, 3283, 3332, 1715, 3381, 3430, 3479, 3528, 3577, 3626, 3675, 3724, 3724, 3773, 3822, 3871, 3920, 3920, 3969, 4018, 4067, 4116, 4116, 4165, 4214, 4263, 4312, 1715, 4361, 4361, 4410, 4459, 4508, 4557, 1715, 1715, 1764, 1715, 1715, 4606, 4655, 4704, 4753, 4802, 4851, 4900, 4949, 1715, 4998, 5047, 1715, 1715, 2352, 5096, 2450, 1715, 5145, 2499, 2548, 2646, 2695, 5194, 2744, 1715, 5243, 2793, 1715, 3136, 5292, 3234, 1715, 5341, 3283, 4606, 5390, 5439, 5488, 3528, 1715, 5537, 5586, 3724, 5635,
				3773, 1715, 5684, 5733, 5782, 5782, 5831, 5880, 3871, 3724, 3920, 5929, 3969, 1715, 5978, 4018, 4067, 3920, 4116, 6027, 4165, 1715, 6076, 6125, 6174, 6174, 6223, 6272, 6321, 4361, 6370, 4410, 1715, 6419, 6468, 6517, 6517, 6566, 6615, 6664, 6713, 6762, 6811, 6860, 1715, 6909, 6958, 1715, 1715, 1715, 2009, 7007, 7056, 7105, 7154, 7203, 7252, 5684, 7301, 7301, 6076, 7350, 7350, 7399, 6419, 7448, 7448, 7497, 1715, 7546, 7595, 1715, 7644, 7693, 7742, 7791, 7840, 7889, 7938, 5831, 6223, 7987, 6566, 8036, 8085, 8134, 8183, 8232, 8281, 8330, 8379, 8428, 8477, 8526, 2009, 8575, 8624, 8673, 1715, 1715, 8722, 8771, 8820, 1715, 1715, 1715, 8869, 8918, 8967, 9016, 9065, 1715, 4263, 4508};

	/**
	 * The packed transition table of the DFA
	 */
	final private static String yy_packed = "\1\40\1\41\10\40\1\42\4\40\1\43\41\40\1\44" + "\1\45\57\44\1\46\1\47\16\46\1\50\1\46\1\51" + "\36\46\1\52\1\53\57\52\1\46\1\47\5\46\1\54" + "\12\46\1\51\37\46\1\47\2\46\1\55\1\56\2\46" + "\1\57\3\46\1\56\5\46\1\56\2\60\2\57\1\46" + "\10\57\1\61\2\57\1\46\5\57\1\46\2\57\1\46" + "\3\57\2\46\1\47\2\46\1\55\1\62\6\46\1\62" + "\5\46\1\62\36\46\1\63\1\64\2\63\1\65\15\63" + "\1\51\36\63\1\46\1\47\2\46\1\66\1\56\2\46" + "\1\67\3\46\1\56\5\46\1\56\4\67\1\46\13\67" + "\1\46\5\67\1\46\2\67\1\46\3\67\2\46\1\47" + "\2\46\1\66\1\56\2\46\1\67\3\46\1\56\5\46" + "\1\56\4\67\1\46\13\67\1\70\5\67\1\46\2\67" + "\1\46\3\67\1\46\1\71\1\47\1\46\1\72\1\73" + "\1\56\3\71\1\74\1\71\1\75\1\56\5\71\1\56" + "\36\71\1\46\1\47\2\46\1\76\15\46\1\51\37\46" + "\1\47\1\77\1\100\1\46\1\56\2\46\1\101\3\46" + "\1\56\5\46\1\56\4\101\1\46\13\101\1\46\5\101" + "\1\46\2\101\1\46\3\101\2\46\1\47\1\77\1\100" + "\1\46\1\56\2\46\1\101\3\46\1\56\5\46\1\56"
				+ "\4\101\1\46\13\101\1\102\5\101\1\46\2\101\1\46" + "\3\101\1\46\1\103\1\47\1\77\1\104\1\103\1\56" + "\3\103\1\105\1\103\1\106\1\56\5\103\1\56\36\103" + "\1\46\1\47\3\46\1\56\6\46\1\56\5\46\1\56" + "\36\46\1\107\1\110\1\111\1\112\4\107\1\113\12\107" + "\4\114\1\107\13\114\1\107\5\114\1\107\2\114\1\107" + "\3\114\1\107\1\46\1\110\1\111\1\112\1\46\1\56" + "\2\46\1\115\3\46\1\56\5\46\1\56\4\115\1\46" + "\13\115\1\46\5\115\1\46\2\115\1\46\3\115\2\46" + "\1\110\1\111\1\112\1\46\1\56\2\46\1\115\3\46" + "\1\56\5\46\1\56\4\115\1\46\13\115\1\116\5\115" + "\1\46\2\115\1\46\3\115\1\46\1\117\1\110\1\111" + "\1\120\1\117\1\56\3\117\1\121\1\117\1\122\1\56" + "\5\117\1\56\36\117\1\46\1\123\1\124\2\46\1\56" + "\6\46\1\56\5\46\1\56\6\46\1\125\1\126\2\46" + "\1\127\11\46\1\126\1\125\11\46\1\47\1\124\2\46" + "\1\56\6\46\1\56\5\46\1\56\4\46\1\130\32\46" + "\1\47\1\124\2\46\1\56\2\46\1\131\3\46\1\56" + "\5\46\1\56\4\131\1\130\13\131\1\46\5\131\1\46"
				+ "\2\131\1\46\3\131\2\46\1\47\1\124\2\46\1\56" + "\6\46\1\56\5\46\1\56\4\46\1\130\7\46\1\132" + "\5\46\1\133\13\46\1\134\1\47\1\124\1\135\1\134" + "\1\56\3\134\1\136\1\134\1\137\1\56\5\134\1\56" + "\4\134\1\140\31\134\1\141\1\47\1\124\1\142\1\141" + "\1\56\3\141\1\143\1\141\1\144\1\56\5\141\1\56" + "\4\141\1\145\31\141\1\146\1\47\1\124\1\147\1\146" + "\1\56\3\146\1\150\1\146\1\151\1\56\5\146\1\56" + "\36\146\1\152\1\153\1\154\56\152\1\155\1\47\1\124" + "\1\156\1\155\1\56\3\155\1\157\1\155\1\160\1\56" + "\5\155\1\56\36\155\1\161\1\162\1\163\56\161\1\164" + "\1\165\57\164\1\40\1\0\10\40\1\0\4\40\1\0" + "\41\40\3\0\1\166\1\167\14\0\1\170\44\0\1\171" + "\2\0\1\172\3\0\1\171\5\0\1\171\4\172\1\0" + "\13\172\1\0\5\172\1\173\2\172\1\0\3\172\6\0" + "\1\171\2\0\1\174\3\0\1\171\5\0\1\171\4\174" + "\1\0\13\174\1\0\5\174\1\0\2\174\1\0\3\174" + "\103\0\1\175\57\0\1\176\47\0\1\177\53\0\1\200" + "\63\0\1\56\6\0\1\56\5\0\1\56\44\0\3\57" + "\4\0\1\57\5\0\4\57\1\0\13\57\1\0\5\57"
				+ "\1\0\2\57\1\0\4\57\6\0\3\57\4\0\1\57" + "\5\0\2\57\1\201\1\57\1\0\13\57\1\0\5\57" + "\1\0\2\57\1\0\4\57\6\0\3\57\4\0\1\57" + "\5\0\2\57\1\202\1\57\1\0\13\57\1\0\5\57" + "\1\0\2\57\1\0\4\57\5\0\1\62\6\0\1\62" + "\5\0\1\62\40\0\1\203\60\0\1\204\64\0\3\67" + "\4\0\1\67\5\0\4\67\1\0\13\67\1\0\5\67" + "\1\0\2\67\1\0\4\67\1\71\2\0\1\205\1\71" + "\1\0\3\71\1\0\1\71\2\0\5\71\1\0\37\71" + "\1\0\1\204\1\205\1\71\1\0\3\71\1\0\1\71" + "\2\0\5\71\1\0\36\71\1\74\1\0\1\206\1\207" + "\1\74\1\206\3\74\1\210\1\74\2\206\5\74\1\206" + "\36\74\1\75\1\0\1\211\1\212\1\75\1\211\3\75" + "\1\211\1\75\1\210\1\211\5\75\1\211\36\75\2\0" + "\1\77\1\213\63\0\3\101\4\0\1\101\5\0\4\101" + "\1\0\13\101\1\0\5\101\1\0\2\101\1\0\4\101" + "\1\103\2\0\1\214\1\103\1\0\3\103\1\0\1\103" + "\2\0\5\103\1\0\37\103\1\0\1\77\1\215\1\103" + "\1\0\3\103\1\0\1\103\2\0\5\103\1\0\36\103" + "\1\105\1\0\1\216\1\217\1\105\1\216\3\105\1\220" + "\1\105\2\216\5\105\1\216\36\105\1\106\1\0\1\221"
				+ "\1\222\1\106\1\221\3\106\1\221\1\106\1\220\1\221" + "\5\106\1\221\36\106\1\107\3\0\17\107\4\0\1\107" + "\13\0\1\107\5\0\1\107\2\0\1\107\3\0\1\107" + "\3\0\1\166\15\0\1\170\41\0\1\223\56\0\1\107" + "\3\0\2\107\3\113\4\107\1\113\5\107\4\114\1\107" + "\13\114\1\107\5\114\1\107\2\114\1\107\3\114\1\113" + "\6\0\3\114\4\0\1\114\5\0\4\114\1\0\13\114" + "\1\0\5\114\1\0\2\114\1\0\4\114\6\0\3\115" + "\4\0\1\115\5\0\4\115\1\0\13\115\1\0\5\115" + "\1\0\2\115\1\0\4\115\1\117\2\0\1\224\1\117" + "\1\0\3\117\1\0\1\117\2\0\5\117\1\0\37\117" + "\1\0\1\223\1\224\1\117\1\0\3\117\1\0\1\117" + "\2\0\5\117\1\0\36\117\1\121\1\0\1\225\1\226" + "\1\121\1\225\3\121\1\227\1\121\2\225\5\121\1\225" + "\36\121\1\122\1\0\1\230\1\231\1\122\1\230\3\122" + "\1\230\1\122\1\227\1\230\5\122\1\230\36\122\3\0" + "\1\166\15\0\1\232\100\0\1\233\52\0\1\234\12\0" + "\1\234\40\0\1\235\32\0\20\236\1\237\40\236\6\0" + "\3\131\4\0\1\131\5\0\4\131\1\0\13\131\1\0" + "\5\131\1\0\2\131\1\0\4\131\44\0\1\240\67\0"
				+ "\1\241\5\0\1\134\2\0\1\242\1\134\1\0\3\134" + "\1\0\1\134\2\0\5\134\1\0\36\134\1\136\1\0" + "\1\243\1\244\1\136\1\243\3\136\1\245\1\136\2\243" + "\5\136\1\243\36\136\1\246\1\0\1\247\1\250\1\251" + "\1\247\3\251\1\247\1\246\1\252\1\253\3\251\1\246" + "\1\251\1\253\4\251\1\246\27\251\2\246\1\140\2\236" + "\1\254\1\140\1\236\3\140\1\236\1\140\2\236\3\140" + "\1\255\1\140\1\236\36\140\1\141\2\0\1\256\1\141" + "\1\0\3\141\1\0\1\141\2\0\5\141\1\0\36\141" + "\1\143\2\257\1\260\1\143\1\257\3\143\1\261\1\143" + "\2\257\5\143\1\257\36\143\1\144\2\262\1\263\1\144" + "\1\262\3\144\1\262\1\144\1\261\1\262\5\144\1\262" + "\36\144\1\145\2\236\1\264\1\145\1\236\3\145\1\236" + "\1\145\2\236\3\145\1\265\1\145\1\236\36\145\1\146" + "\2\0\1\266\1\146\1\0\3\146\1\0\1\146\2\0" + "\5\146\1\0\36\146\1\150\1\0\1\267\1\270\1\150" + "\1\267\3\150\1\271\1\150\2\267\5\150\1\267\36\150" + "\1\272\1\0\1\273\1\274\1\275\1\273\3\275\1\273" + "\1\272\1\276\1\277\3\275\1\272\1\275\1\277\4\275"
				+ "\1\272\27\275\2\272\2\152\1\0\60\152\1\0\16\152" + "\1\300\37\152\1\155\2\0\1\301\1\155\1\0\3\155" + "\1\0\1\155\2\0\5\155\1\0\36\155\1\157\1\0" + "\1\302\1\303\1\157\1\302\3\157\1\304\1\157\2\302" + "\5\157\1\302\36\157\1\305\1\0\1\306\1\307\1\310" + "\1\306\3\310\1\306\1\305\1\311\1\312\3\310\1\305" + "\1\310\1\312\4\310\1\305\27\310\2\305\2\161\1\0" + "\60\161\1\0\16\161\1\313\37\161\7\0\1\314\17\0" + "\1\315\36\0\1\171\2\0\1\40\3\0\1\171\5\0" + "\1\171\4\40\1\0\13\40\1\0\5\40\1\0\2\40" + "\1\0\3\40\1\0\1\316\1\0\3\316\1\317\3\172" + "\1\316\1\0\1\316\1\317\1\172\1\316\1\0\2\316" + "\1\317\4\172\1\316\13\172\1\316\5\172\1\316\2\172" + "\1\320\4\172\15\0\1\321\6\0\1\322\34\0\1\316" + "\1\0\3\316\1\317\3\174\1\316\1\0\1\316\1\317" + "\1\174\1\316\1\0\2\316\1\317\4\174\1\316\13\174" + "\1\316\5\174\1\316\2\174\1\323\4\174\27\0\1\315" + "\33\0\1\324\60\0\1\325\64\0\3\57\4\0\1\57" + "\5\0\3\57\1\326\1\0\13\57\1\0\5\57\1\0" + "\2\57\1\0\4\57\6\0\3\57\4\0\1\57\5\0"
				+ "\4\57\1\0\13\57\1\0\1\57\1\327\3\57\1\0" + "\2\57\1\0\4\57\1\206\1\0\7\206\1\210\47\206" + "\1\211\1\0\11\211\1\210\45\211\1\216\1\0\7\216" + "\1\220\47\216\1\221\1\0\11\221\1\220\45\221\1\225" + "\1\0\7\225\1\227\47\225\1\230\1\0\11\230\1\227" + "\45\230\30\0\1\330\25\0\1\330\35\0\1\331\12\0" + "\1\331\47\0\1\332\62\0\1\333\76\0\1\334\3\0" + "\1\243\1\0\7\243\1\245\47\243\1\246\1\0\1\247" + "\1\335\1\246\1\247\3\246\1\247\1\246\1\245\1\247" + "\5\246\1\247\36\246\1\247\1\0\11\247\1\245\45\247" + "\1\246\1\0\1\247\1\335\1\246\1\247\3\246\1\247" + "\1\246\1\336\1\247\5\246\1\247\36\246\13\0\1\337" + "\45\0\1\247\1\0\11\247\1\336\45\247\11\257\1\261" + "\47\257\13\262\1\261\45\262\1\267\1\0\7\267\1\271" + "\47\267\1\272\1\0\1\273\1\340\1\272\1\273\3\272" + "\1\273\1\272\1\271\1\273\5\272\1\273\36\272\1\273" + "\1\0\11\273\1\271\45\273\1\272\1\0\1\273\1\340" + "\1\272\1\273\3\272\1\273\1\272\1\341\1\273\5\272" + "\1\273\36\272\13\0\1\342\45\0\1\273\1\0\11\273"
				+ "\1\341\45\273\2\152\1\0\24\152\1\343\31\152\1\302" + "\1\0\7\302\1\304\47\302\1\305\1\0\1\306\1\344" + "\1\305\1\306\3\305\1\306\1\305\1\304\1\306\5\305" + "\1\306\36\305\1\306\1\0\11\306\1\304\45\306\1\305" + "\1\0\1\306\1\344\1\305\1\306\3\305\1\306\1\305" + "\1\345\1\306\5\305\1\306\36\305\13\0\1\346\45\0" + "\1\306\1\0\11\306\1\345\45\306\2\161\1\0\24\161" + "\1\347\31\161\7\0\1\350\101\0\1\351\30\0\1\316" + "\1\0\10\316\1\0\4\316\1\0\34\316\1\0\5\316" + "\1\0\3\316\1\317\4\316\1\0\1\316\1\317\2\316" + "\1\0\2\316\1\317\31\316\1\352\4\316\15\0\1\321" + "\36\0\1\353\21\0\1\354\12\0\3\354\2\0\1\354" + "\11\0\2\354\1\0\1\354\2\0\2\354\10\0\3\57" + "\4\0\1\57\5\0\4\57\1\0\11\57\1\355\1\57" + "\1\0\5\57\1\0\2\57\1\0\4\57\33\0\1\356" + "\12\0\1\356\40\0\1\357\57\0\1\360\66\0\1\361" + "\12\0\1\361\40\0\1\362\35\0\2\363\1\0\3\363" + "\2\0\1\252\4\363\1\0\6\363\1\0\27\363\5\0" + "\2\364\1\0\3\364\2\0\1\276\4\364\1\0\6\364"
				+ "\1\0\27\364\2\0\2\152\1\0\25\152\1\365\30\152" + "\3\0\2\366\1\0\3\366\2\0\1\311\4\366\1\0" + "\6\366\1\0\27\366\2\0\2\161\1\0\25\161\1\367" + "\30\161\31\0\1\370\103\0\1\352\21\0\1\354\12\0" + "\3\354\2\0\1\354\11\0\2\354\1\0\1\354\1\0" + "\1\353\2\354\10\0\3\57\4\0\1\57\5\0\4\57" + "\1\0\6\57\1\371\4\57\1\0\5\57\1\0\2\57" + "\1\0\4\57\44\0\1\372\54\0\1\373\55\0\1\374" + "\60\0\1\375\63\0\1\376\20\0\2\152\1\0\26\152" + "\1\377\27\152\2\161\1\0\26\161\1\u0100\27\161\32\0" + "\1\u0101\34\0\3\57\4\0\1\57\5\0\4\57\1\0" + "\3\57\1\u0102\7\57\1\0\2\57\1\u0102\2\57\1\0" + "\2\57\1\0\4\57\45\0\1\u0103\52\0\1\u0104\63\0" + "\1\u0105\43\0\1\u0106\63\0\1\u0107\25\0\1\u0107\2\0" + "\2\152\1\0\27\152\1\u0108\26\152\2\161\1\0\27\161" + "\1\u0109\26\161\33\0\1\u010a\62\0\1\u010b\56\0\1\u010c" + "\12\0\1\u010c\45\0\1\u010d\12\0\1\u010d\12\0\2\152" + "\1\0\30\152\1\u010e\25\152\2\161\1\0\30\161\1\u010f" + "\25\161\32\0\1\u0110\26\0\2\152\1\0\27\152\1\u0111"
				+ "\26\152\2\161\1\0\27\161\1\u0112\26\161\27\0\1\u0113" + "\31\0\2\152\1\0\24\152\1\u0114\31\152\2\161\1\0" + "\24\161\1\u0115\31\161";

	/**
	 * The transition table of the DFA
	 */
	final private static int yytrans[] = yy_unpack(yy_packed);


	/* error codes */
	final private static int YY_UNKNOWN_ERROR = 0;
	// final private static int YY_ILLEGAL_STATE = 1;
	final private static int YY_NO_MATCH = 2;
	final private static int YY_PUSHBACK_2BIG = 3;

	/* error messages for the codes above */
	final private static String YY_ERROR_MSG[] = {"Unkown internal scanner error", //$NON-NLS-1$
				"Internal error: unknown state", //$NON-NLS-1$
				"Error: could not match input", //$NON-NLS-1$
				"Error: pushback value was too large" //$NON-NLS-1$
	};

	/**
	 * YY_ATTRIBUTE[aState] contains the attributes of state
	 * <code>aState</code>
	 */
	private final static byte YY_ATTRIBUTE[] = {1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 9, 1, 9, 1, 1, 9, 9, 1, 1, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 9, 1, 9, 9, 1, 0, 1, 0, 1, 0, 0, 0, 9, 1, 1, 9, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 9, 0, 0, 9, 9, 9, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 9, 0, 1, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 9, 9, 1, 1, 0, 9, 9, 9, 1, 1, 0, 1, 1, 9, 1, 1};

	/** the input device */
	private java.io.Reader yy_reader;

	/** the current state of the DFA */
	private int yy_state;

	/** the current lexical state */
	private int yy_lexical_state = YYINITIAL;

	/**
	 * this buffer contains the current text to be matched and is the source
	 * of the yytext() string
	 */
	private char yy_buffer[] = new char[16384];

	/** the textposition at the last accepting state */
	private int yy_markedPos;

	/** the textposition at the last state to be included in yytext */
	private int yy_pushbackPos;

	/** the current text position in the buffer */
	private int yy_currentPos;

	/** startRead marks the beginning of the yytext() string in the buffer */
	private int yy_startRead;

	/**
	 * endRead marks the last character in the buffer, that has been read from
	 * input
	 */
	private int yy_endRead;

	/** number of newlines encountered up to the start of the matched text */
	private int yyline;

	/** the number of characters up to the start of the matched text */
	private int yychar;

	/**
	 * the number of characters from the last newline up to the start of the
	 * matched text
	 */
	// private int yycolumn;
	/**
	 * yy_atBOL == true <=>the scanner is currently at the beginning of a line
	 */
	// private boolean yy_atBOL;
	/** yy_atEOF == true <=>the scanner has returned a value for EOF */
	private boolean yy_atEOF;

	/** denotes if the user-EOF-code has already been executed */
	private boolean yy_eof_done;

	/* user code: */
	private int fTokenCount = 0;

	// required holders for white-space compacting
	private boolean fShouldLoadBuffered = false;
	private String fBufferedContext = null;
	private int fBufferedStart = 1;
	private int fBufferedLength = 0;
	private ContextRegionContainer fBufferedEmbeddedContainer = null;
	private String f_context = null;

	// state stack for handling embedded regions
	private IntStack fStateStack = new IntStack();
	// a "hint" as to what an embedded region should be evaluated
	 String fEmbeddedHint = UNDEFINED;
	// a "hint" as to what state to enter once an embedded region has
	//   been completed
	 int fEmbeddedPostState = YYINITIAL;
	// the container used to create embedded regions
	private ContextRegionContainer fEmbeddedContainer = null;
	private static final String PROXY_CONTEXT = "PROXY_CONTEXT";

	private String context = null;
	private int start = 0;
	private int textLength = 0;
	private int length = 0;

	// offset for tracking position specific block tags
	private int fOffset = 0;

	// the name of the current tag being opened
	private String fCurrentTagName = null;

	// the list of tag name BlockMarkers
	private List fBlockMarkers = new ArrayList();

	// required to not seek text blocks on an end tag
	private boolean fIsBlockingEnabled = false;
	private boolean fIsCaseSensitiveBlocking = true;

	private XMLParserRegionFactory fRegionFactory = new XMLParserRegionFactory();

	 static final String rcsver = "$Id: XMLTokenizer.java,v 1.4 2004/12/07 21:46:18 david_williams Exp $";//$NON-NLS-1$

	/**
	 * user method
	 */
	public final void addBlockMarker(BlockMarker marker) {
		if (containsTagName(marker.getTagName()))
			return;
		fBlockMarkers.add(marker);
	}

	/**
	 * user method
	 */
	public final void removeBlockMarker(BlockMarker marker) {
		fBlockMarkers.remove(marker);
	}

	/**
	 * user method
	 */
	public final void removeBlockMarker(String tagname) {
		if (fBlockMarkers != null) {
			Iterator blocks = fBlockMarkers.iterator();
			while (blocks.hasNext()) {
				if (((BlockMarker) blocks.next()).getTagName().equals(tagname))
					blocks.remove();
			}
		}
	}

	/* user method */
	public final boolean isCaseSensitiveBlocking() {
		return fIsCaseSensitiveBlocking;
	}

	/* user method */
	public final void setCaseSensitiveBlocking(boolean newValue) {
		fIsCaseSensitiveBlocking = newValue;
	}

	/* user method */
	public boolean getBlockMarkerCaseSensitivity() {
		return getBlockMarkerCaseSensitivity(fCurrentTagName);
	}

	/* user method */
	public boolean getBlockMarkerCaseSensitivity(String name) {
		Iterator iterator = fBlockMarkers.iterator();
		while (iterator.hasNext()) {
			BlockMarker marker = (BlockMarker) iterator.next();
			boolean casesensitive = marker.isCaseSensitive();
			if (casesensitive && marker.getTagName().equals(name))
				return casesensitive;
			else if (!casesensitive && marker.getTagName().equalsIgnoreCase(name))
				return casesensitive;
		}
		return true;
	}

	/* user method */
	public String getBlockMarkerContext() {
		return getBlockMarkerContext(fCurrentTagName);
	}

	/* user method */
	public String getBlockMarkerContext(String name) {
		Iterator iterator = fBlockMarkers.iterator();
		while (iterator.hasNext()) {
			BlockMarker marker = (BlockMarker) iterator.next();
			if (marker.getTagName().equals(name))
				return marker.getContext();
		}
		return BLOCK_TEXT;
	}

	/* user method */
	public List getBlockMarkers() {
		return fBlockMarkers;
	}

	/* user method */
	public final int getOffset() {
		return fOffset + yychar;
	}

	private final boolean isBlockMarker() {
		return isBlockMarker(fCurrentTagName);
	}

	private final boolean isBlockMarker(String tagName) {
		if (!fIsBlockingEnabled)
			return false;
		return containsTagName(tagName);
	}

	/**
	 * user method
	 */
	public final void beginBlockTagScan(String newTagName) {
		beginBlockMarkerScan(newTagName, BLOCK_TEXT);
	}

	/**
	 * user method
	 * 
	 * Special tokenizer setup. Allows tokenization to be initiated at the
	 * start of a text block within a "newTagName" tag.
	 * 
	 * Example: Tokenizer toker = new Tokenizer();
	 * toker.setCaseSensitiveBlocking(false); toker.reset(new
	 * java.io.StringReader("afiuhqwkejhtasihgalkwhtq </scripter> </scr>
	 * </script>asgdasga")); toker.beginBlockMarkerScan("script", BLOCK_TEXT);
	 * toker.getRegions();
	 * 
	 * Returns: BLOCK_TEXT: 0-40 XML_END_TAG_OPEN: 41-42 XML_TAG_NAME: 43-48
	 * XML_TAG_CLOSE: 49-49 XML_CONTENT: 50-57
	 *  
	 */
	public final void beginBlockMarkerScan(String newTagName, String blockcontext) {
		yybegin(ST_BLOCK_TAG_SCAN);
		fCurrentTagName = newTagName;
	}

	/**
	 * Method doScan.
	 * 
	 * Returns a context region for all of the text from the current position
	 * upto the end of input or to right *before* the first occurence of
	 * searchString
	 * 
	 * @param searchString -
	 *            target string to search for ex.: "-->", " </tagname"
	 * @param requireTailSeparator -
	 *            whether the target must be immediately followed by
	 *            whitespace or '>'
	 * @param context -
	 *            the context of the scanned region if non-zero length
	 * @param exitState -
	 *            the state to go to if the region was of non-zero length
	 * @param abortState -
	 *            the state to go to if the searchString was found immediately
	 * @return String - the context found: the desired context on a non-zero
	 *         length match, the abortContext on immediate success
	 * @throws IOException
	 */
	private final String doScan(String searchString, boolean requireTailSeparator, String searchContext, int exitState, int immediateFallbackState) throws IOException {
		boolean stillSearching = true;
		// Disable further block (probably)
		fIsBlockingEnabled = false;
		int searchStringLength = searchString.length();
		int n = 0;
		char lastCheckChar;
		int i;
		boolean same = false;
		while (stillSearching) {
			n = 0;
			// Ensure that enough data from the input exists to compare
			// against the search String.
			n = yy_advance();
			while (n != YYEOF && yy_currentPos < searchStringLength)
				n = yy_advance();
			// If the input was too short or we've exhausted the input, stop
			// immediately.
			if (n == YYEOF) {
				stillSearching = false;
			} else {
				same = true;
				// Ensure that we've not encountered a complete block (<%%>)
				// that was *shorter* than the closeTagString and
				// thus found twice at current-targetLength [since the first
				// scan would have come out this far anyway].
				// Check the characters in the target versus the last
				// targetLength characters read from the buffer
				// and see if it matches

				// safety check for array accesses (yy_currentPos is the
				// *last* character we can check against)
				if (yy_currentPos >= searchStringLength && yy_currentPos <= yy_buffer.length) {
					for (i = 0; i < searchStringLength; i++) {
						if (same && fIsCaseSensitiveBlocking)
							same = yy_buffer[i + yy_currentPos - searchStringLength] == searchString.charAt(i);
						else if (same && !fIsCaseSensitiveBlocking)
							same = Character.toLowerCase(yy_buffer[i + yy_currentPos - searchStringLength]) == Character.toLowerCase(searchString.charAt(i));
					}
				}
				// safety check failed; no match is possible right now
				else {
					same = false;
				}
				if (same && requireTailSeparator && yy_currentPos < yy_buffer.length) {
					// Additional check for close tags to ensure that
					// targetString="</script" doesn't match
					// "</scriptS"
					lastCheckChar = yy_buffer[yy_currentPos];
					// Succeed on "</script>" and "</script "
					if (lastCheckChar == '>' || Character.isWhitespace(lastCheckChar))
						stillSearching = false;
				} else {
					stillSearching = !same || (yy_currentPos < yy_startRead + searchStringLength);
				}
			}
		}
		if (n != YYEOF || same) {
			// We've stopped short of the end or definitely found a match
			yy_markedPos = yy_currentPos - searchStringLength;
			yy_currentPos = yy_markedPos + 1;
			// If the searchString occurs at the very beginning of what would
			// have
			// been a Block, resume scanning normally immediately
			if (yy_markedPos == yy_startRead) {
				yybegin(immediateFallbackState);
				return primGetNextToken();
			}
		} else {
			// We ran through the rest of the input
			yy_markedPos = yy_currentPos;
			yy_currentPos++;
		}
		yybegin(exitState);
		// If the ending occurs at the very beginning of what would have
		// been a Block, resume scanning normally immediately
		if (yy_markedPos == yy_startRead)
			return primGetNextToken();
		return searchContext;
	}

	/**
	 * user method
	 * 
	 * A generic lookahead-like operation
	 */
	private final String doBlockScan(String target, String targetContext, int immediateFallbackState) throws IOException {
		return doScan(target, false, targetContext, immediateFallbackState, immediateFallbackState);
	}

	/**
	 * user method does a lookahead for the current tag name
	 */
	private final String doBlockTagScan() throws IOException {
		fIsCaseSensitiveBlocking = getBlockMarkerCaseSensitivity();
		return doScan("</" + fCurrentTagName, true, getBlockMarkerContext(fCurrentTagName), YYINITIAL, YYINITIAL);
	}

	/**
	 * user method
	 * 
	 * Converts the raw context String returned by the primGetNextToken()
	 * method into a full ITextRegion by pulling in values for the current
	 * offset within the scanning text.
	 * 
	 * Returns null when EOF is encountered and attaches intermittently
	 * discovered whitespace onto the end of useful regions.
	 * 
	 * Note that this algorithm caches the token following the one being
	 * returned so that whitespace can be collapsed.
	 */
	public final ITextRegion getNextToken() throws IOException {
		fEmbeddedContainer = null;
		// load the starting non-whitespace token (assume that it is so)
		if (fShouldLoadBuffered) {
			if (fBufferedEmbeddedContainer != null) {
				ITextRegion container = fBufferedEmbeddedContainer;
				fBufferedEmbeddedContainer = null;
				fShouldLoadBuffered = false;
				return container;
			}
			context = fBufferedContext;
			start = fBufferedStart;
			textLength = length = fBufferedLength;
			fShouldLoadBuffered = false;
		} else {
			context = primGetNextToken();
			if (context == PROXY_CONTEXT) {
				return fEmbeddedContainer;
			} else if (context == XML_TAG_NAME) {
				if (containsTagName(yy_buffer, yy_startRead, yy_markedPos - yy_startRead))
					fCurrentTagName = yytext();
				else
					fCurrentTagName = null;
			} else if (context == XML_TAG_OPEN) {
				fIsBlockingEnabled = true;
			} else if (context == XML_END_TAG_OPEN) {
				fIsBlockingEnabled = false;
			}
			start = yychar;
			textLength = length = yylength();
			if (yy_atEOF) {
				fTokenCount++;
				return null;
			}
		}
		// store the next token
		f_context = primGetNextToken();
		if (f_context == PROXY_CONTEXT) {
			fBufferedEmbeddedContainer = fEmbeddedContainer;
			fShouldLoadBuffered = true;
		} else if (f_context == XML_TAG_NAME) {
			if (containsTagName(yy_buffer, yy_startRead, yy_markedPos - yy_startRead))
				fCurrentTagName = yytext();
			else
				fCurrentTagName = null;
		} else if (f_context == XML_TAG_OPEN) {
			fIsBlockingEnabled = true;
		} else if (f_context == XML_END_TAG_OPEN) {
			fIsBlockingEnabled = false;
		}
		fBufferedContext = f_context;
		fBufferedStart = yychar;
		fBufferedLength = yylength();
		fShouldLoadBuffered = true;
		if (fBufferedContext == WHITE_SPACE) {
			fShouldLoadBuffered = false;
			length += fBufferedLength;
		}
		if (context == null) {
			// EOF
			if (Debug.debugTokenizer) {
				System.out.println(getClass().getName() + " discovered " + fTokenCount + " tokens."); //$NON-NLS-2$//$NON-NLS-1$
			}
			return null;
		}
		fTokenCount++;
		return fRegionFactory.createToken(context, start, textLength, length, null, fCurrentTagName);
	}

	/* user method */
	public XMLTokenizer() {
		super();
	}

	/* user method */
	public XMLTokenizer(char[] charArray) {
		this(new CharArrayReader(charArray));
	}

	/* user method */
	public void reset(char[] charArray) {
		reset(new CharArrayReader(charArray), 0);
	}

	/* user method */
	public void reset(char[] charArray, int newOffset) {
		reset(new CharArrayReader(charArray), newOffset);
	}

	/* user method */
	public void reset(java.io.InputStream in) {
		reset(new java.io.InputStreamReader(in), 0);
	}

	/* user method */
	public void reset(java.io.InputStream in, int newOffset) {
		reset(new java.io.InputStreamReader(in), newOffset);
	}

	/* user method */
	public void reset(java.io.Reader in) {
		reset(in, 0);
	}

	/**
	 * user method *
	 * 
	 * Reset internal counters and vars to "newly created" values, in the
	 * hopes that resetting a pre-existing tokenizer is faster than creating a
	 * new one.
	 * 
	 * This method contains code blocks that were essentially duplicated from
	 * the <em>generated</em> output of this specification before this
	 * method was added. Those code blocks were under the above copyright.
	 */
	public void reset(java.io.Reader in, int newOffset) {
		if (Debug.debugTokenizer) {
			System.out.println("resetting tokenizer");//$NON-NLS-1$
		}
		fOffset = newOffset;

		/* the input device */
		yy_reader = in;

		/* the current state of the DFA */
		yy_state = 0;

		/* the current lexical state */
		yy_lexical_state = YYINITIAL;

		/*
		 * this buffer contains the current text to be matched and is the
		 * source of the yytext() string
		 */
		java.util.Arrays.fill(yy_buffer, (char) 0);

		/* the textposition at the last accepting state */
		yy_markedPos = 0;

		/* the textposition at the last state to be included in yytext */
		yy_pushbackPos = 0;

		/* the current text position in the buffer */
		yy_currentPos = 0;

		/* startRead marks the beginning of the yytext() string in the buffer */
		yy_startRead = 0;

		/**
		 * endRead marks the last character in the buffer, that has been read
		 * from input
		 */
		yy_endRead = 0;

		/* number of newlines encountered up to the start of the matched text */
		yyline = 0;

		/* the number of characters up to the start of the matched text */
		yychar = 0;

		/* yy_atEOF == true <=> the scanner has returned a value for EOF */
		yy_atEOF = false;

		/* denotes if the user-EOF-code has already been executed */
		yy_eof_done = false;


		/* user vars: */
		fTokenCount = 0;

		fShouldLoadBuffered = false;
		fBufferedContext = null;
		fBufferedStart = 1;
		fBufferedLength = 0;
		fStateStack = new IntStack();

		context = null;
		start = 0;
		textLength = 0;
		length = 0;

		fEmbeddedContainer = null;
	}

	/**
	 * user method
	 * 
	 * @see com.ibm.sed.parser.BlockTokenizer#newInstance()
	 */
	public BlockTokenizer newInstance() {
		XMLTokenizer newInstance = new XMLTokenizer();
		// global tagmarkers can be shared; they have no state and
		// are never destroyed (e.g. 'release')
		for (int i = 0; i < fBlockMarkers.size(); i++) {
			BlockMarker blockMarker = (BlockMarker) fBlockMarkers.get(i);
			if (blockMarker.isGlobal())
				newInstance.addBlockMarker(blockMarker);
		}
		return newInstance;
	}

	/* user method */
	private final String scanXMLCommentText() throws IOException {
		// Scan for '-->' and return the text up to that point as
		//   XML_COMMENT_TEXT unless the string occurs IMMEDIATELY, in which
		//  case change to the ST_XML_COMMENT_END state and return the next
		//  context as usual.
		return doScan("-->", false, XML_COMMENT_TEXT, ST_XML_COMMENT_END, ST_XML_COMMENT_END);
	}


	/**
	 * Creates a new scanner There is also a java.io.InputStream version of
	 * this constructor.
	 * 
	 * @param in
	 *            the java.io.Reader to read input from.
	 */
	public XMLTokenizer(java.io.Reader in) {
		this.yy_reader = in;
	}

	/**
	 * Creates a new scanner. There is also java.io.Reader version of this
	 * constructor.
	 * 
	 * @param in
	 *            the java.io.Inputstream to read input from.
	 */
	public XMLTokenizer(java.io.InputStream in) {
		this(new java.io.InputStreamReader(in));
	}

	/**
	 * Unpacks the compressed DFA transition table.
	 * 
	 * @param packed
	 *            the packed transition table
	 * @return the unpacked transition table
	 */
	private static int[] yy_unpack(String packed) {
		int[] trans = new int[9114];
		int i = 0; /* index in packed string */
		int j = 0; /* index in unpacked array */
		while (i < 3174) {
			int count = packed.charAt(i++);
			int value = packed.charAt(i++);
			value--;
			do
				trans[j++] = value;
			while (--count > 0);
		}
		return trans;
	}

	/**
	 * Unpacks the compressed character translation table.
	 * 
	 * @param packed
	 *            the packed character translation table
	 * @return the unpacked character translation table
	 */
	private static char[] yy_unpack_cmap(String packed) {
		char[] map = new char[0x10000];
		int i = 0; /* index in packed string */
		int j = 0; /* index in unpacked array */
		while (i < 1372) {
			int count = packed.charAt(i++);
			char value = packed.charAt(i++);
			do
				map[j++] = value;
			while (--count > 0);
		}
		return map;
	}


	/**
	 * Gets the next input character.
	 * 
	 * @return the next character of the input stream, EOF if the end of the
	 *         stream is reached.
	 * @exception IOException
	 *                if any I/O-Error occurs
	 */
	private int yy_advance() throws java.io.IOException {

		/* standard case */
		if (yy_currentPos < yy_endRead)
			return yy_buffer[yy_currentPos++];

		/* if the eof is reached, we don't need to work hard */
		if (yy_atEOF)
			return YYEOF;

		/* otherwise: need to refill the buffer */

		/* first: make room (if you can) */
		if (yy_startRead > 0) {
			System.arraycopy(yy_buffer, yy_startRead, yy_buffer, 0, yy_endRead - yy_startRead);

			/* translate stored positions */
			yy_endRead -= yy_startRead;
			yy_currentPos -= yy_startRead;
			yy_markedPos -= yy_startRead;
			yy_pushbackPos -= yy_startRead;
			yy_startRead = 0;
		}

		/* is the buffer big enough? */
		if (yy_currentPos >= yy_buffer.length) {
			/* if not: blow it up */
			char newBuffer[] = new char[yy_currentPos * 2];
			System.arraycopy(yy_buffer, 0, newBuffer, 0, yy_buffer.length);
			yy_buffer = newBuffer;
		}

		/* finally: fill the buffer with new input */
		int numRead = yy_reader.read(yy_buffer, yy_endRead, yy_buffer.length - yy_endRead);

		if (numRead == -1)
			return YYEOF;

		yy_endRead += numRead;

		return yy_buffer[yy_currentPos++];
	}


	/**
	 * Closes the input stream.
	 */
	final public void yyclose() throws java.io.IOException {
		yy_atEOF = true; /* indicate end of file */
		yy_endRead = yy_startRead; /* invalidate buffer */
		yy_reader.close();
	}


	/**
	 * Returns the current lexical state.
	 */
	final public int yystate() {
		return yy_lexical_state;
	}

	/**
	 * Enters a new lexical state
	 * 
	 * @param newState
	 *            the new lexical state
	 */
	final public void yybegin(int newState) {
		yy_lexical_state = newState;
	}


	/**
	 * Returns the text matched by the current regular expression.
	 */
	final public String yytext() {
		return new String(yy_buffer, yy_startRead, yy_markedPos - yy_startRead);
	}

	/**
	 * Returns the length of the matched text region.
	 */
	final public int yylength() {
		return yy_markedPos - yy_startRead;
	}


	/**
	 * Reports an error that occured while scanning - from the SED JFlex
	 * skeleton
	 * 
	 * @param errorCode
	 *            the code of the errormessage to display
	 */
	private void yy_ScanError(int errorCode) {
		try {
			Logger.log(Logger.ERROR, YY_ERROR_MSG[errorCode]);
		} catch (ArrayIndexOutOfBoundsException e) {
			Logger.log(Logger.ERROR, YY_ERROR_MSG[YY_UNKNOWN_ERROR]);
		}
		// DO NOT EXIT the VM on an error
		// System.exit(1);
	}


	/**
	 * Pushes the specified amount of characters back into the input stream.
	 * 
	 * They will be read again by then next call of the scanning method
	 * 
	 * @param number
	 *            the number of characters to be read again. This number must
	 *            not be greater than yylength()!
	 */
	 void yypushback(int number) {
		if (number > yylength())
			yy_ScanError(YY_PUSHBACK_2BIG);

		yy_markedPos -= number;
	}

	/**
	 * user method - skeleton.sed
	 */
	protected final boolean containsTagName(char[] markerTagName, int offset, int tagnameLength) {
		for (int j = 0; j < fBlockMarkers.size(); j++) {
			BlockMarker marker = (BlockMarker) fBlockMarkers.get(j);
			if (marker.getTagName().length() == tagnameLength) {
				boolean matchesSoFar = true;
				for (int i = 0; i < tagnameLength && matchesSoFar; i++) {
					if (marker.isCaseSensitive()) {
						if (marker.getTagName().charAt(i) != markerTagName[i + offset])
							matchesSoFar = false;
					} else {
						if (Character.toLowerCase(marker.getTagName().charAt(i)) != Character.toLowerCase(markerTagName[i + offset]))
							matchesSoFar = false;
					}
				}
				if (matchesSoFar)
					return true;
			}
		}
		return false;
	}

	/**
	 * user method - skeleton.sed
	 * 
	 * Return ALL of the regions scannable within the remaining text Note: for
	 * verification use
	 */
	public final List getRegions() {
		List tokens = new ArrayList();
		ITextRegion region = null;
		try {
			region = getNextToken();
			while (region != null) {
				if (region != null) {
					tokens.add(region);
				}
				region = getNextToken();
			}
		} catch (StackOverflowError e) {
			Logger.logException(getClass().getName() + ": input could not be tokenized correctly at position " + getOffset(), e);//$NON-NLS-1$
			throw e;
		} catch (Exception e) {
			// Since this is convenience method and NOT the recommended
			// way of getting tokens, many errors are simply hidden
			Logger.logException("Exception not handled retrieving regions: " + e.getLocalizedMessage(), e);//$NON-NLS-1$
		}
		return tokens;
	}

	/**
	 * user method - skeleton.sed
	 */
	private final void dump(String s) {
		if (Debug.debugTokenizer) {
			System.out.println(s + " (" + yychar + "-" + //$NON-NLS-2$//$NON-NLS-1$
						(yylength() + yychar) + "):\'" + //$NON-NLS-1$
						StringUtils.escape(yytext()) + "\'");//$NON-NLS-1$
		}
	}

	/* user method - skeleton.sed */
	public final boolean isEOF() {
		return yy_atEOF;
	}

	/* user method - skeleton.sed */
	protected final boolean containsTagName(String markerTagName) {
		Iterator blocks = fBlockMarkers.iterator();
		while (blocks.hasNext()) {
			BlockMarker marker = (BlockMarker) blocks.next();
			if (marker.isCaseSensitive()) {
				if (marker.getTagName().equals(markerTagName))
					return true;
			} else {
				if (marker.getTagName().equalsIgnoreCase(markerTagName))
					return true;
			}
		}
		return false;
	}

	/**
	 * Contains user EOF-code, which will be executed exactly once, when the
	 * end of file is reached
	 */
	private void yy_do_eof() {
		if (!yy_eof_done) {
			yy_eof_done = true;
			// do nothing, this is the downstream parser's job

		}
	}


	/**
	 * Resumes scanning until the next regular expression is matched, the end
	 * of input is encountered or an I/O-Error occurs.
	 * 
	 * @return the next token
	 * @exception IOException
	 *                if any I/O-Error occurs
	 */
	public String primGetNextToken() throws java.io.IOException {
		int yy_input;
		int yy_action;


		while (true) {

			yychar += yylength();

			boolean yy_counted = false;
			for (yy_currentPos = yy_startRead; yy_currentPos < yy_markedPos; yy_currentPos++) {
				switch (yy_buffer[yy_currentPos]) {
					case '\r' :
						yyline++;
						yy_counted = true;
						break;
					case '\n' :
						if (yy_counted)
							yy_counted = false;
						else {
							yyline++;
						}
						break;
					default :
						yy_counted = false;
				}
			}

			if (yy_counted) {
				if (yy_advance() == '\n')
					yyline--;
				if (!yy_atEOF)
					yy_currentPos--;
			}

			yy_action = -1;

			yy_currentPos = yy_startRead = yy_markedPos;

			yy_state = yy_lexical_state;


			yy_forAction : {
				while (true) {

					yy_input = yy_advance();

					if (yy_input == YYEOF)
						break yy_forAction;

					int yy_next = yytrans[yy_rowMap[yy_state] + yycmap[yy_input]];
					if (yy_next == -1)
						break yy_forAction;
					yy_state = yy_next;

					int yy_attributes = YY_ATTRIBUTE[yy_state];
					if ((yy_attributes & 1) > 0) {
						yy_action = yy_state;
						yy_markedPos = yy_currentPos;
						if ((yy_attributes & 8) > 0)
							break yy_forAction;
					}

				}
			}


			switch (yy_action) {

				case 274 :
				case 275 :
				case 276 : {
					if (Debug.debugTokenizer)
						dump("\nCDATA start");//$NON-NLS-1$
					fStateStack.push(yystate());
					yybegin(ST_CDATA_TEXT);
					return XML_CDATA_OPEN;
				}
				case 278 :
					break;
				case 268 : {
					if (Debug.debugTokenizer)
						dump("element");//$NON-NLS-1$
					yybegin(ST_XML_ELEMENT_DECLARATION);
					return XML_ELEMENT_DECLARATION;
				}
				case 279 :
					break;
				case 267 : {
					if (Debug.debugTokenizer)
						dump("attlist");//$NON-NLS-1$
					yybegin(ST_XML_ATTLIST_DECLARATION);
					return XML_ATTLIST_DECLARATION;
				}
				case 280 :
					break;
				case 266 : {
					if (Debug.debugTokenizer)
						dump("doctype");//$NON-NLS-1$
					yybegin(ST_XML_DOCTYPE_DECLARATION);
					return XML_DOCTYPE_DECLARATION;
				}
				case 281 :
					break;
				case 262 : {
					if (Debug.debugTokenizer)
						dump("doctype external id");//$NON-NLS-1$
					fEmbeddedHint = XML_DOCTYPE_EXTERNAL_ID_PUBREF;
					yybegin(ST_XML_DOCTYPE_ID_PUBLIC);
					return XML_DOCTYPE_EXTERNAL_ID_PUBLIC;
				}
				case 282 :
					break;
				case 261 : {
					if (Debug.debugTokenizer)
						dump("doctype external id");//$NON-NLS-1$
					fEmbeddedHint = XML_DOCTYPE_EXTERNAL_ID_SYSREF;
					yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
					return XML_DOCTYPE_EXTERNAL_ID_SYSTEM;
				}
				case 283 :
					break;
				case 257 : {
					if (Debug.debugTokenizer)
						dump("DHTML processing instruction target");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_DHTML_ATTRIBUTE_NAME);
					return XML_TAG_NAME;
				}
				case 284 :
					break;
				case 234 : {
					if (Debug.debugTokenizer)
						dump("\nCharRef");//$NON-NLS-1$
					return XML_CHAR_REFERENCE;
				}
				case 285 :
					break;
				case 231 : {
					if (Debug.debugTokenizer)
						dump("\ncomment start");//$NON-NLS-1$
					fEmbeddedHint = XML_COMMENT_TEXT;
					fEmbeddedPostState = ST_XML_COMMENT;
					yybegin(ST_XML_COMMENT);
					return XML_COMMENT_OPEN;
				}
				case 286 :
					break;
				case 213 : {
					if (Debug.debugTokenizer)
						dump("XML processing instruction target");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_XML_PI_ATTRIBUTE_NAME);
					return XML_TAG_NAME;
				}
				case 287 :
					break;
				case 212 : {
					if (Debug.debugTokenizer)
						dump("comment end");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					yybegin(YYINITIAL);
					return XML_COMMENT_CLOSE;
				}
				case 288 :
					break;
				case 211 : {
					if (Debug.debugTokenizer)
						dump("CDATA end");//$NON-NLS-1$
					yybegin(fStateStack.pop());
					return XML_CDATA_CLOSE;
				}
				case 289 :
					break;
				case 210 : {
					if (Debug.debugTokenizer)
						dump("\nPEReference");//$NON-NLS-1$
					return XML_PE_REFERENCE;
				}
				case 290 :
					break;
				case 207 : {
					if (Debug.debugTokenizer)
						dump("\nEntityRef");//$NON-NLS-1$
					return XML_ENTITY_REFERENCE;
				}
				case 291 :
					break;
				case 158 :
				case 172 :
				case 180 : {
					return XML_DOCTYPE_INTERNAL_SUBSET;
				}
				case 292 :
					break;
				case 146 : {
					yybegin(YYINITIAL);
					fEmbeddedHint = UNDEFINED;
					if (Debug.debugTokenizer)
						dump("empty tag close");//$NON-NLS-1$
					return XML_EMPTY_TAG_CLOSE;
				}
				case 293 :
					break;
				case 131 : {
					if (Debug.debugTokenizer)
						dump("XML processing instruction end");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					yybegin(YYINITIAL);
					return XML_PI_CLOSE;
				}
				case 294 :
					break;
				case 130 : {
					// ended with nothing inside
					fEmbeddedHint = UNDEFINED;
					yybegin(YYINITIAL);
					return XML_PI_CLOSE;
				}
				case 295 :
					break;
				case 127 : {
					if (Debug.debugTokenizer)
						dump("processing instruction end");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					yybegin(YYINITIAL);
					return XML_PI_CLOSE;
				}
				case 296 :
					break;
				case 119 : {
					fStateStack.push(yystate());
					if (Debug.debugTokenizer)
						dump("\ndeclaration start");//$NON-NLS-1$
					yybegin(ST_XML_DECLARATION);
					return XML_DECLARATION_OPEN;
				}
				case 297 :
					break;
				case 118 : {
					if (Debug.debugTokenizer)
						dump("\nprocessing instruction start");//$NON-NLS-1$
					yybegin(ST_PI);
					return XML_PI_OPEN;
				}
				case 298 :
					break;
				case 62 : {
					if (Debug.debugTokenizer)
						dump("DHTML processing instruction end");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					yybegin(YYINITIAL);
					return XML_PI_CLOSE;
				}
				case 299 :
					break;
				case 56 :
				case 58 :
				case 59 :
				case 60 :
				case 135 : {
					if (Debug.debugTokenizer)
						dump("XML processing instruction attribute value");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_XML_PI_ATTRIBUTE_NAME);
					return XML_TAG_ATTRIBUTE_VALUE;
				}
				case 300 :
					break;
				case 55 : {
					if (Debug.debugTokenizer)
						dump("XML processing instruction '='");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_VALUE;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_XML_PI_ATTRIBUTE_VALUE);
					return XML_TAG_ATTRIBUTE_EQUALS;
				}
				case 301 :
					break;
				case 54 : {
					if (Debug.debugTokenizer)
						dump("XML processing instruction attribute name");//$NON-NLS-1$
					yybegin(ST_XML_PI_EQUALS);
					return XML_TAG_ATTRIBUTE_NAME;
				}
				case 302 :
					break;
				case 50 :
				case 51 :
				case 52 : {
					// block scan until close is found
					return doScan("?>", false, XML_PI_CONTENT, ST_XML_PI_TAG_CLOSE, ST_XML_PI_TAG_CLOSE);
				}
				case 303 :
					break;
				case 49 : {
					yybegin(ST_PI_CONTENT);
					return WHITE_SPACE;
				}
				case 304 :
					break;
				case 46 :
				case 47 :
				case 48 :
				case 128 :
				case 129 :
				case 214 :
				case 236 :
				case 248 : {
					if (Debug.debugTokenizer)
						dump("processing instruction target");//$NON-NLS-1$
					fEmbeddedHint = XML_CONTENT;
					yybegin(ST_PI_WS);
					return XML_TAG_NAME;
				}
				case 305 :
					break;
				case 41 :
				case 42 : {
					if (Debug.debugTokenizer)
						dump("comment content");//$NON-NLS-1$
					return scanXMLCommentText();
				}
				case 306 :
					break;
				case 40 : {
					if (Debug.debugTokenizer)
						dump("LINE FEED");//$NON-NLS-1$
					return WHITE_SPACE;
				}
				case 307 :
					break;
				case 0 :
				case 31 :
				case 121 :
				case 123 :
				case 205 :
				case 206 :
				case 233 : {
					if (Debug.debugTokenizer)
						dump("\nXML content");//$NON-NLS-1$
					return XML_CONTENT;
				}
				case 308 :
					break;
				case 5 :
				case 8 :
				case 9 :
				case 10 :
				case 12 :
				case 13 :
				case 14 :
				case 15 :
				case 17 :
				case 18 :
				case 19 :
				case 20 :
				case 21 :
				case 22 :
				case 23 :
				case 24 :
				case 25 :
				case 26 :
				case 28 :
				case 45 : {
					if (Debug.debugTokenizer)
						dump("white space");//$NON-NLS-1$
					return WHITE_SPACE;
				}
				case 309 :
					break;
				case 16 :
				case 70 : {
					if (Debug.debugTokenizer)
						dump("inappropriate tag name");//$NON-NLS-1$
					yybegin(YYINITIAL);
					return XML_CONTENT;
				}
				case 310 :
					break;
				case 27 :
				case 105 :
				case 106 :
				case 191 :
				case 226 :
				case 244 :
				case 254 :
				case 263 :
				case 269 :
				case 272 : {
					if (Debug.debugTokenizer)
						dump("elementdecl contentspec");//$NON-NLS-1$
					return XML_ELEMENT_DECL_CONTENT;
				}
				case 311 :
					break;
				case 29 :
				case 112 :
				case 113 :
				case 202 :
				case 230 :
				case 246 :
				case 255 :
				case 264 :
				case 270 :
				case 273 : {
					if (Debug.debugTokenizer)
						dump("attlist contentspec");//$NON-NLS-1$
					return XML_ATTLIST_DECL_CONTENT;
				}
				case 312 :
					break;
				case 32 :
				case 71 :
				case 82 : {
					if (Debug.debugTokenizer)
						dump("\nstart tag open");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_NAME;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_XML_TAG_NAME);
					return XML_TAG_OPEN;
				}
				case 313 :
					break;
				case 33 :
				case 34 :
				case 37 :
				case 38 :
				case 39 :
				case 43 :
				case 44 :
				case 53 :
				case 57 :
				case 61 :
				case 63 :
				case 67 :
				case 73 :
				case 79 :
				case 84 :
				case 85 :
				case 86 :
				case 87 :
				case 89 :
				case 90 :
				case 92 :
				case 97 :
				case 102 :
				case 109 : {
					if (Debug.debugTokenizer)
						System.out.println("!!!unexpected!!!: \"" + yytext() + "\":" + //$NON-NLS-2$//$NON-NLS-1$
									yychar + "-" + (yychar + yylength()));//$NON-NLS-1$
					return UNDEFINED;
				}
				case 314 :
					break;
				case 35 :
				case 36 : {
					if (Debug.debugTokenizer)
						dump("CDATA text");//$NON-NLS-1$
					fEmbeddedHint = XML_CDATA_TEXT;
					fEmbeddedPostState = ST_CDATA_TEXT;
					String blockContext = doBlockScan("]]>", XML_CDATA_TEXT, ST_CDATA_END);//$NON-NLS-1$
					if (blockContext == XML_CDATA_TEXT)
						yybegin(ST_CDATA_END);
					return blockContext;
				}
				case 315 :
					break;
				case 64 : {
					if (Debug.debugTokenizer)
						dump("DHTML processing instruction attribute name");//$NON-NLS-1$
					yybegin(ST_DHTML_EQUALS);
					return XML_TAG_ATTRIBUTE_NAME;
				}
				case 316 :
					break;
				case 65 : {
					if (Debug.debugTokenizer)
						dump("DHTML processing instruction '='");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_VALUE;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_DHTML_ATTRIBUTE_VALUE);
					return XML_TAG_ATTRIBUTE_EQUALS;
				}
				case 317 :
					break;
				case 66 :
				case 68 :
				case 69 :
				case 143 : {
					if (Debug.debugTokenizer)
						dump("DHTML processing instruction attribute value");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_DHTML_ATTRIBUTE_NAME);
					return XML_TAG_ATTRIBUTE_VALUE;
				}
				case 318 :
					break;
				case 72 : {
					if (Debug.debugTokenizer)
						dump("tag close");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					if (isBlockMarker()) {
						fEmbeddedHint = getBlockMarkerContext();
						fEmbeddedPostState = ST_BLOCK_TAG_SCAN;
						yybegin(ST_BLOCK_TAG_SCAN);
					} else
						yybegin(YYINITIAL);
					return XML_TAG_CLOSE;
				}
				case 319 :
					break;
				case 74 :
				case 75 : {
					if (Debug.debugTokenizer)
						dump("tag name");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_XML_ATTRIBUTE_NAME);
					return XML_TAG_NAME;
				}
				case 320 :
					break;
				case 76 : {
					if (Debug.debugTokenizer)
						dump("attr name");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_XML_EQUALS);
					return XML_TAG_ATTRIBUTE_NAME;
				}
				case 321 :
					break;
				case 77 : {
					if (Debug.debugTokenizer)
						dump("equals");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_VALUE;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_XML_ATTRIBUTE_VALUE);
					return XML_TAG_ATTRIBUTE_EQUALS;
				}
				case 322 :
					break;
				case 78 :
				case 80 :
				case 81 :
				case 150 : {
					if (Debug.debugTokenizer)
						dump("attr value");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_ATTRIBUTE_NAME;
					fEmbeddedPostState = ST_XML_EQUALS;
					yybegin(ST_XML_ATTRIBUTE_NAME);
					return XML_TAG_ATTRIBUTE_VALUE;
				}
				case 323 :
					break;
				case 83 : {
					if (Debug.debugTokenizer)
						dump("declaration end");//$NON-NLS-1$
					if (Debug.debugTokenizer) {
						if (fStateStack.peek() != YYINITIAL)
							System.out.println("end embedded region");//$NON-NLS-1$
					}
					yybegin(fStateStack.pop());
					return XML_DECLARATION_CLOSE;
				}
				case 324 :
					break;
				case 88 : {
					if (Debug.debugTokenizer)
						dump("doctype type");//$NON-NLS-1$
					yybegin(ST_XML_DOCTYPE_EXTERNAL_ID);
					return XML_DOCTYPE_NAME;
				}
				case 325 :
					break;
				case 91 :
				case 93 :
				case 94 :
				case 95 :
				case 164 :
				case 165 :
				case 168 :
				case 169 :
				case 221 : {
					if (Debug.debugTokenizer)
						dump("doctype public reference");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					fEmbeddedPostState = YYINITIAL;
					yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
					return XML_DOCTYPE_EXTERNAL_ID_PUBREF;
				}
				case 326 :
					break;
				case 96 :
				case 98 :
				case 99 :
				case 100 :
				case 176 : {
					if (Debug.debugTokenizer)
						dump("doctype system reference");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					fEmbeddedPostState = YYINITIAL;
					yybegin(ST_XML_DECLARATION_CLOSE);
					return XML_DOCTYPE_EXTERNAL_ID_SYSREF;
				}
				case 327 :
					break;
				case 101 :
				case 103 :
				case 104 :
				case 184 :
				case 185 :
				case 188 :
				case 189 :
				case 224 : {
					if (Debug.debugTokenizer)
						dump("elementdecl name");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					fEmbeddedPostState = YYINITIAL;
					yybegin(ST_XML_ELEMENT_DECLARATION_CONTENT);
					return XML_ELEMENT_DECL_NAME;
				}
				case 328 :
					break;
				case 107 : {
					if (Debug.debugTokenizer)
						dump("elementdecl close");//$NON-NLS-1$
					if (Debug.debugTokenizer) {
						if (fStateStack.peek() != YYINITIAL)
							System.out.println("end embedded region");//$NON-NLS-1$
					}
					yybegin(fStateStack.pop());
					return XML_DECLARATION_CLOSE;
				}
				case 329 :
					break;
				case 108 :
				case 110 :
				case 111 :
				case 195 :
				case 196 :
				case 199 :
				case 200 :
				case 228 : {
					if (Debug.debugTokenizer)
						dump("attlist name");//$NON-NLS-1$
					fEmbeddedHint = UNDEFINED;
					fEmbeddedPostState = YYINITIAL;
					yybegin(ST_XML_ATTLIST_DECLARATION_CONTENT);
					return XML_ATTLIST_DECL_NAME;
				}
				case 330 :
					break;
				case 114 : {
					if (Debug.debugTokenizer)
						dump("attlist close");//$NON-NLS-1$
					if (Debug.debugTokenizer) {
						if (fStateStack.peek() != YYINITIAL)
							System.out.println("end embedded region");//$NON-NLS-1$
					}
					yybegin(fStateStack.pop());
					return XML_DECLARATION_CLOSE;
				}
				case 331 :
					break;
				case 117 : {
					if (Debug.debugTokenizer)
						dump("\nend tag open");//$NON-NLS-1$
					fEmbeddedHint = XML_TAG_NAME;
					fEmbeddedPostState = ST_XML_ATTRIBUTE_NAME;
					yybegin(ST_XML_TAG_NAME);
					return XML_END_TAG_OPEN;
				}
				case 332 :
					break;
				case 115 :
				case 116 : {
					return doBlockTagScan();
				}
				case 333 :
					break;
				default :
					if (yy_input == YYEOF && yy_startRead == yy_currentPos) {
						yy_atEOF = true;
						yy_do_eof();
						return null;
					} else {
						yy_ScanError(YY_NO_MATCH);
					}
			}
		}
	}


}