/* The following code was generated by JFlex 1.2.2 on 10/17/07 4:27 AM */

/*******************************************************************************
 * Copyright (c) 2004, 2007 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/

package org.eclipse.wst.xml.core.internal.parser;

import java.io.CharArrayReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.eclipse.wst.sse.core.internal.ltk.parser.BlockMarker;
import org.eclipse.wst.sse.core.internal.ltk.parser.BlockTokenizer;
import org.eclipse.wst.sse.core.internal.provisional.text.ITextRegion;
import org.eclipse.wst.sse.core.internal.util.Debug;
import org.eclipse.wst.sse.core.utils.StringUtils;
import org.eclipse.wst.xml.core.internal.Logger;
import org.eclipse.wst.xml.core.internal.parser.regions.XMLParserRegionFactory;
import org.eclipse.wst.xml.core.internal.regions.DOMRegionContext;


/**
 * This class is a scanner generated by 
 * <a href="http://www.informatik.tu-muenchen.de/~kleing/jflex/">JFlex</a> 1.2.2
 * on 10/17/07 4:27 AM from the specification file
 * <tt>file:/D:/eclipse.wtp/workspace/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLTokenizer.jflex</tt>
 */
public class XMLTokenizer implements BlockTokenizer, DOMRegionContext {

  /** this character denotes the end of file */
  final public static int YYEOF = -1;

  /** lexical states */
  final public static int ST_XML_DOCTYPE_EXTERNAL_ID = 23;
  final public static int ST_XML_ELEMENT_DECLARATION_CONTENT = 27;
  final public static int ST_DHTML_ATTRIBUTE_NAME = 12;
  final public static int ST_XML_PI_TAG_CLOSE = 11;
  final public static int ST_XML_DECLARATION_CLOSE = 21;
  final public static int ST_XML_PI_ATTRIBUTE_VALUE = 10;
  final public static int ST_DHTML_EQUALS = 13;
  final public static int ST_XML_TAG_NAME = 16;
  final public static int ST_XML_ATTRIBUTE_VALUE = 19;
  final public static int ST_DHTML_ATTRIBUTE_VALUE = 14;
  final public static int ST_XML_DOCTYPE_ID_SYSTEM = 25;
  final public static int ST_XML_ATTRIBUTE_NAME = 17;
  final public static int ST_XML_ELEMENT_DECLARATION = 26;
  final public static int ST_XML_DOCTYPE_DECLARATION = 22;
  final public static int ST_XML_ATTLIST_DECLARATION = 28;
  final public static int ST_XML_COMMENT_END = 4;
  final public static int ST_CDATA_TEXT = 1;
  final public static int ST_DHTML_TAG_CLOSE = 15;
  final public static int ST_XML_COMMENT = 3;
  final public static int ST_PI_CONTENT = 7;
  final public static int ST_PI_WS = 6;
  final public static int ST_CDATA_END = 2;
  final public static int ST_XML_ATTLIST_DECLARATION_CONTENT = 29;
  final public static int ST_BLOCK_TAG_SCAN = 30;
  final public static int ST_XML_PI_EQUALS = 9;
  final public static int ST_XML_DECLARATION = 20;
  final public static int YYINITIAL = 0;
  final public static int ST_XML_DOCTYPE_ID_PUBLIC = 24;
  final public static int ST_XML_EQUALS = 18;
  final public static int ST_PI = 5;
  final public static int ST_XML_PI_ATTRIBUTE_NAME = 8;

  /** 
   * Translates characters to character classes
   */
  final private static String yycmap_packed = 
    "\11\0\1\5\1\22\2\0\1\14\22\0\1\14\1\21\1\11\1\55"+
    "\1\16\1\17\1\12\1\13\1\16\1\16\1\16\1\16\1\16\1\7"+
    "\1\6\1\3\12\15\1\10\1\61\1\1\1\45\1\2\1\4\1\16"+
    "\1\34\1\62\1\32\1\33\1\50\1\57\1\36\1\36\1\42\1\36"+
    "\1\36\1\27\1\25\1\44\1\43\1\47\1\36\1\40\1\56\1\35"+
    "\1\60\2\36\1\23\1\46\1\36\1\31\1\0\1\20\1\0\1\10"+
    "\1\0\1\52\1\62\1\63\1\53\1\37\1\57\1\36\1\66\1\42"+
    "\2\36\1\30\1\26\1\44\1\43\1\47\1\36\1\40\1\41\1\51"+
    "\1\60\1\36\1\36\1\24\1\54\1\36\1\0\1\0\72\0\1\65"+
    "\10\0\27\64\1\0\37\64\1\0\72\64\2\0\13\64\2\0\10\64"+
    "\1\0\65\64\1\0\104\64\11\0\44\64\3\0\2\64\4\0\36\64"+
    "\70\0\131\64\22\0\7\64\16\0\2\65\56\0\106\65\32\0\2\65"+
    "\44\0\1\64\1\65\3\64\1\0\1\64\1\0\24\64\1\0\54\64"+
    "\1\0\7\64\3\0\1\64\1\0\1\64\1\0\1\64\1\0\1\64"+
    "\1\0\22\64\15\0\14\64\1\0\102\64\1\0\14\64\1\0\44\64"+
    "\1\0\4\65\11\0\65\64\2\0\2\64\2\0\2\64\3\0\34\64"+
    "\2\0\10\64\2\0\2\64\67\0\46\64\2\0\1\64\7\0\46\64"+
    "\12\0\21\65\1\0\27\65\1\0\3\65\1\0\1\65\1\0\2\65"+
    "\1\0\1\65\13\0\33\64\5\0\3\64\56\0\32\64\5\0\1\65"+
    "\12\64\10\65\15\0\12\65\6\0\1\65\107\64\2\0\5\64\1\0"+
    "\17\64\1\0\4\64\1\0\1\64\17\65\2\64\2\65\1\0\4\65"+
    "\2\0\12\65\u0207\0\3\65\1\0\65\64\2\0\1\65\1\64\20\65"+
    "\3\0\4\65\3\0\12\64\2\65\2\0\12\65\21\0\3\65\1\0"+
    "\10\64\2\0\2\64\2\0\26\64\1\0\7\64\1\0\1\64\3\0"+
    "\4\64\2\0\1\65\1\0\7\65\2\0\2\65\2\0\3\65\11\0"+
    "\1\65\4\0\2\64\1\0\3\64\2\65\2\0\12\65\2\64\20\0"+
    "\1\65\2\0\6\64\4\0\2\64\2\0\26\64\1\0\7\64\1\0"+
    "\2\64\1\0\2\64\1\0\2\64\2\0\1\65\1\0\5\65\4\0"+
    "\2\65\2\0\3\65\13\0\4\64\1\0\1\64\7\0\12\65\2\65"+
    "\3\64\14\0\3\65\1\0\7\64\1\0\1\64\1\0\3\64\1\0"+
    "\26\64\1\0\7\64\1\0\2\64\1\0\5\64\2\0\1\65\1\64"+
    "\10\65\1\0\3\65\1\0\3\65\22\0\1\64\5\0\12\65\21\0"+
    "\3\65\1\0\10\64\2\0\2\64\2\0\26\64\1\0\7\64\1\0"+
    "\2\64\2\0\4\64\2\0\1\65\1\64\6\65\3\0\2\65\2\0"+
    "\3\65\10\0\2\65\4\0\2\64\1\0\3\64\4\0\12\65\22\0"+
    "\2\65\1\0\6\64\3\0\3\64\1\0\4\64\3\0\2\64\1\0"+
    "\1\64\1\0\2\64\3\0\2\64\3\0\3\64\3\0\10\64\1\0"+
    "\3\64\4\0\5\65\3\0\3\65\1\0\4\65\11\0\1\65\17\0"+
    "\11\65\21\0\3\65\1\0\10\64\1\0\3\64\1\0\27\64\1\0"+
    "\12\64\1\0\5\64\4\0\7\65\1\0\3\65\1\0\4\65\7\0"+
    "\2\65\11\0\2\64\4\0\12\65\22\0\2\65\1\0\10\64\1\0"+
    "\3\64\1\0\27\64\1\0\12\64\1\0\5\64\4\0\7\65\1\0"+
    "\3\65\1\0\4\65\7\0\2\65\7\0\1\64\1\0\2\64\4\0"+
    "\12\65\22\0\2\65\1\0\10\64\1\0\3\64\1\0\27\64\1\0"+
    "\20\64\4\0\6\65\2\0\3\65\1\0\4\65\11\0\1\65\10\0"+
    "\2\64\4\0\12\65\221\0\56\64\1\0\1\64\1\65\2\64\7\65"+
    "\5\0\6\64\1\65\10\65\1\0\12\65\47\0\2\64\1\0\1\64"+
    "\2\0\2\64\1\0\1\64\2\0\1\64\6\0\4\64\1\0\7\64"+
    "\1\0\3\64\1\0\1\64\1\0\1\64\2\0\2\64\1\0\2\64"+
    "\1\0\1\64\1\65\2\64\6\65\1\0\2\65\1\64\2\0\5\64"+
    "\1\0\1\65\1\0\6\65\2\0\12\65\76\0\2\65\6\0\12\65"+
    "\13\0\1\65\1\0\1\65\1\0\1\65\4\0\2\65\10\64\1\0"+
    "\41\64\7\0\24\65\1\0\6\65\4\0\6\65\1\0\1\65\1\0"+
    "\25\65\3\0\7\65\1\0\1\65\346\0\46\64\12\0\47\64\11\0"+
    "\1\64\1\0\2\64\1\0\3\64\1\0\1\64\1\0\2\64\1\0"+
    "\5\64\51\0\1\64\1\0\1\64\1\0\1\64\13\0\1\64\1\0"+
    "\1\64\1\0\1\64\3\0\2\64\3\0\1\64\5\0\3\64\1\0"+
    "\1\64\1\0\1\64\1\0\1\64\1\0\1\64\3\0\2\64\3\0"+
    "\2\64\1\0\1\64\50\0\1\64\11\0\1\64\2\0\1\64\2\0"+
    "\2\64\7\0\2\64\1\0\1\64\1\0\7\64\50\0\1\64\4\0"+
    "\1\64\10\0\1\64\u0c06\0\234\64\4\0\132\64\6\0\26\64\2\0"+
    "\6\64\2\0\46\64\2\0\6\64\2\0\10\64\1\0\1\64\1\0"+
    "\1\64\1\0\1\64\1\0\37\64\2\0\65\64\1\0\7\64\1\0"+
    "\1\64\3\0\3\64\1\0\7\64\3\0\4\64\2\0\6\64\4\0"+
    "\15\64\5\0\3\64\1\0\7\64\323\0\15\65\4\0\1\65\104\0"+
    "\1\64\3\0\2\64\2\0\1\64\121\0\3\64\u0e82\0\1\65\1\0"+
    "\1\64\31\0\11\64\6\65\1\0\5\65\13\0\124\64\4\0\2\65"+
    "\2\0\2\65\2\0\132\64\1\0\3\65\6\0\50\64\u1cd3\0\u51a6\64"+
    "\u0c5a\0\u2ba4\64\134\0\u0800\0\u1ffe\0\2\0";

  /** 
   * Translates characters to character classes
   */
  final private static char [] yycmap = yy_unpack_cmap(yycmap_packed);

  /** 
   * Translates a state to a row index in the transition table
   */
  final private static int yy_rowMap [] = { 
        0,    55,   110,   165,   220,   275,   330,   385,   440,   495, 
      550,   605,   660,   715,   770,   825,   880,   935,   990,  1045, 
     1100,  1155,  1210,  1265,  1320,  1375,  1430,  1485,  1540,  1595, 
     1650,  1705,  1760,  1815,  1870,  1925,  1980,  1925,  1980,  2035, 
     1925,  1925,  1980,  2090,  2145,  2200,  2255,  2310,  2365,  2420, 
     2475,  1925,  1980,  2530,  2585,  2640,  1925,  2695,  2695,  2750, 
     2805,  2860,  2530,  1925,  2915,  2970,  1925,  3025,  3080,  3135, 
     3190,  3245,  3300,  1925,  3355,  3410,  3465,  3520,  1925,  3575, 
     3630,  3685,  3740,  3795,  1925,  3850,  3905,  3960,  4015,  4070, 
     4125,  4180,  4235,  4235,  4290,  4345,  4400,  4455,  4455,  4510, 
     4565,  4620,  4675,  4675,  4730,  4785,  4840,  4895,  1925,  4950, 
     4950,  5005,  5060,  5115,  5170,  1925,  1925,  1980,  1925,  1925, 
     5225,  5280,  5335,  5390,  5445,  5500,  5555,  5610,  1925,  5665, 
     5720,  5775,  1925,  1925,  2695,  5830,  2805,  1925,  5885,  2860, 
     2915,  3025,  3080,  5940,  3135,  1925,  5995,  3190,  1925,  3575, 
     6050,  3685,  1925,  6105,  3740,  5225,  6160,  6215,  6270,  4015, 
     1925,  6325,  6380,  4235,  6435,  4290,  1925,  6490,  6545,  6600, 
     6600,  6655,  6710,  4400,  4235,  4455,  6765,  4510,  1925,  6820, 
     4565,  4620,  4455,  4675,  6875,  4730,  1925,  6930,  6985,  7040, 
     7040,  7095,  7150,  7205,  4950,  7260,  5005,  1925,  7315,  7370, 
     7425,  7425,  7480,  7535,  7590,  7645,  7700,  7755,  7810,  1925, 
     7865,  7920,  1925,  1925,  1925,  2255,  7975,  8030,  8085,  8140, 
     8195,  8250,  8305,  6490,  8360,  8360,  6930,  8415,  8415,  8470, 
     7315,  8525,  8525,  8580,  1925,  8635,  8690,  1925,  8745,  8800, 
     8855,  8910,  8965,  9020,  9075,  9130,  6655,  7095,  9185,  7480, 
     9240,  9295,  9350,  9405,  9460,  9515,  9570,  9625,  9680,  9735, 
     9790,  9845,  9900,  9955, 10010, 10065, 10120,  1925,  1925, 10175, 
    10230, 10285, 10340, 10395,  1925,  1925,  1925, 10450, 10505, 10560, 
    10615, 10670, 10725,  1925, 10780,  4840,  5115, 10835, 10890, 10945, 
    11000,  2255
  };

  /** 
   * The packed transition table of the DFA
   */
  final private static String yy_packed = 
    "\1\40\1\41\10\40\1\42\4\40\1\43\47\40\1\44"+
    "\1\45\65\44\1\46\1\47\16\46\1\50\1\46\1\51"+
    "\44\46\1\52\1\53\65\52\1\46\1\47\5\46\1\54"+
    "\12\46\1\51\45\46\1\47\2\46\1\55\1\56\2\46"+
    "\1\57\3\46\1\56\5\46\1\56\1\60\1\61\4\57"+
    "\1\46\10\57\1\62\2\57\1\46\7\57\1\46\3\57"+
    "\1\46\3\57\1\46\1\57\1\46\1\47\2\46\1\55"+
    "\1\63\6\46\1\63\5\46\1\63\44\46\1\64\1\65"+
    "\2\64\1\66\15\64\1\51\44\64\1\46\1\47\2\46"+
    "\1\67\1\56\2\46\1\70\3\46\1\56\5\46\1\56"+
    "\6\70\1\46\13\70\1\46\7\70\1\46\3\70\1\46"+
    "\3\70\1\46\1\70\1\46\1\47\2\46\1\67\1\56"+
    "\2\46\1\70\3\46\1\56\5\46\1\56\6\70\1\46"+
    "\13\70\1\71\7\70\1\46\3\70\1\46\3\70\1\46"+
    "\1\70\1\72\1\47\1\46\1\73\1\74\1\56\3\72"+
    "\1\75\1\72\1\76\1\56\5\72\1\56\44\72\1\46"+
    "\1\47\2\46\1\77\15\46\1\51\45\46\1\47\1\100"+
    "\1\101\1\46\1\56\2\46\1\102\3\46\1\56\5\46"+
    "\1\56\6\102\1\46\13\102\1\46\7\102\1\46\3\102"+
    "\1\46\3\102\1\46\1\102\1\46\1\47\1\100\1\101"+
    "\1\46\1\56\2\46\1\102\3\46\1\56\5\46\1\56"+
    "\6\102\1\46\13\102\1\103\7\102\1\46\3\102\1\46"+
    "\3\102\1\46\1\102\1\104\1\47\1\100\1\105\1\104"+
    "\1\56\3\104\1\106\1\104\1\107\1\56\5\104\1\56"+
    "\44\104\1\46\1\47\3\46\1\56\6\46\1\56\5\46"+
    "\1\56\44\46\1\110\1\111\1\112\1\113\4\110\1\114"+
    "\12\110\6\115\1\110\13\115\1\110\7\115\1\110\3\115"+
    "\1\110\3\115\1\110\1\115\1\46\1\111\1\112\1\113"+
    "\1\46\1\56\2\46\1\116\3\46\1\56\5\46\1\56"+
    "\6\116\1\46\13\116\1\46\7\116\1\46\3\116\1\46"+
    "\3\116\1\46\1\116\1\46\1\111\1\112\1\113\1\46"+
    "\1\56\2\46\1\116\3\46\1\56\5\46\1\56\6\116"+
    "\1\46\13\116\1\117\7\116\1\46\3\116\1\46\3\116"+
    "\1\46\1\116\1\120\1\111\1\112\1\121\1\120\1\56"+
    "\3\120\1\122\1\120\1\123\1\56\5\120\1\56\44\120"+
    "\1\46\1\124\1\125\2\46\1\56\6\46\1\56\5\46"+
    "\1\56\10\46\1\126\1\127\2\46\1\130\10\46\1\130"+
    "\1\46\1\127\1\126\14\46\1\47\1\125\2\46\1\56"+
    "\6\46\1\56\5\46\1\56\6\46\1\131\36\46\1\47"+
    "\1\125\2\46\1\56\2\46\1\132\3\46\1\56\5\46"+
    "\1\56\6\132\1\131\13\132\1\46\7\132\1\46\3\132"+
    "\1\46\3\132\1\46\1\132\1\46\1\47\1\125\2\46"+
    "\1\56\6\46\1\56\5\46\1\56\6\46\1\131\7\46"+
    "\1\133\5\46\1\134\6\46\1\133\10\46\1\135\1\47"+
    "\1\125\1\136\1\135\1\56\3\135\1\137\1\135\1\140"+
    "\1\56\5\135\1\56\6\135\1\141\35\135\1\142\1\47"+
    "\1\125\1\143\1\142\1\56\3\142\1\144\1\142\1\145"+
    "\1\56\5\142\1\56\6\142\1\146\35\142\1\147\1\47"+
    "\1\125\1\150\1\147\1\56\3\147\1\151\1\147\1\152"+
    "\1\56\5\147\1\56\44\147\1\153\1\154\1\155\64\153"+
    "\1\156\1\47\1\125\1\157\1\156\1\56\3\156\1\160"+
    "\1\156\1\161\1\56\5\156\1\56\44\156\1\162\1\163"+
    "\1\164\64\162\1\165\1\166\65\165\1\40\1\0\10\40"+
    "\1\0\4\40\1\0\47\40\3\0\1\167\1\170\14\0"+
    "\1\171\52\0\1\172\2\0\1\173\3\0\1\172\5\0"+
    "\1\172\6\173\1\0\13\173\1\0\7\173\1\174\3\173"+
    "\1\0\3\173\1\0\1\173\5\0\1\172\2\0\1\175"+
    "\3\0\1\172\5\0\1\172\6\175\1\0\13\175\1\0"+
    "\7\175\1\0\3\175\1\0\3\175\1\0\1\175\110\0"+
    "\1\176\65\0\1\177\55\0\1\200\61\0\1\201\71\0"+
    "\1\56\6\0\1\56\5\0\1\56\52\0\3\57\4\0"+
    "\1\57\5\0\6\57\1\0\13\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\6\0\3\57\4\0\1\57\5\0"+
    "\2\57\2\202\2\57\1\0\13\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\6\0\3\57\4\0\1\57\5\0"+
    "\2\57\1\202\1\203\2\57\1\0\13\57\1\0\7\57"+
    "\1\0\3\57\1\0\5\57\6\0\3\57\4\0\1\57"+
    "\5\0\2\57\2\204\2\57\1\0\13\57\1\0\7\57"+
    "\1\0\3\57\1\0\5\57\5\0\1\63\6\0\1\63"+
    "\5\0\1\63\46\0\1\205\66\0\1\206\72\0\3\70"+
    "\4\0\1\70\5\0\6\70\1\0\13\70\1\0\7\70"+
    "\1\0\3\70\1\0\5\70\1\72\2\0\1\207\1\72"+
    "\1\0\3\72\1\0\1\72\2\0\5\72\1\0\45\72"+
    "\1\0\1\206\1\207\1\72\1\0\3\72\1\0\1\72"+
    "\2\0\5\72\1\0\44\72\1\75\1\0\1\210\1\211"+
    "\1\75\1\210\3\75\1\212\1\75\2\210\5\75\1\210"+
    "\44\75\1\76\1\0\1\213\1\214\1\76\1\213\3\76"+
    "\1\213\1\76\1\212\1\213\5\76\1\213\44\76\2\0"+
    "\1\100\1\215\71\0\3\102\4\0\1\102\5\0\6\102"+
    "\1\0\13\102\1\0\7\102\1\0\3\102\1\0\5\102"+
    "\1\104\2\0\1\216\1\104\1\0\3\104\1\0\1\104"+
    "\2\0\5\104\1\0\45\104\1\0\1\100\1\217\1\104"+
    "\1\0\3\104\1\0\1\104\2\0\5\104\1\0\44\104"+
    "\1\106\1\0\1\220\1\221\1\106\1\220\3\106\1\222"+
    "\1\106\2\220\5\106\1\220\44\106\1\107\1\0\1\223"+
    "\1\224\1\107\1\223\3\107\1\223\1\107\1\222\1\223"+
    "\5\107\1\223\44\107\1\110\3\0\17\110\6\0\1\110"+
    "\13\0\1\110\7\0\1\110\3\0\1\110\3\0\1\110"+
    "\4\0\1\167\15\0\1\171\47\0\1\225\64\0\1\110"+
    "\3\0\2\110\3\114\4\110\1\114\5\110\6\115\1\110"+
    "\13\115\1\110\7\115\1\110\3\115\1\110\3\115\1\114"+
    "\1\115\6\0\3\115\4\0\1\115\5\0\6\115\1\0"+
    "\13\115\1\0\7\115\1\0\3\115\1\0\5\115\6\0"+
    "\3\116\4\0\1\116\5\0\6\116\1\0\13\116\1\0"+
    "\7\116\1\0\3\116\1\0\5\116\1\120\2\0\1\226"+
    "\1\120\1\0\3\120\1\0\1\120\2\0\5\120\1\0"+
    "\45\120\1\0\1\225\1\226\1\120\1\0\3\120\1\0"+
    "\1\120\2\0\5\120\1\0\44\120\1\122\1\0\1\227"+
    "\1\230\1\122\1\227\3\122\1\231\1\122\2\227\5\122"+
    "\1\227\44\122\1\123\1\0\1\232\1\233\1\123\1\232"+
    "\3\123\1\232\1\123\1\231\1\232\5\123\1\232\44\123"+
    "\3\0\1\167\15\0\1\234\110\0\1\235\60\0\1\236"+
    "\13\0\1\236\44\0\2\237\36\0\20\240\1\241\46\240"+
    "\6\0\3\132\4\0\1\132\5\0\6\132\1\0\13\132"+
    "\1\0\7\132\1\0\3\132\1\0\5\132\46\0\1\242"+
    "\5\0\1\242\72\0\1\243\6\0\1\135\2\0\1\244"+
    "\1\135\1\0\3\135\1\0\1\135\2\0\5\135\1\0"+
    "\44\135\1\137\1\0\1\245\1\246\1\137\1\245\3\137"+
    "\1\247\1\137\2\245\5\137\1\245\44\137\1\250\1\0"+
    "\1\251\1\252\1\253\1\251\3\253\1\251\1\250\1\254"+
    "\1\255\3\253\1\250\1\253\1\255\6\253\1\250\32\253"+
    "\2\250\1\253\1\141\2\240\1\256\1\141\1\240\3\141"+
    "\1\240\1\141\2\240\3\141\1\257\1\141\1\240\44\141"+
    "\1\142\2\0\1\260\1\142\1\0\3\142\1\0\1\142"+
    "\2\0\5\142\1\0\44\142\1\144\2\261\1\262\1\144"+
    "\1\261\3\144\1\263\1\144\2\261\5\144\1\261\44\144"+
    "\1\145\2\264\1\265\1\145\1\264\3\145\1\264\1\145"+
    "\1\263\1\264\5\145\1\264\44\145\1\146\2\240\1\266"+
    "\1\146\1\240\3\146\1\240\1\146\2\240\3\146\1\267"+
    "\1\146\1\240\44\146\1\147\2\0\1\270\1\147\1\0"+
    "\3\147\1\0\1\147\2\0\5\147\1\0\44\147\1\151"+
    "\1\0\1\271\1\272\1\151\1\271\3\151\1\273\1\151"+
    "\2\271\5\151\1\271\44\151\1\274\1\0\1\275\1\276"+
    "\1\277\1\275\3\277\1\275\1\274\1\300\1\301\3\277"+
    "\1\274\1\277\1\301\6\277\1\274\32\277\2\274\1\277"+
    "\2\153\1\0\66\153\1\0\16\153\1\302\45\153\1\156"+
    "\2\0\1\303\1\156\1\0\3\156\1\0\1\156\2\0"+
    "\5\156\1\0\44\156\1\160\1\0\1\304\1\305\1\160"+
    "\1\304\3\160\1\306\1\160\2\304\5\160\1\304\44\160"+
    "\1\307\1\0\1\310\1\311\1\312\1\310\3\312\1\310"+
    "\1\307\1\313\1\314\3\312\1\307\1\312\1\314\6\312"+
    "\1\307\32\312\2\307\1\312\2\162\1\0\66\162\1\0"+
    "\16\162\1\315\45\162\7\0\1\316\21\0\1\317\42\0"+
    "\1\172\2\0\1\40\3\0\1\172\5\0\1\172\6\40"+
    "\1\0\13\40\1\0\7\40\1\0\3\40\1\0\3\40"+
    "\1\0\1\40\1\320\1\0\3\320\1\321\3\173\1\320"+
    "\1\0\1\320\1\321\1\173\1\320\1\0\2\320\1\321"+
    "\6\173\1\320\13\173\1\320\7\173\1\320\3\173\1\322"+
    "\5\173\15\0\1\323\6\0\1\324\42\0\1\320\1\0"+
    "\3\320\1\321\3\175\1\320\1\0\1\320\1\321\1\175"+
    "\1\320\1\0\2\320\1\321\6\175\1\320\13\175\1\320"+
    "\7\175\1\320\3\175\1\325\5\175\31\0\1\317\37\0"+
    "\1\326\66\0\1\327\72\0\3\57\4\0\1\57\5\0"+
    "\4\57\2\330\1\0\13\57\1\0\7\57\1\0\3\57"+
    "\1\0\5\57\6\0\3\57\4\0\1\57\5\0\4\57"+
    "\1\330\1\331\1\0\13\57\1\0\7\57\1\0\3\57"+
    "\1\0\5\57\6\0\3\57\4\0\1\57\5\0\6\57"+
    "\1\0\13\57\1\0\1\57\1\332\5\57\1\0\3\57"+
    "\1\0\5\57\1\210\1\0\7\210\1\212\55\210\1\213"+
    "\1\0\11\213\1\212\53\213\1\220\1\0\7\220\1\222"+
    "\55\220\1\223\1\0\11\223\1\222\53\223\1\227\1\0"+
    "\7\227\1\231\55\227\1\232\1\0\11\232\1\231\53\232"+
    "\32\0\1\333\30\0\1\333\40\0\1\334\13\0\1\334"+
    "\54\0\1\335\10\0\1\335\57\0\1\336\14\0\1\336"+
    "\72\0\1\337\4\0\1\245\1\0\7\245\1\247\55\245"+
    "\1\250\1\0\1\251\1\340\1\250\1\251\3\250\1\251"+
    "\1\250\1\247\1\251\5\250\1\251\44\250\1\251\1\0"+
    "\11\251\1\247\53\251\1\250\1\0\1\251\1\340\1\250"+
    "\1\251\3\250\1\251\1\250\1\341\1\251\5\250\1\251"+
    "\44\250\13\0\1\342\53\0\1\251\1\0\11\251\1\341"+
    "\53\251\11\261\1\263\55\261\13\264\1\263\53\264\1\271"+
    "\1\0\7\271\1\273\55\271\1\274\1\0\1\275\1\343"+
    "\1\274\1\275\3\274\1\275\1\274\1\273\1\275\5\274"+
    "\1\275\44\274\1\275\1\0\11\275\1\273\53\275\1\274"+
    "\1\0\1\275\1\343\1\274\1\275\3\274\1\275\1\274"+
    "\1\344\1\275\5\274\1\275\44\274\13\0\1\345\53\0"+
    "\1\275\1\0\11\275\1\344\53\275\2\153\1\0\26\153"+
    "\1\346\35\153\1\304\1\0\7\304\1\306\55\304\1\307"+
    "\1\0\1\310\1\347\1\307\1\310\3\307\1\310\1\307"+
    "\1\306\1\310\5\307\1\310\44\307\1\310\1\0\11\310"+
    "\1\306\53\310\1\307\1\0\1\310\1\347\1\307\1\310"+
    "\3\307\1\310\1\307\1\350\1\310\5\307\1\310\44\307"+
    "\13\0\1\351\53\0\1\310\1\0\11\310\1\350\53\310"+
    "\2\162\1\0\26\162\1\352\35\162\7\0\1\353\111\0"+
    "\1\354\34\0\1\320\1\0\10\320\1\0\4\320\1\0"+
    "\41\320\1\0\6\320\1\0\3\320\1\321\4\320\1\0"+
    "\1\320\1\321\2\320\1\0\2\320\1\321\36\320\1\355"+
    "\5\320\15\0\1\323\43\0\1\356\22\0\1\357\14\0"+
    "\3\357\2\0\1\357\10\0\1\357\1\0\2\357\3\0"+
    "\1\357\2\0\2\357\11\0\1\57\1\360\1\57\4\0"+
    "\1\57\5\0\6\57\1\0\13\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\6\0\3\57\4\0\1\57\5\0"+
    "\6\57\1\0\11\57\1\361\1\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\35\0\1\362\13\0\1\362\44\0"+
    "\2\363\63\0\2\364\75\0\1\365\13\0\1\365\44\0"+
    "\2\366\41\0\2\367\1\0\3\367\2\0\1\254\4\367"+
    "\1\0\10\367\1\0\32\367\2\0\1\367\3\0\2\370"+
    "\1\0\3\370\2\0\1\300\4\370\1\0\10\370\1\0"+
    "\32\370\2\0\1\370\2\153\1\0\27\153\1\371\34\153"+
    "\3\0\2\372\1\0\3\372\2\0\1\313\4\372\1\0"+
    "\10\372\1\0\32\372\2\0\1\372\2\162\1\0\27\162"+
    "\1\373\34\162\33\0\1\374\114\0\1\355\22\0\1\357"+
    "\14\0\3\357\2\0\1\357\10\0\1\357\1\0\2\357"+
    "\3\0\1\357\1\0\1\356\2\357\11\0\3\57\4\0"+
    "\1\57\5\0\6\57\1\0\7\57\1\375\3\57\1\0"+
    "\7\57\1\0\3\57\1\0\5\57\6\0\3\57\4\0"+
    "\1\57\5\0\6\57\1\0\6\57\1\376\4\57\1\0"+
    "\7\57\1\0\3\57\1\0\5\57\46\0\1\377\5\0"+
    "\1\377\54\0\1\u0100\63\0\1\u0101\10\0\1\u0101\55\0"+
    "\1\u0102\10\0\1\u0102\60\0\1\u0103\24\0\2\153\1\0"+
    "\30\153\1\u0104\33\153\2\162\1\0\30\162\1\u0105\33\162"+
    "\34\0\1\u0106\40\0\3\57\4\0\1\57\5\0\6\57"+
    "\1\0\13\57\1\0\3\57\1\u0107\3\57\1\0\3\57"+
    "\1\0\5\57\6\0\3\57\4\0\1\57\5\0\6\57"+
    "\1\0\3\57\1\u0108\7\57\1\0\3\57\1\u0108\3\57"+
    "\1\0\3\57\1\0\5\57\47\0\1\u0109\60\0\1\u010a"+
    "\14\0\1\u010a\54\0\1\u010b\47\0\2\u010c\72\0\1\u010d"+
    "\30\0\1\u010d\3\0\2\153\1\0\31\153\1\u010e\32\153"+
    "\2\162\1\0\31\162\1\u010f\32\162\35\0\1\u0110\37\0"+
    "\3\57\4\0\1\57\5\0\6\57\1\0\13\57\1\0"+
    "\6\57\1\u0111\1\0\3\57\1\0\5\57\5\0\1\u0112"+
    "\3\57\3\0\1\u0112\1\57\4\0\1\u0112\6\57\1\0"+
    "\13\57\1\0\7\57\1\0\3\57\1\0\5\57\37\0"+
    "\1\u0113\10\0\1\u0113\53\0\1\u0114\13\0\1\u0114\52\0"+
    "\1\u0115\13\0\1\u0115\15\0\2\153\1\0\32\153\1\u0116"+
    "\31\153\2\162\1\0\32\162\1\u0117\31\162\34\0\1\u0118"+
    "\40\0\3\57\4\0\1\57\5\0\5\57\1\u0119\1\0"+
    "\13\57\1\0\7\57\1\0\3\57\1\0\5\57\5\0"+
    "\1\u0112\6\0\1\u0112\5\0\1\u0112\44\0\2\153\1\0"+
    "\31\153\1\u011a\32\153\2\162\1\0\31\162\1\u011b\32\162"+
    "\31\0\1\u011c\43\0\3\57\4\0\1\57\5\0\6\57"+
    "\1\0\5\57\1\u011d\5\57\1\0\7\57\1\0\3\57"+
    "\1\0\5\57\2\153\1\0\26\153\1\u011e\35\153\2\162"+
    "\1\0\26\162\1\u011f\35\162\6\0\3\57\4\0\1\57"+
    "\5\0\6\57\1\0\7\57\1\u0120\3\57\1\0\7\57"+
    "\1\0\3\57\1\0\5\57\6\0\3\57\4\0\1\57"+
    "\5\0\6\57\1\0\13\57\1\0\7\57\1\0\3\57"+
    "\1\0\4\57\1\u0121\6\0\3\57\4\0\1\57\5\0"+
    "\6\57\1\0\5\57\1\u0122\5\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\6\0\3\57\4\0\1\57\5\0"+
    "\6\57\1\0\5\57\1\u0123\5\57\1\0\7\57\1\0"+
    "\3\57\1\0\5\57\6\0\3\57\4\0\1\57\5\0"+
    "\6\57\1\0\13\57\1\0\3\57\1\u0124\3\57\1\0"+
    "\3\57\1\0\5\57";

  /** 
   * The transition table of the DFA
   */
  final private static int yytrans [] = yy_unpack(yy_packed);


  /* error codes */
  final private static int YY_UNKNOWN_ERROR = 0;
  // final private static int YY_ILLEGAL_STATE = 1;
  final private static int YY_NO_MATCH = 2;
  final private static int YY_PUSHBACK_2BIG = 3;

  /* error messages for the codes above */
  final private static String YY_ERROR_MSG[] = {
    "Unkown internal scanner error",		//$NON-NLS-1$
    "Internal error: unknown state",		//$NON-NLS-1$
    "Error: could not match input",		//$NON-NLS-1$
    "Error: pushback value was too large"	//$NON-NLS-1$
  };

  /**
   * YY_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
   */
  private final static byte YY_ATTRIBUTE[] = {
     1,  0,  0,  0,  0,  1,  0,  0,  1,  1,  1,  0,  1,  1,  1,  1, 
     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  1, 
     1,  1,  1,  9,  1,  9,  1,  1,  9,  9,  1,  1,  1,  1,  1,  1, 
     1,  1,  1,  9,  1,  1,  1,  1,  9,  1,  1,  1,  1,  1,  1,  9, 
     1,  1,  9,  1,  1,  1,  1,  1,  1,  9,  1,  1,  1,  1,  9,  1, 
     1,  1,  1,  1,  9,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 
     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  9,  1,  1,  1, 
     1,  1,  1,  9,  9,  1,  9,  9,  1,  0,  1,  0,  1,  0,  0,  0, 
     9,  1,  1,  1,  9,  9,  0,  0,  0,  9,  0,  0,  0,  0,  0,  0, 
     0,  9,  0,  0,  9,  0,  0,  0,  9,  0,  0,  0,  0,  0,  0,  0, 
     9,  0,  0,  0,  0,  0,  9,  1,  0,  0,  1,  1,  0,  0,  1,  0, 
     0,  0,  9,  0,  0,  0,  1,  0,  0,  0,  9,  1,  0,  0,  1,  1, 
     0,  1,  0,  0,  0,  9,  1,  0,  0,  1,  1,  0,  1,  0,  0,  1, 
     1,  9,  0,  0,  9,  9,  9,  1,  1,  1,  0,  0,  0,  0,  0,  0, 
     1,  0,  0,  1,  0,  1,  0,  1,  0,  1,  9,  0,  1,  9,  0,  1, 
     1,  0,  0,  0,  0,  0,  0,  0,  1,  0,  1,  0,  1,  1,  0,  0, 
     0,  0,  0,  1,  1,  0,  1,  1,  0,  0,  0,  9,  9,  1,  1,  0, 
     1,  1,  9,  9,  9,  1,  1,  0,  1,  1,  1,  9,  1,  1,  1,  1, 
     1,  1,  1,  1
  };

  /** the input device */
  private java.io.Reader yy_reader;

  /** the current state of the DFA */
  private int yy_state;

  /** the current lexical state */
  private int yy_lexical_state = YYINITIAL;

  /** this buffer contains the current text to be matched and is
      the source of the yytext() string */
  private char yy_buffer[] = new char[16384];

  /** the textposition at the last accepting state */
  private int yy_markedPos;

  /** the textposition at the last state to be included in yytext */
  private int yy_pushbackPos;

  /** the current text position in the buffer */
  private int yy_currentPos;

  /** startRead marks the beginning of the yytext() string in the buffer */
  private int yy_startRead;

  /** endRead marks the last character in the buffer, that has been read
      from input */
  private int yy_endRead;

  /** number of newlines encountered up to the start of the matched text */
  private int yyline;

  /** the number of characters up to the start of the matched text */
  private int yychar;

  /**
   * the number of characters from the last newline up to the start of the 
   * matched text
   */
  // private int yycolumn; 

  /** 
   * yy_atBOL == true <=> the scanner is currently at the beginning of a line
   */
  // private boolean yy_atBOL;

  /** yy_atEOF == true <=> the scanner has returned a value for EOF */
  private boolean yy_atEOF;

  /** denotes if the user-EOF-code has already been executed */
  private boolean yy_eof_done;

  /* user code: */
	private int fTokenCount = 0;
 
	// required holders for white-space compacting
	private boolean fShouldLoadBuffered = false;
	private String fBufferedContext = null;
	private int fBufferedStart = 1;
	private int fBufferedLength = 0;
	private String f_context = null;

	// state stack for handling embedded regions
	private IntStack fStateStack = new IntStack();

	private String context = null;
	private int start = 0;
	private int textLength = 0;
	private int length = 0;

	// offset for tracking position specific block tags
	private int fOffset = 0;
	
	// the name of the current tag being opened
	private String fCurrentTagName = null;

	// the list of tag name BlockMarkers
	private List fBlockMarkers = new ArrayList();

	// required to not seek text blocks on an end tag
	private boolean fIsBlockingEnabled = false;
	private boolean fIsCaseSensitiveBlocking = true;

	private XMLParserRegionFactory fRegionFactory = new XMLParserRegionFactory();
/**
 * user method 
 */
public final void addBlockMarker(BlockMarker marker) {
	if(containsTagName(marker.getTagName()))
		return;
	fBlockMarkers.add(marker);
}
/**
 * user method 
 */
public final void removeBlockMarker(BlockMarker marker) {
	fBlockMarkers.remove(marker);
}
/**
 * user method 
 */
public final void removeBlockMarker(String tagname) {
	if (fBlockMarkers != null) {
		Iterator blocks = fBlockMarkers.iterator();
		while (blocks.hasNext()) {
			if (((BlockMarker) blocks.next()).getTagName().equals(tagname))
				blocks.remove();
		}
	}
}
/* user method */
public final boolean isCaseSensitiveBlocking() {
	return fIsCaseSensitiveBlocking;
}
/* user method */
public final void setCaseSensitiveBlocking(boolean newValue) {
	fIsCaseSensitiveBlocking = newValue;
}
/* user method */
public boolean getBlockMarkerCaseSensitivity() {
        return getBlockMarkerCaseSensitivity(fCurrentTagName);
}
/* user method */
public boolean getBlockMarkerCaseSensitivity(String name) {
	Iterator iterator = fBlockMarkers.iterator();
	while(iterator.hasNext()) {
		BlockMarker marker = (BlockMarker)iterator.next();
		boolean casesensitive = marker.isCaseSensitive();
		if(casesensitive && marker.getTagName().equals(name))
			return casesensitive;
		else if(!casesensitive && marker.getTagName().equalsIgnoreCase(name))
			return casesensitive;
	}
	return true;
}
/* user method */
public String getBlockMarkerContext() {
	return getBlockMarkerContext(fCurrentTagName);
}
/* user method */
public String getBlockMarkerContext(String name) {
	Iterator iterator = fBlockMarkers.iterator();
	while(iterator.hasNext()) {
		BlockMarker marker = (BlockMarker)iterator.next();
		if(marker.getTagName().equals(name))
			return marker.getContext();
	}
	return BLOCK_TEXT;
}
/* user method */
public List getBlockMarkers() {
	return fBlockMarkers;
}
/* user method */
public final int getOffset() {
	return fOffset + yychar;
}
private final boolean isBlockMarker() {
	return isBlockMarker(fCurrentTagName);
}
private final boolean isBlockMarker(String tagName) {
	if (!fIsBlockingEnabled)
		return false;
	return containsTagName(tagName);
}
/**
 * user method
 */
public final void beginBlockTagScan(String newTagName) {
	beginBlockMarkerScan(newTagName, BLOCK_TEXT);
}
/**
 * user method
 *
 * Special tokenizer setup.  Allows tokenization to be initiated at the
 * start of a text block within a "newTagName" tag.
 *
 * Example: 
 *	Tokenizer toker = new Tokenizer();
 *	toker.setCaseSensitiveBlocking(false);
 *	toker.reset(new java.io.StringReader("afiuhqwkejhtasihgalkwhtq</scripter></scr></script>asgdasga"));
 *	toker.beginBlockMarkerScan("script", BLOCK_TEXT);
 *	toker.getRegions(); 
 *
 * Returns:
 *	BLOCK_TEXT: 0-40
 *	XML_END_TAG_OPEN: 41-42
 *	XML_TAG_NAME: 43-48
 *	XML_TAG_CLOSE: 49-49
 *	XML_CONTENT: 50-57
 *
 */
public final void beginBlockMarkerScan(String newTagName, String blockcontext) {
	yybegin(ST_BLOCK_TAG_SCAN);
	fCurrentTagName = newTagName;
}
/**
 * Method doScan.
 * 
 * Returns a context region for all of the text from the current position upto the end of input or
 * to right *before* the first occurence of searchString
 * 
 * @param searchString - target string to search for ex.: "-->", "</tagname"
 * @param requireTailSeparator - whether the target must be immediately followed by whitespace or '>'
 * @param context - the context of the scanned region if non-zero length
 * @param exitState - the state to go to if the region was of non-zero length
 * @param abortState - the state to go to if the searchString was found immediately
 * @return String - the context found: the desired context on a non-zero length match, the abortContext on immediate success
 * @throws IOException
 */
private final String doScan(String searchString, boolean requireTailSeparator, String searchContext, int exitState, int immediateFallbackState) throws IOException {
	boolean stillSearching = true;
	// Disable further block (probably)
	fIsBlockingEnabled = false;
	int searchStringLength = searchString.length();
	int n = 0;
	char lastCheckChar;
	int i;
	boolean same = false;
	while (stillSearching) {
		n = 0;
		// Ensure that enough data from the input exists to compare against the search String.
		n = yy_advance();
		while(n != YYEOF && yy_currentPos < searchStringLength)
			n = yy_advance();
		// If the input was too short or we've exhausted the input, stop immediately.
		if (n == YYEOF) {
			stillSearching = false;
		}
		else {
			same = true;
			// Ensure that we've not encountered a complete block (<%%>) that was *shorter* than the closeTagString and
			// thus found twice at current-targetLength [since the first scan would have come out this far anyway].
			// Check the characters in the target versus the last targetLength characters read from the buffer
			// and see if it matches
			
			// safety check for array accesses (yy_currentPos is the *last* character we can check against)
			if(yy_currentPos >= searchStringLength && yy_currentPos <= yy_buffer.length) {
				for(i = 0; i < searchStringLength; i++) {
					if(same && fIsCaseSensitiveBlocking)
						same = yy_buffer[i + yy_currentPos - searchStringLength] == searchString.charAt(i);
					else if(same && !fIsCaseSensitiveBlocking)
						same = Character.toLowerCase(yy_buffer[i + yy_currentPos - searchStringLength]) == Character.toLowerCase(searchString.charAt(i));
				}
			}
			// safety check failed; no match is possible right now
			else {
				same = false;
			}
			if (same && requireTailSeparator && yy_currentPos < yy_buffer.length) {
				// Additional check for close tags to ensure that targetString="</script" doesn't match
				// "</scriptS"
				lastCheckChar = yy_buffer[yy_currentPos];
				// Succeed on "</script>" and "</script "
				if(lastCheckChar == '>' || Character.isWhitespace(lastCheckChar))
					stillSearching = false;
			}
			else {
				stillSearching = !same || (yy_currentPos < yy_startRead + searchStringLength);
			}
		}
	}
	if (n != YYEOF || same) {
		// We've stopped short of the end or definitely found a match
		yy_markedPos = yy_currentPos - searchStringLength;
		yy_currentPos = yy_markedPos + 1;
		// If the searchString occurs at the very beginning of what would have
		// been a Block, resume scanning normally immediately
		if (yy_markedPos == yy_startRead) {
			yybegin(immediateFallbackState);
			return primGetNextToken();
		}
	}
	else {
		// We ran through the rest of the input
		yy_markedPos = yy_currentPos;
		yy_currentPos++;
	}
	yybegin(exitState);
	// If the ending occurs at the very beginning of what would have
	// been a Block, resume scanning normally immediately
	if(yy_markedPos == yy_startRead)
		return primGetNextToken();
	return searchContext;
}
/**
 * user method
 *
 * A generic lookahead-like operation
 */
private final String doBlockScan(String target, String targetContext, int immediateFallbackState) throws IOException {
	return doScan(target, false, targetContext, immediateFallbackState, immediateFallbackState);
}
/**
 * user method 
 * does a lookahead for the current tag name
 */
private final String doBlockTagScan() throws IOException {
        fIsCaseSensitiveBlocking = getBlockMarkerCaseSensitivity();
	return doScan("</" + fCurrentTagName, true, getBlockMarkerContext(fCurrentTagName), YYINITIAL, YYINITIAL);
}
/**
 * user method
 *
 * Converts the raw context String returned by the primGetNextToken()
 * method into a full ITextRegion by pulling in values for the
 * current offset within the scanning text.
 *
 * Returns null when EOF is encountered and attaches intermittently
 * discovered whitespace onto the end of useful regions.
 *
 * Note that this algorithm caches the token following the one being returned
 * so that whitespace can be collapsed.
 */
public final ITextRegion getNextToken() throws IOException {
	// load the starting non-whitespace token (assume that it is so)
	if (fShouldLoadBuffered) {
		context = fBufferedContext;
		start = fBufferedStart;
		textLength = length = fBufferedLength;
		fShouldLoadBuffered = false;
	}
	else {
		context = primGetNextToken();
		if (context == XML_TAG_NAME) {
			if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
				fCurrentTagName = yytext();
			else
				fCurrentTagName = null;
		}
		else if (context == XML_TAG_OPEN) {
			fIsBlockingEnabled = true;
		}
		else if (context == XML_END_TAG_OPEN) {
			fIsBlockingEnabled = false;
		}
		start = yychar;
		textLength = length = yylength();
		if (yy_atEOF) {
			fTokenCount++;
			return null;
		}
	}
	// store the next token
	f_context = primGetNextToken();
	if (f_context == XML_TAG_NAME) {
		if(containsTagName(yy_buffer, yy_startRead, yy_markedPos-yy_startRead))
			fCurrentTagName = yytext();
		else
			fCurrentTagName = null;
	}
	else if (f_context == XML_TAG_OPEN) {
		fIsBlockingEnabled = true;
	}
	else if (f_context == XML_END_TAG_OPEN) {
		fIsBlockingEnabled = false;
	}
	fBufferedContext = f_context;
	fBufferedStart = yychar;
	fBufferedLength = yylength();
	fShouldLoadBuffered = true;
	if (fBufferedContext == WHITE_SPACE) {
		fShouldLoadBuffered = false;
		length += fBufferedLength;
	}
	if (context == null) {
		// EOF
		if (Debug.debugTokenizer) {
			System.out.println(getClass().getName() + " discovered " + fTokenCount + " tokens."); //$NON-NLS-2$//$NON-NLS-1$
		}
		return null;
	}
	fTokenCount++;
	return fRegionFactory.createToken(context, start, textLength, length, null, fCurrentTagName);
}
/* user method */
public XMLTokenizer(){
	super();
}
/* user method */
public XMLTokenizer(char[] charArray){
		this(new CharArrayReader(charArray));
}
/* user method */
public void reset(char[] charArray) {
	reset(new CharArrayReader(charArray), 0);
}
/* user method */
public void reset(char[] charArray, int newOffset) {
	reset(new CharArrayReader(charArray), newOffset);
}
/* user method */
public void reset(java.io.InputStream in) {
	reset(new java.io.InputStreamReader(in), 0);
}
/* user method */
public void reset(java.io.InputStream in, int newOffset) {
	reset(new java.io.InputStreamReader(in), newOffset);
}
/* user method */
public void reset(java.io.Reader in) {
	reset(in, 0);
}
/**
 * user method *
 *
 * Reset internal counters and vars to "newly created" values, in the hopes
 * that resetting a pre-existing tokenizer is faster than creating a new one.
 *
 * This method contains code blocks that were essentially duplicated from the
 * <em>generated</em> output of this specification before this method was
 * added.  Those code blocks were under the above copyright.
 */
public void reset(java.io.Reader in, int newOffset) {
	if (Debug.debugTokenizer) {
		System.out.println("resetting tokenizer");//$NON-NLS-1$
	}
	fOffset = newOffset;

	/* the input device */
	yy_reader = in;

	/* the current state of the DFA */
	yy_state = 0;

	/* the current lexical state */
	yy_lexical_state = YYINITIAL;

	/* this buffer contains the current text to be matched and is
	the source of the yytext() string */
	java.util.Arrays.fill(yy_buffer, (char)0);

	/* the textposition at the last accepting state */
	yy_markedPos = 0;

	/* the textposition at the last state to be included in yytext */
	yy_pushbackPos = 0;

	/* the current text position in the buffer */
	yy_currentPos = 0;

	/* startRead marks the beginning of the yytext() string in the buffer */
	yy_startRead = 0;

	/** 
	 * endRead marks the last character in the buffer, that has been read
	 * from input 
	 */
	yy_endRead = 0;

	/* number of newlines encountered up to the start of the matched text */
	yyline = 0;

	/* the number of characters up to the start of the matched text */
	yychar = 0;

	/* yy_atEOF == true <=> the scanner has returned a value for EOF */
	yy_atEOF = false;

	/* denotes if the user-EOF-code has already been executed */
	yy_eof_done = false;


	/* user vars: */
	fTokenCount = 0;
 
	fShouldLoadBuffered = false;
	fBufferedContext = null;
	fBufferedStart = 1;
	fBufferedLength = 0;
	fStateStack = new IntStack();

	context = null;
	start = 0;
	textLength = 0;
	length = 0;
}

	/**
	 * user method
	 *
	 */
	public BlockTokenizer newInstance() {
		XMLTokenizer newInstance = new XMLTokenizer();
		// global tagmarkers can be shared; they have no state and 
		// are never destroyed (e.g. 'release')
		for(int i = 0; i < fBlockMarkers.size(); i++) {
			BlockMarker blockMarker = (BlockMarker) fBlockMarkers.get(i);
			if(blockMarker.isGlobal())
				newInstance.addBlockMarker(blockMarker);
		}
		return newInstance;
	}
/* user method */
private final String scanXMLCommentText() throws IOException {
	// Scan for '-->' and return the text up to that point as
	//   XML_COMMENT_TEXT unless the string occurs IMMEDIATELY, in which
	//  case change to the ST_XML_COMMENT_END state and return the next
	//  context as usual.
	return doScan("-->", false, XML_COMMENT_TEXT, ST_XML_COMMENT_END, ST_XML_COMMENT_END);
}


  /**
   * Creates a new scanner
   * There is also a java.io.InputStream version of this constructor.
   *
   * @param   in  the java.io.Reader to read input from.
   */
  public XMLTokenizer(java.io.Reader in) {
    this.yy_reader = in;
  }

  /**
   * Creates a new scanner.
   * There is also java.io.Reader version of this constructor.
   *
   * @param   in  the java.io.Inputstream to read input from.
   */
  public XMLTokenizer(java.io.InputStream in) {
    this(new java.io.InputStreamReader(in));
  }

  /** 
   * Unpacks the compressed DFA transition table.
   *
   * @param packed   the packed transition table
   * @return         the unpacked transition table
   */
  private static int [] yy_unpack(String packed) {
    int [] trans = new int[11055];
    int i = 0;  /* index in packed string  */
    int j = 0;  /* index in unpacked array */
    while (i < 3734) {
      int count = packed.charAt(i++);
      int value = packed.charAt(i++);
      value--;
      do trans[j++] = value; while (--count > 0);
    }
    return trans;
  }

  /** 
   * Unpacks the compressed character translation table.
   *
   * @param packed   the packed character translation table
   * @return         the unpacked character translation table
   */
  private static char [] yy_unpack_cmap(String packed) {
    char [] map = new char[0x10000];
    int i = 0;  /* index in packed string  */
    int j = 0;  /* index in unpacked array */
    while (i < 1372) {
      int  count = packed.charAt(i++);
      char value = packed.charAt(i++);
      do map[j++] = value; while (--count > 0);
    }
    return map;
  }


  /**
   * Gets the next input character.
   *
   * @return      the next character of the input stream, EOF if the
   *              end of the stream is reached.
   * @exception   IOException  if any I/O-Error occurs
   */
  private int yy_advance() throws java.io.IOException {

    /* standard case */
    if (yy_currentPos < yy_endRead) return yy_buffer[yy_currentPos++];

    /* if the eof is reached, we don't need to work hard */ 
    if (yy_atEOF) return YYEOF;

    /* otherwise: need to refill the buffer */

    /* first: make room (if you can) */
    if (yy_startRead > 0) {
      System.arraycopy(yy_buffer, yy_startRead, 
                       yy_buffer, 0, 
                       yy_endRead-yy_startRead);

      /* translate stored positions */
      yy_endRead-= yy_startRead;
      yy_currentPos-= yy_startRead;
      yy_markedPos-= yy_startRead;
      yy_pushbackPos-= yy_startRead;
      yy_startRead = 0;
    }

    /* is the buffer big enough? */
    if (yy_currentPos >= yy_buffer.length) {
      /* if not: blow it up */
      char newBuffer[] = new char[yy_currentPos*2];
      System.arraycopy(yy_buffer, 0, newBuffer, 0, yy_buffer.length);
      yy_buffer = newBuffer;
    }

    /* finally: fill the buffer with new input */
    int numRead = yy_reader.read(yy_buffer, yy_endRead, 
                                            yy_buffer.length-yy_endRead);

    if ( numRead == -1 ) return YYEOF;

    yy_endRead+= numRead;

    return yy_buffer[yy_currentPos++];
  }

    
  /**
   * Closes the input stream.
   */
  final public void yyclose() throws java.io.IOException {
    yy_atEOF = true;            /* indicate end of file */
    yy_endRead = yy_startRead;  /* invalidate buffer    */
    yy_reader.close();
  }


  /**
   * Returns the current lexical state.
   */
  final public int yystate() {
    return yy_lexical_state;
  }

  /**
   * Enters a new lexical state
   *
   * @param newState the new lexical state
   */
  final public void yybegin(int newState) {
    yy_lexical_state = newState;
  }


  /**
   * Returns the text matched by the current regular expression.
   */
  final public String yytext() {
    return new String( yy_buffer, yy_startRead, yy_markedPos-yy_startRead );
  }

  /**
   * Returns the length of the matched text region.
   */
  final public int yylength() {
    return yy_markedPos-yy_startRead;
  }


  /**
   * Reports an error that occured while scanning - from the SED JFlex skeleton
   *
   * @param   errorCode  the code of the errormessage to display
   */
  private void yy_ScanError(int errorCode) {
    try {
      Logger.log(Logger.ERROR, YY_ERROR_MSG[errorCode]);
    }
    catch (ArrayIndexOutOfBoundsException e) {
      Logger.log(Logger.ERROR, YY_ERROR_MSG[YY_UNKNOWN_ERROR]);
    }
    // DO NOT EXIT the VM on an error
    // System.exit(1);
  } 


  /**
   * Pushes the specified amount of characters back into the input stream.
   *
   * They will be read again by then next call of the scanning method
   *
   * @param number  the number of characters to be read again.
   *                This number must not be greater than yylength()!
   */
  void yypushback(int number) {
    if ( number > yylength() )
      yy_ScanError(YY_PUSHBACK_2BIG);

    yy_markedPos -= number;
  }

	/**
	 * user method - skeleton.sed
	 */
	protected final boolean containsTagName(char[] markerTagName, int offset, int tagnameLength) {
		for(int j = 0; j < fBlockMarkers.size(); j++) {
			BlockMarker marker = (BlockMarker)fBlockMarkers.get(j);
			if(marker.getTagName().length() == tagnameLength) {
				boolean matchesSoFar = true;
				for(int i = 0; i < tagnameLength && matchesSoFar; i++) {
					if(marker.isCaseSensitive()) {
						if(marker.getTagName().charAt(i) != markerTagName[i + offset])
							matchesSoFar = false;
					}
					else {
						if(Character.toLowerCase(marker.getTagName().charAt(i)) != Character.toLowerCase(markerTagName[i + offset]))
							matchesSoFar = false;
					}
				}
				if(matchesSoFar)
					return true;
			}
		}
		return false;
	}

	/**
	 * user method - skeleton.sed
	 *
	 * Return ALL of the regions scannable within the remaining text
	 * Note: for verification use
	 */
	public final List getRegions() {
		List tokens = new ArrayList();
		ITextRegion region = null;
		try {
			region = getNextToken();
			while(region != null) {
				if (region != null) {
					tokens.add(region);
				}
				region = getNextToken();
			}
		}
		catch (StackOverflowError e) {
			Logger.logException(getClass().getName()+": input could not be tokenized correctly at position " + getOffset(), e);//$NON-NLS-1$
			throw e;
		}
		catch (Exception e) {
			// Since this is convenience method and NOT the recommended 
			// way of getting tokens, many errors are simply hidden
			Logger.logException("Exception not handled retrieving regions: " + e.getLocalizedMessage(), e);//$NON-NLS-1$
		}
		return tokens;
	}
	/**
	 * user method - skeleton.sed
	 */
	private final void dump(String s) {
		if (Debug.debugTokenizer) {
			System.out.println(s + " (" + yychar + "-" + //$NON-NLS-2$//$NON-NLS-1$
				(yylength() + yychar) + "):\'" +//$NON-NLS-1$
					StringUtils.escape(yytext()) + "\'");//$NON-NLS-1$
		}
	}
	/* user method  - skeleton.sed */
	public final boolean isEOF() {
		return yy_atEOF;
	}
/* user method - skeleton.sed */
protected final boolean containsTagName(String markerTagName) {
	Iterator blocks = fBlockMarkers.iterator();
	while(blocks.hasNext()) {
		BlockMarker marker = (BlockMarker)blocks.next();
		if(marker.isCaseSensitive()) {
			if(marker.getTagName().equals(markerTagName))
				return true;
		}
		else {
			if(marker.getTagName().equalsIgnoreCase(markerTagName))
				return true;
		}
	}
	return false;
}

  /**
   * Contains user EOF-code, which will be executed exactly once,
   * when the end of file is reached
   */
  private void yy_do_eof() {
    if (!yy_eof_done) {
      yy_eof_done = true;
    // do nothing, this is the downstream parser's job

    }
  }


  /**
   * Resumes scanning until the next regular expression is matched,
   * the end of input is encountered or an I/O-Error occurs.
   *
   * @return      the next token
   * @exception   IOException  if any I/O-Error occurs
   */
  public String primGetNextToken() throws java.io.IOException {
    int yy_input;
    int yy_action;


    while (true) {

      yychar+= yylength();

      boolean yy_counted = false;
      for (yy_currentPos = yy_startRead; yy_currentPos < yy_markedPos;
                                                      yy_currentPos++) {
        switch (yy_buffer[yy_currentPos]) {
        case '\r':
          yyline++;
          yy_counted = true;
          break;
        case '\n':
          if (yy_counted)
            yy_counted = false;
          else {
            yyline++;
          }
          break;
        default:
          yy_counted = false;
        }
      }

      if (yy_counted) {
        if ( yy_advance() == '\n' ) yyline--;
        if ( !yy_atEOF ) yy_currentPos--;
      }

      yy_action = -1;

      yy_currentPos = yy_startRead = yy_markedPos;

      yy_state = yy_lexical_state;


      yy_forAction: {
        while (true) {
    
          yy_input = yy_advance();

          if ( yy_input == YYEOF ) break yy_forAction;

          int yy_next = yytrans[ yy_rowMap[yy_state] + yycmap[yy_input] ];
          if (yy_next == -1) break yy_forAction;
          yy_state = yy_next;

          int yy_attributes = YY_ATTRIBUTE[yy_state];
          if ( (yy_attributes & 1) > 0 ) {
            yy_action = yy_state; 
            yy_markedPos = yy_currentPos; 
            if ( (yy_attributes & 8) > 0 ) break yy_forAction;
          }

        }
      }


      switch (yy_action) {    

        case 291: 
          { 
	if(Debug.debugTokenizer)
		dump("XSL processing instruction target");//$NON-NLS-1$
        yybegin(ST_XML_PI_ATTRIBUTE_NAME);
        return XML_TAG_NAME;
 }
        case 293: break;
        case 283: 
        case 285: 
        case 286: 
          { 
	if(Debug.debugTokenizer)
		dump("\nCDATA start");//$NON-NLS-1$
	fStateStack.push(yystate());
	yybegin(ST_CDATA_TEXT);
	return XML_CDATA_OPEN;
 }
        case 294: break;
        case 276: 
          { 
	if(Debug.debugTokenizer)
		dump("element");//$NON-NLS-1$
	yybegin(ST_XML_ELEMENT_DECLARATION);
	return XML_ELEMENT_DECLARATION;
 }
        case 295: break;
        case 275: 
          { 
	if(Debug.debugTokenizer)
		dump("attlist");//$NON-NLS-1$
	yybegin(ST_XML_ATTLIST_DECLARATION);
	return XML_ATTLIST_DECLARATION;
 }
        case 296: break;
        case 274: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype");//$NON-NLS-1$
	yybegin(ST_XML_DOCTYPE_DECLARATION);
	return XML_DOCTYPE_DECLARATION;
 }
        case 297: break;
        case 268: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype external id");//$NON-NLS-1$
	yybegin(ST_XML_DOCTYPE_ID_PUBLIC);
	return XML_DOCTYPE_EXTERNAL_ID_PUBLIC;
 }
        case 298: break;
        case 267: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype external id");//$NON-NLS-1$
	yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
	return XML_DOCTYPE_EXTERNAL_ID_SYSTEM;
 }
        case 299: break;
        case 263: 
        case 273: 
          { 
	if(Debug.debugTokenizer)
		dump("DHTML processing instruction target");//$NON-NLS-1$
        yybegin(ST_DHTML_ATTRIBUTE_NAME);
        return XML_TAG_NAME;
 }
        case 300: break;
        case 237: 
          { 
	if(Debug.debugTokenizer)
		dump("\nCharRef");//$NON-NLS-1$
	return XML_CHAR_REFERENCE;
 }
        case 301: break;
        case 234: 
          { 
	if(Debug.debugTokenizer)
		dump("\ncomment start");//$NON-NLS-1$
	yybegin(ST_XML_COMMENT);
	return XML_COMMENT_OPEN;
 }
        case 302: break;
        case 215: 
        case 216: 
          { 
	if(Debug.debugTokenizer)
		dump("XML processing instruction target");//$NON-NLS-1$
        yybegin(ST_XML_PI_ATTRIBUTE_NAME);
        return XML_TAG_NAME;
 }
        case 303: break;
        case 214: 
          { 
	if(Debug.debugTokenizer)
		dump("comment end");//$NON-NLS-1$
	yybegin(YYINITIAL);
	return XML_COMMENT_CLOSE;
 }
        case 304: break;
        case 213: 
          { 
	if(Debug.debugTokenizer)
		dump("CDATA end");//$NON-NLS-1$
	yybegin(fStateStack.pop());
	return XML_CDATA_CLOSE;
 }
        case 305: break;
        case 212: 
          { 
	if(Debug.debugTokenizer)
		dump("\nPEReference");//$NON-NLS-1$
	return XML_PE_REFERENCE;
 }
        case 306: break;
        case 209: 
          { 
	if(Debug.debugTokenizer)
		dump("\nEntityRef");//$NON-NLS-1$
	return XML_ENTITY_REFERENCE;
 }
        case 307: break;
        case 160: 
        case 174: 
        case 182: 
          { 
	return XML_DOCTYPE_INTERNAL_SUBSET;
 }
        case 308: break;
        case 148: 
          { 
        yybegin(YYINITIAL);
	if(Debug.debugTokenizer)
		dump("empty tag close");//$NON-NLS-1$
        return XML_EMPTY_TAG_CLOSE;
 }
        case 309: break;
        case 133: 
          { 
	if(Debug.debugTokenizer)
		dump("XML processing instruction end");//$NON-NLS-1$
        yybegin(YYINITIAL);
        return XML_PI_CLOSE;
 }
        case 310: break;
        case 132: 
          { 
		// ended with nothing inside
        yybegin(YYINITIAL);
        return XML_PI_CLOSE;
 }
        case 311: break;
        case 128: 
          { 
	if(Debug.debugTokenizer)
		dump("processing instruction end");//$NON-NLS-1$
        yybegin(YYINITIAL);
        return XML_PI_CLOSE;
 }
        case 312: break;
        case 120: 
          { 
	fStateStack.push(yystate());
	if(Debug.debugTokenizer)
		dump("\ndeclaration start");//$NON-NLS-1$
        yybegin(ST_XML_DECLARATION);
	return XML_DECLARATION_OPEN;
 }
        case 313: break;
        case 119: 
          { 
	if(Debug.debugTokenizer)
		dump("\nprocessing instruction start");//$NON-NLS-1$
	yybegin(ST_PI);
        return XML_PI_OPEN;
 }
        case 314: break;
        case 63: 
          { 
	if(Debug.debugTokenizer)
		dump("DHTML processing instruction end");//$NON-NLS-1$
        yybegin(YYINITIAL);
        return XML_PI_CLOSE;
 }
        case 315: break;
        case 57: 
        case 59: 
        case 60: 
        case 61: 
        case 137: 
          { 
	if(Debug.debugTokenizer)
		dump("XML processing instruction attribute value");//$NON-NLS-1$
        yybegin(ST_XML_PI_ATTRIBUTE_NAME);
        return XML_TAG_ATTRIBUTE_VALUE;
 }
        case 316: break;
        case 56: 
          { 
	if(Debug.debugTokenizer)
		dump("XML processing instruction '='");//$NON-NLS-1$
        yybegin(ST_XML_PI_ATTRIBUTE_VALUE);
        return XML_TAG_ATTRIBUTE_EQUALS;
 }
        case 317: break;
        case 55: 
          { 
	if(Debug.debugTokenizer)
		dump("XML processing instruction attribute name");//$NON-NLS-1$
        yybegin(ST_XML_PI_EQUALS);
        return XML_TAG_ATTRIBUTE_NAME;
 }
        case 318: break;
        case 51: 
        case 52: 
        case 53: 
          { 
	// block scan until close is found
	return doScan("?>", false, XML_PI_CONTENT, ST_XML_PI_TAG_CLOSE, ST_XML_PI_TAG_CLOSE);
 }
        case 319: break;
        case 50: 
          { 
        yybegin(ST_PI_CONTENT);
        return WHITE_SPACE;
 }
        case 320: break;
        case 46: 
        case 47: 
        case 48: 
        case 49: 
        case 129: 
        case 130: 
        case 131: 
        case 217: 
        case 239: 
        case 240: 
        case 252: 
        case 253: 
        case 262: 
        case 272: 
        case 280: 
        case 284: 
        case 287: 
        case 288: 
        case 289: 
        case 290: 
          { 
	if(Debug.debugTokenizer)
		dump("processing instruction target");//$NON-NLS-1$
        yybegin(ST_PI_WS);
        return XML_TAG_NAME;
 }
        case 321: break;
        case 41: 
        case 42: 
          { 
	if(Debug.debugTokenizer)
		dump("comment content");//$NON-NLS-1$
	return scanXMLCommentText();
 }
        case 322: break;
        case 40: 
          { 
	if(Debug.debugTokenizer)
		dump("LINE FEED");//$NON-NLS-1$
	return WHITE_SPACE;
 }
        case 323: break;
        case 0: 
        case 31: 
        case 122: 
        case 124: 
        case 207: 
        case 208: 
        case 236: 
          { 
	if(Debug.debugTokenizer)
		dump("\nXML content");//$NON-NLS-1$
	return XML_CONTENT;
 }
        case 324: break;
        case 5: 
        case 8: 
        case 9: 
        case 10: 
        case 12: 
        case 13: 
        case 14: 
        case 15: 
        case 17: 
        case 18: 
        case 19: 
        case 20: 
        case 21: 
        case 22: 
        case 23: 
        case 24: 
        case 25: 
        case 26: 
        case 28: 
        case 45: 
          { 
	if(Debug.debugTokenizer)
		dump("white space");//$NON-NLS-1$
        return WHITE_SPACE;
 }
        case 325: break;
        case 16: 
        case 71: 
          { 
	if(Debug.debugTokenizer)
		dump("inappropriate tag name");//$NON-NLS-1$
	yybegin(YYINITIAL);
        return XML_CONTENT;
 }
        case 326: break;
        case 27: 
        case 106: 
        case 107: 
        case 193: 
        case 229: 
        case 248: 
        case 259: 
        case 269: 
        case 277: 
        case 281: 
          { 
	if(Debug.debugTokenizer)
		dump("elementdecl contentspec");//$NON-NLS-1$
	return XML_ELEMENT_DECL_CONTENT;
 }
        case 327: break;
        case 29: 
        case 113: 
        case 114: 
        case 204: 
        case 233: 
        case 250: 
        case 260: 
        case 270: 
        case 278: 
        case 282: 
          { 
	if(Debug.debugTokenizer)
		dump("attlist contentspec");//$NON-NLS-1$
	return XML_ATTLIST_DECL_CONTENT;
 }
        case 328: break;
        case 32: 
        case 72: 
        case 83: 
          { 
	if(Debug.debugTokenizer)
		dump("\nstart tag open");//$NON-NLS-1$
        yybegin(ST_XML_TAG_NAME);
        return XML_TAG_OPEN;
 }
        case 329: break;
        case 33: 
        case 34: 
        case 37: 
        case 38: 
        case 39: 
        case 43: 
        case 44: 
        case 54: 
        case 58: 
        case 62: 
        case 64: 
        case 68: 
        case 74: 
        case 80: 
        case 85: 
        case 86: 
        case 87: 
        case 88: 
        case 90: 
        case 91: 
        case 93: 
        case 98: 
        case 103: 
        case 110: 
          { 
	if (Debug.debugTokenizer)
		System.out.println("!!!unexpected!!!: \"" + yytext() + "\":" + //$NON-NLS-2$//$NON-NLS-1$
			yychar + "-" + (yychar + yylength()));//$NON-NLS-1$
	return UNDEFINED;
 }
        case 330: break;
        case 35: 
        case 36: 
          { 
	if(Debug.debugTokenizer)
		dump("CDATA text");//$NON-NLS-1$
	String blockContext = doBlockScan("]]>", XML_CDATA_TEXT, ST_CDATA_END);//$NON-NLS-1$
	if(blockContext == XML_CDATA_TEXT)
		yybegin(ST_CDATA_END);
	return blockContext;
 }
        case 331: break;
        case 65: 
          { 
	if(Debug.debugTokenizer)
		dump("DHTML processing instruction attribute name");//$NON-NLS-1$
        yybegin(ST_DHTML_EQUALS);
        return XML_TAG_ATTRIBUTE_NAME;
 }
        case 332: break;
        case 66: 
          { 
	if(Debug.debugTokenizer)
		dump("DHTML processing instruction '='");//$NON-NLS-1$
        yybegin(ST_DHTML_ATTRIBUTE_VALUE);
        return XML_TAG_ATTRIBUTE_EQUALS;
 }
        case 333: break;
        case 67: 
        case 69: 
        case 70: 
        case 145: 
          { 
	if(Debug.debugTokenizer)
		dump("DHTML processing instruction attribute value");//$NON-NLS-1$
        yybegin(ST_DHTML_ATTRIBUTE_NAME);
        return XML_TAG_ATTRIBUTE_VALUE;
 }
        case 334: break;
        case 73: 
          { 
	if(Debug.debugTokenizer)
		dump("tag close");//$NON-NLS-1$
	if(isBlockMarker()) {
        	yybegin(ST_BLOCK_TAG_SCAN);
	}
	else
        	yybegin(YYINITIAL);
        return XML_TAG_CLOSE;
 }
        case 335: break;
        case 75: 
        case 76: 
          { 
	if(Debug.debugTokenizer)
		dump("tag name");//$NON-NLS-1$
        yybegin(ST_XML_ATTRIBUTE_NAME);
        return XML_TAG_NAME;
 }
        case 336: break;
        case 77: 
          { 
	if(Debug.debugTokenizer)
		dump("attr name");//$NON-NLS-1$
        yybegin(ST_XML_EQUALS);
        return XML_TAG_ATTRIBUTE_NAME;
 }
        case 337: break;
        case 78: 
          { 
	if(Debug.debugTokenizer)
		dump("equals");//$NON-NLS-1$
        yybegin(ST_XML_ATTRIBUTE_VALUE);
        return XML_TAG_ATTRIBUTE_EQUALS;
 }
        case 338: break;
        case 79: 
        case 81: 
        case 82: 
        case 152: 
          { 
	if(Debug.debugTokenizer)
		dump("attr value");//$NON-NLS-1$
        yybegin(ST_XML_ATTRIBUTE_NAME);
        return XML_TAG_ATTRIBUTE_VALUE;
 }
        case 339: break;
        case 84: 
          { 
	if(Debug.debugTokenizer)
		dump("declaration end");//$NON-NLS-1$
	if (Debug.debugTokenizer) {
		if(fStateStack.peek()!=YYINITIAL)
			System.out.println("end embedded region");//$NON-NLS-1$
	}
	yybegin(fStateStack.pop());
	return XML_DECLARATION_CLOSE;
 }
        case 340: break;
        case 89: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype type");//$NON-NLS-1$
	yybegin(ST_XML_DOCTYPE_EXTERNAL_ID);
	return XML_DOCTYPE_NAME;
 }
        case 341: break;
        case 92: 
        case 94: 
        case 95: 
        case 96: 
        case 166: 
        case 167: 
        case 170: 
        case 171: 
        case 224: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype public reference");//$NON-NLS-1$
	yybegin(ST_XML_DOCTYPE_ID_SYSTEM);
	return XML_DOCTYPE_EXTERNAL_ID_PUBREF;
 }
        case 342: break;
        case 97: 
        case 99: 
        case 100: 
        case 101: 
        case 178: 
          { 
	if(Debug.debugTokenizer)
		dump("doctype system reference");//$NON-NLS-1$
	yybegin(ST_XML_DECLARATION_CLOSE);
	return XML_DOCTYPE_EXTERNAL_ID_SYSREF;
 }
        case 343: break;
        case 102: 
        case 104: 
        case 105: 
        case 186: 
        case 187: 
        case 190: 
        case 191: 
        case 227: 
          { 
	if(Debug.debugTokenizer)
		dump("elementdecl name");//$NON-NLS-1$
	yybegin(ST_XML_ELEMENT_DECLARATION_CONTENT);
	return XML_ELEMENT_DECL_NAME;
 }
        case 344: break;
        case 108: 
          { 
	if(Debug.debugTokenizer)
		dump("elementdecl close");//$NON-NLS-1$
	if (Debug.debugTokenizer) {
		if(fStateStack.peek()!=YYINITIAL)
			System.out.println("end embedded region");//$NON-NLS-1$
	}
	yybegin(fStateStack.pop());
	return XML_DECLARATION_CLOSE;
 }
        case 345: break;
        case 109: 
        case 111: 
        case 112: 
        case 197: 
        case 198: 
        case 201: 
        case 202: 
        case 231: 
          { 
	if(Debug.debugTokenizer)
		dump("attlist name");//$NON-NLS-1$
	yybegin(ST_XML_ATTLIST_DECLARATION_CONTENT);
	return XML_ATTLIST_DECL_NAME;
 }
        case 346: break;
        case 115: 
          { 
	if(Debug.debugTokenizer)
		dump("attlist close");//$NON-NLS-1$
	if (Debug.debugTokenizer) {
		if(fStateStack.peek()!=YYINITIAL)
			System.out.println("end embedded region");//$NON-NLS-1$
	}
	yybegin(fStateStack.pop());
	return XML_DECLARATION_CLOSE;
 }
        case 347: break;
        case 118: 
          { 
	if(Debug.debugTokenizer)
		dump("\nend tag open");//$NON-NLS-1$
        yybegin(ST_XML_TAG_NAME);
        return XML_END_TAG_OPEN;
 }
        case 348: break;
        case 116: 
        case 117: 
          { 
		return doBlockTagScan();
	 }
        case 349: break;
        default: 
          if (yy_input == YYEOF && yy_startRead == yy_currentPos) {
            yy_atEOF = true;
            yy_do_eof();
              return null;
          } 
          else {
            yy_ScanError(YY_NO_MATCH);
          }
      }
    }
  }    


}