merged from HEAD: JavaScriptStreamTokenTestCase + encodiing fixes

diff --git a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/formatter/tests/AllTests.java b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/formatter/tests/AllTests.java
index 898a4e8..5847293 100644
--- a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/formatter/tests/AllTests.java
+++ b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/formatter/tests/AllTests.java

@@ -17,6 +17,7 @@
 
 import org.eclipse.dltk.javascript.parser.tests.ANTLRParserTestCase;
 import org.eclipse.dltk.javascript.parser.tests.JavaScriptParserTestCase;
+import org.eclipse.dltk.javascript.parser.tests.JavaScriptStreamTokenTestCase;
 
 public class AllTests {
 
@@ -39,6 +40,7 @@
 		suite.addTest(RegExpTest.suite());
 		suite.addTest(KeywordsTest.suite());
 		suite.addTest(E4XTest.suite());
+		suite.addTestSuite(JavaScriptStreamTokenTestCase.class);
 		suite.addTestSuite(JavaScriptParserTestCase.class);
 		suite.addTestSuite(ANTLRParserTestCase.class);
 		// suite.addTest(JavaScriptLibTest.suite());

diff --git a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/internal/parser/tests/JavaScriptTokenStreamTester.java b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/internal/parser/tests/JavaScriptTokenStreamTester.java
index 2aefd2d..d139333 100644
--- a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/internal/parser/tests/JavaScriptTokenStreamTester.java
+++ b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/internal/parser/tests/JavaScriptTokenStreamTester.java

@@ -16,9 +16,10 @@
 import java.io.IOException;
 import java.util.List;
 
+import junit.framework.Assert;
+
 import org.antlr.runtime.ANTLRReaderStream;
 import org.antlr.runtime.CommonTokenStream;
-import org.eclipse.core.runtime.Assert;
 import org.eclipse.dltk.ast.declarations.ModuleDeclaration;
 import org.eclipse.dltk.compiler.problem.IProblem;
 import org.eclipse.dltk.compiler.problem.IProblemReporter;
@@ -30,9 +31,14 @@
 public class JavaScriptTokenStreamTester extends AbstractTester {
 
 	public static void tokenize(String resourceName) throws IOException {
+		tokenize(resourceName, JavaScriptFormatterTestsPlugin.CONTEXT
+				.getCharset());
+	}
 
-		String source = getScriptContent(resourceName,
-				JavaScriptFormatterTestsPlugin.CONTEXT.getCharset());
+	public static void tokenize(String resourceName, String charset)
+			throws IOException {
+
+		String source = getScriptContent(resourceName, charset);
 
 		JSLexer lexer = new JSLexer(new ANTLRReaderStream(new CharArrayReader(
 				source.toCharArray())));
@@ -42,7 +48,7 @@
 
 		List tokens = stream.getTokens();
 
-		Assert.isTrue(tokens.size() > 0);
+		Assert.assertTrue(tokens.size() > 0);
 
 		// for (int i = 0; i < tokens.size(); i++) {
 		// Token tk = (Token) tokens.get(i);
@@ -51,8 +57,12 @@
 	}
 
 	public static void parse(String resourceName) throws IOException {
-		String source = getScriptContent(resourceName,
-				JavaScriptFormatterTestsPlugin.CONTEXT.getCharset());
+		parse(resourceName, JavaScriptFormatterTestsPlugin.CONTEXT.getCharset());
+	}
+
+	public static void parse(String resourceName, String charset)
+			throws IOException {
+		String source = getScriptContent(resourceName, charset);
 
 		JavaScriptParser parser = new JavaScriptParser();
 
@@ -69,7 +79,7 @@
 					}
 				});
 
-		Assert.isNotNull(root);
+		Assert.assertNotNull(root);
 
 		// System.out.println(((Script) root).toSourceString(""));
 	}

diff --git a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/parser/tests/JavaScriptStreamTokenTestCase.java b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/parser/tests/JavaScriptStreamTokenTestCase.java
index 2715ddc..19e781e 100644
--- a/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/parser/tests/JavaScriptStreamTokenTestCase.java
+++ b/tests/org.eclipse.dltk.javascript.formatter.tests/src/org/eclipse/dltk/javascript/parser/tests/JavaScriptStreamTokenTestCase.java

@@ -18,14 +18,15 @@
 
 import org.eclipse.dltk.javascript.internal.parser.tests.JavaScriptTokenStreamTester;
 
+@SuppressWarnings("nls")
 public class JavaScriptStreamTokenTestCase extends TestCase {
 
 	public void testE4XTokenizer() throws IOException {
-		JavaScriptTokenStreamTester.tokenize("e4x.js");
+		JavaScriptTokenStreamTester.tokenize("e4x.js", "UTF-8");
 	}
 
 	public void testE4XParser() throws IOException {
-		JavaScriptTokenStreamTester.parse("e4x.js");
+		JavaScriptTokenStreamTester.parse("e4x.js", "UTF-8");
 	}
 
 	public void testE4XSimpleTokenizer() throws IOException {