documentation/303_runtime_concepts.html - www.eclipse.org/Xtext - Git at Google

 <!DOCTYPE html>
 <html>

   <head>
 	<meta charset="UTF-8">
 	<title>Xtext - Language Implementation</title>

 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
 	<meta name="description"
 		content="The website of Eclipse Xtext, an open-source framework for development of programming languages and domain-specific languages">
 	<meta name="author" content="Sven Efftinge">
 	<meta name="author" content="Miro Spoenemann">
 	<!--  styles -->
 	<!-- Le HTML5 shim, for IE6-8 support of HTML5 elements -->
 	<!--[if lt IE 9]>
 	  <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
 	<![endif]-->

 	<!-- Le fav and touch icons -->
 	<link rel="shortcut icon" href="/Xtext/images/favicon.png">

 	<link href="/Xtext/css/bootstrap.css" rel="stylesheet" type='text/css'>
 	<link href="/Xtext/css/bootstrap-responsive.css" rel="stylesheet" type='text/css'>
 	<link href="/Xtext/css/shield-responsive.css" rel="stylesheet" type='text/css'>
 	<link href='/Xtext/css/fonts.css' rel='stylesheet' type='text/css'>
 	<link href="/Xtext/css/prettyPhoto.css" rel="stylesheet" media="screen" type='text/css'>
 	<link href="/Xtext/css/prettify.css" type="text/css" rel="stylesheet"/>
 	<link href="/Xtext/css/style.css" rel="stylesheet" type='text/css'>
 	<!-- cover flow -->
 	<link href="/Xtext/css/coverflow.css" rel="stylesheet" type='text/css'>
 	<!--[if lt IE 9]>
 	  <link href="/css/iebugs.css" rel="stylesheet" type='text/css'>
 	<![endif]-->

 	<!-- BEGIN Cookie Consent
 	<link rel="stylesheet" type="text/css" href="//cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.0.3/cookieconsent.min.css" />
 	<script src="//cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.0.3/cookieconsent.min.js"></script>
 	<script>
 	window.addEventListener("load", function(){
 	window.cookieconsent.initialise({
 		"palette": {
 		"popup": {
 		  "background": "#000"
 		},
 		"button": {
 		  "background": "#f1d600"
 		}
 		},
 		"theme": "edgeless",
 		"type": "opt-in",
 		onInitialise: function (status) {
 		  var type = this.options.type;
 		  var didConsent = this.hasConsented();
 		  if (type == 'opt-in' && didConsent) {
 		    // TODO: enable cookies
 		  }
 		  if (type == 'opt-out' && !didConsent) {
 		    // TODO: disable cookies
 		  }
 		},
 		onStatusChange: function(status, chosenBefore) {
 		  var type = this.options.type;
 		  var didConsent = this.hasConsented();
 		  if (type == 'opt-in' && didConsent) {
 		    // TODO: enable cookies
 		  }
 		  if (type == 'opt-out' && !didConsent) {
 		    // TODO: disable cookies
 		  }
 		},
 		onRevokeChoice: function() {
 		  var type = this.options.type;
 		  if (type == 'opt-in') {
 		    // TODO: disable cookies
 		  }
 		  if (type == 'opt-out') {
 		    // TODO: enable cookies
 		  }
 		},
 		"content": {
 		"href": "http://www.eclipse.org/legal/privacy.php"
 		}
 	})});
 	</script>
 	END Cookie Consent -->
 </head>


   <body>

     <header class="site-header">

   <!-- Navbar -->
   <div class="navbar navbar-fixed-top">
     <div class="navbar-inner">
       <div class="container">
         <a class="btn btn-navbar" data-toggle="collapse"
           data-target=".nav-collapse"> <span class="icon-bar"></span> <span
           class="icon-bar"></span> <span class="icon-bar"></span>
         </a> <a class="brand" href="/Xtext/index.html"></a>
         <div class="nav-collapse collapse" style="height: 0px;">
           <ul class="nav">
             <!--li ><a href="/Xtext/news.html">News</a></li-->
             <li ><a href="/Xtext/download.html">Download</a></li>
             <li ><a href="/Xtext/documentation/index.html">Documentation</a></li>
             <li ><a href="/Xtext/community.html">Community</a></li>
             <li class="dropdown">
               <a class="dropdown-toggle" data-toggle="dropdown" href="#">Support &amp; Trainings<span class="caret"></span></a>
               <ul class="dropdown-menu">
                 <li><a href="https://www.itemis.com/en/xtext/support-and-team/" target="_blank">itemis</a></li>
                 <li><a href="https://www.typefox.io/language-engineering/" target="_blank">TypeFox</a></li>
               </ul>
             </li>
             <li ><a href="http://xtend-lang.org">Xtend</a></li>
           </ul>
           <!--div class="nav pull-right">
             <li ><a><iframe src="https://ghbtns.com/github-btn.html?user=eclipse&repo=xtext&type=star&count=true" frameborder="0" scrolling="0" width="170px" height="20px"></iframe></a></li>
           </div-->
         </div>
         <!--/.nav-collapse -->
       </div>
     </div>
   </div>
   <!-- Navbar End -->

 </header>


     <div class="page-content">
   <script>
     function startSearch(event) {
         if (event.keyCode == 13) {
             var q = 'site:eclipse.org/Xtext/documentation+' + event.target.value;
             window.open('https://www.google.com/search?q=' + q, "_self");
         }
     }
   </script>
   <div class="wrapper">
   	<div id="page">
       <div class="inner">
         <div id="maincontainer" class="container">
           <span class="edit-on-github pull-right">
             <a href="https://github.com/eclipse/xtext/edit/website-published/xtext-website/documentation/303_runtime_concepts.md">Edit on Github</a>
           </span>
           <div class="span3" style="margin-left: 0px;">
             <div class="search-bar">
               <img src="/Xtext/images/search-gray.png"/>
               <input type="search" id="google-search" onkeyup="startSearch(event);"/>
             </div>
             <ul id="nav-outline" style="margin-left: 0px;">
 	<li class="nav-part">Getting Started</li>
 	<li><a href="102_domainmodelwalkthrough.html">15 Minutes Tutorial</a></li>
 	<li><a href="103_domainmodelnextsteps.html">15 Minutes Tutorial - Extended</a></li>
 	<li><a href="104_jvmdomainmodel.html">Five simple steps to your JVM language</a></li>

 	<li class="nav-part">Reference Documentation</li>
 	<li><a href="301_grammarlanguage.html">The Grammar Language</a></li>
 	<li><a href="302_configuration.html">Configuration</a></li>
 	<li><a href="303_runtime_concepts.html">Language Implementation</a></li>
 	<li><a href="305_xbase.html">Integration with Java</a></li>
 	<li><a href="307_special_languages.html">Typical Language Configurations</a></li>
 	<li><a href="308_emf_integration.html">Integration with EMF</a></li>
 	<li><a href="310_eclipse_support.html">Eclipse Support</a></li>
 	<li><a href="330_web_support.html">Web Editor Support</a></li>
 	<li><a href="340_lsp_support.html">LSP Support</a></li>
 	<li><a href="350_continuous_integration.html">Continuous Integration</a></li>
 </ul>
           </div>
           <div class="span8 doc-contents">
             <h1 id="runtime-concepts">Language Implementation</h1>

 <p>This chapter describes the platform-independent language features that are not covered by the <a href="301_grammarlanguage.html">grammar language</a>.</p>

 <p>Some code examples in this chapter are given in the Xtend language, as it is much better suited for these tasks than Java. Please refer to the <a href="http://www.xtend-lang.org">Xtend documentation</a> for further details. For Java developers it’s extremely easy to learn, as the basics are similar and you only need to understand the additional powerful concepts.</p>

 <h2 id="code-generation">Code Generation</h2>

 <p>Once you have a language you probably want to do something with it. There are two options, you can either write an interpreter that inspects the AST and does something based on that or you translate your language to another programming language or configuration files. In this section we’re going to show how to implement a code generator for an Xtext-based language.</p>

 <h3 id="igenerator2">IGenerator2</h3>

 <p>If you go with the default MWE workflow for your language and you haven’t used Xbase, you are provided with a callback stub that implements <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/IGenerator2.java">IGenerator2</a> by extending the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/AbstractGenerator.java">AbstractGenerator</a> base class. It has three methods that are called from the builder infrastructure whenever a DSL file has changed or should be translated otherwise. The three parameters passed in to those method are:</p>

 <ul>
   <li>The resource to be processed</li>
   <li>An instance of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/IFileSystemAccess2.java">IFileSystemAccess2</a></li>
   <li>An instance of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/IGeneratorContext.java">IGeneratorContext</a></li>
 </ul>

 <p>The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/IFileSystemAccess2.java">IFileSystemAccess2</a> API abstracts over the different file systems the code generator may run over. When the code generator is triggered within the incremental build infrastructure in Eclipse the underlying file system is the one provided by Eclipse, and when the code generator is executed outside Eclipse, say in a headless build, it is <code>java.io.File</code>.</p>

 <p>A very simple implementation of a code generator for the <a href="https://github.com/eclipse/xtext-eclipse/blob/master/org.eclipse.xtext.xtext.ui.examples/projects/fowlerdsl/org.eclipse.xtext.example.fowlerdsl/src/org/eclipse/xtext/example/fowlerdsl/Statemachine.xtext">state machine example</a> could be the following:</p>

 <pre><code class="language-xtend">class StatemachineGenerator extends AbstractGenerator {

     override doGenerate(Resource resource, IFileSystemAccess2 fsa, IGeneratorContext context) {
         fsa.generateFile("relative/path/AllTheStates.txt", '''
             «FOR state : resource.allContents.filter(State).toIterable»
                 State «state.name»
             «ENDFOR»
         ''')
     }

 }
 </code></pre>

 <h3 id="output-configurations">Output Configurations</h3>

 <p>You don’t want to deal with platform or even installation dependent paths in your code generator, rather you want to be able to configure the code generator with some basic outlet roots where the different generated files should be placed under. This is what output configurations are made for.</p>

 <p>By default every language will have a single outlet, which points to <code>&lt;project-root&gt;/src-gen/</code>. The files that go here are treated as fully derived and will be erased by the compiler automatically when a new file should be generated. If you need additional outlets or want to have a different default configuration, you need to implement the interface <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/generator/IOutputConfigurationProvider.java">IOutputConfigurationProvider</a>. It’s straightforward to understand and the default implementation gives you a good idea about how to implement it.</p>

 <p>With this implementation you lay out the basic defaults which can be changed by users on a workspace or per project level using the preferences.</p>

 <h2 id="validation">Validation</h2>

 <p>Static analysis is one of the most interesting aspects when developing a programming language. The users of your languages will be grateful if they get informative feedback as they type. In Xtext there are basically three different kinds of validation.</p>

 <h3 id="automatic-validation">Automatic Validation</h3>

 <p>Some implementation aspects (e.g. the grammar, scoping) of a language have an impact on what is required for a document or semantic model to be valid. Xtext automatically takes care of this.</p>

 <h4 id="syntactical-validation">Lexer/Parser: Syntactical Validation</h4>

 <p>The syntactical correctness of any textual input is validated automatically by the parser. The error messages are generated by the underlying parser technology. One can use the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/ISyntaxErrorMessageProvider.java">ISyntaxErrorMessageProvider</a> API to customize these messages. Any syntax errors can be retrieved from the Resource using the common EMF API: the <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java"><code>Resource.getErrors()</code></a> and <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java"><code>Resource.getWarnings()</code></a> method invocations.</p>

 <h4 id="linking-validation">Linker: Cross-reference Validation</h4>

 <p>Any broken cross-references can be checked generically. As cross-reference resolution is done lazily (see <a href="#linking">linking</a>), any broken links are resolved lazily as well. If you want to validate whether all links are valid, you will have to navigate through the model so that all installed EMF proxies get resolved. This is done automatically in the editor.</p>

 <p>Similarly to syntax errors, any unresolvable cross-links will be reported and can be obtained through the <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java"><code>Resource.getErrors()</code></a> and <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java"><code>Resource.getWarnings()</code></a> method invocations.</p>

 <h4 id="concrete-syntax-validation">Serializer: Concrete Syntax Validation</h4>

 <p>The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/IConcreteSyntaxValidator.java">IConcreteSyntaxValidator</a> validates all constraints that are implied by a grammar. Meeting these constraints is mandatory for a model to be serialized.</p>

 <p>Example:</p>

 <pre><code class="language-xtext">MyRule:
     ({MySubRule} "sub")? (strVal+=ID intVal+=INT)*;
 </code></pre>

 <p>This implies several constraints:</p>

 <ol>
   <li>Types: only instances of <em>MyRule</em> and <em>MySubRule</em> are allowed for this rule. Subtypes are prohibited, since the parser never instantiates unknown subtypes.</li>
   <li>Features: In case the <em>MyRule</em> and <em>MySubRule</em> have <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EStructuralFeature.java">EStructuralFeatures</a> besides <em>strVal</em> and <em>intVal</em>, only <em>strVal</em> and <em>intVal</em> may have <a href="#transient-values">non-transient values</a>.</li>
   <li>Quantities: The following condition must be true: <code>strVal.size() == intVal.size()</code>.</li>
   <li>Values: It must be possible to <a href="#value-converter">convert all values</a> to valid tokens for the used terminal rules <em>ID</em> and <em>INT</em>.</li>
 </ol>

 <p>The typical use case for the concrete syntax validator is validation in non-Xtext-editors that use an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/XtextResource.java">XtextResource</a>. This is the case when combining GMF and Xtext, for example. Another use case is when the semantic model is modified “manually” (not by the parser) and then serialized again. Since it is very difficult for the serializer to provide meaningful error messages, the concrete syntax validator is executed by default before serialization. A textual Xtext editor itself is <em>not</em> a valid use case. Here, the parser ensures that all syntactical constraints are met. Therefore there is no value in additionally running the concrete syntax validator.</p>

 <p>There are some limitations to the concrete syntax validator which result from the fact that it treats the grammar as declarative, which is something the parser doesn’t always do.</p>

 <ul>
   <li>Grammar rules containing assigned actions (e.g. <code>{MyType.myFeature=current}</code>) are ignored. Unassigned actions (e.g. <code>{MyType}</code>), however, are supported.</li>
   <li>Grammar rules that delegate to one or more rules containing assigned actions via unassigned rule calls are ignored.</li>
   <li>Orders within list-features cannot be validated. e.g. <code>Rule: (foo+=R1 foo+=R2)*</code> implies that <em>foo</em> is expected to contain instances of <em>R1</em> and <em>R2</em> in an alternating order.</li>
 </ul>

 <p>To use concrete syntax validation you can let Guice inject an instance of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/IConcreteSyntaxValidator.java">IConcreteSyntaxValidator</a> and use it directly. Furthermore, there is an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/impl/ConcreteSyntaxEValidator.java">adapter</a> which allows to use the concrete syntax validator as an <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EValidator.java">EValidator</a>. You can, for example, enable it in your runtime module, by adding:</p>

 <pre><code class="language-java">@SingletonBinding(eager = true)
 public Class&lt;? extends ConcreteSyntaxEValidator&gt; bindConcreteSyntaxEValidator() {
     return ConcreteSyntaxEValidator.class;
 }
 </code></pre>

 <p>To customize the error messages please see <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/IConcreteSyntaxDiagnosticProvider.java">IConcreteSyntaxDiagnosticProvider</a> and subclass <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/impl/ConcreteSyntaxDiagnosticProvider.java">ConcreteSyntaxDiagnosticProvider</a>.</p>

 <h3 id="custom-validation">Custom Validation</h3>

 <p>In addition to the afore mentioned kinds of validation, which are more or less done automatically, you can specify additional constraints specific for your Ecore model. The Xtext language generator will provide you two Java classes. The first is an abstract class generated to <em>src-gen/</em> which extends the library class <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/AbstractDeclarativeValidator.java">AbstractDeclarativeValidator</a>. This one just registers the EPackages for which this validator introduces constraints. The other class is a subclass of that abstract class and is generated to the <em>src/</em> folder in order to be edited by you. That is where you put the constraints in.</p>

 <p>The purpose of the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/AbstractDeclarativeValidator.java">AbstractDeclarativeValidator</a> is to allow you to write constraints in a declarative way - as the class name already suggests. That is instead of writing exhaustive if-else constructs or extending the generated EMF switch you just have to add the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/Check.java">Check</a> annotation to any method and it will be invoked automatically when validation takes place. Moreover you can state for what type the respective constraint method is, just by declaring a typed parameter. This also lets you avoid any type casts. In addition to the reflective invocation of validation methods the AbstractDeclarativeValidator provides a few convenient assertions.</p>

 <p>The Check annotation has a parameter that can be used to declare when a check should run: <em>FAST</em> will run whenever a file is modified, <em>NORMAL</em> checks will run when saving the file, and <em>EXPENSIVE</em> checks will run when explicitly validating the file via the menu option. Here is an example written in Java:</p>

 <pre><code class="language-java">public class DomainmodelValidator extends AbstractDomainmodelValidator {

     @Check
     public void checkNameStartsWithCapital(Entity entity) {
         if (!Character.isUpperCase(entity.getName().charAt(0))) {
             warning("Name should start with a capital",
                 DomainmodelPackage.Literals.TYPE__NAME);
         }
     }
 }
 </code></pre>

 <p>You can use the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/IResourceValidator.java">IResourceValidator</a> to validate a given resource programmatically. Example:</p>

 <pre><code class="language-java">@Inject IResourceValidator resourceValidator;

 public void checkResource(Resource resource) {
     List&lt;Issue&gt; issues = resourceValidator.validate(resource,
             CheckMode.ALL, CancelIndicator.NullImpl);
     for (Issue issue: issues) {
         switch (issue.getSeverity()) {
             case ERROR:
                 System.out.println("ERROR: " + issue.getMessage());
                 break;
             case WARNING:
                 System.out.println("WARNING: " + issue.getMessage());
                 break;
             default: // do nothing
         }
     }
 }
 </code></pre>

 <p>You can also implement quick fixes for individual validation errors and warnings. See the <a href="310_eclipse_support.html#quick-fixes">section on quick fixes</a> for details.</p>

 <h2 id="linking">Linking</h2>

 <p>The linking feature allows for specification of cross-references within an Xtext grammar. The following things are needed for the linking:</p>

 <ol>
   <li>Declaration of a <a href="301_grammarlanguage.html#cross-references">cross-reference in the grammar</a> (or at least in the Ecore model)</li>
   <li>Specification of linking semantics (usually provided via the <a href="#scoping">scoping API</a>)</li>
 </ol>

 <h3 id="lazy-linking">Lazy Linking</h3>

 <p>Xtext uses lazy linking by default and we encourage users to stick to this because it provides many advantages, one of which is improved performance in all scenarios where you don’t have to load all transitively referenced resources. Furthermore it automatically solves situations where one link relies on other links (cyclic linking dependencies are not supported, though).</p>

 <p>When parsing a given input string, say</p>

 <p><code>ref Entity01</code></p>

 <p>the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/linking/lazy/LazyLinker.java">LazyLinker</a> first creates an EMF proxy and assigns it to the corresponding <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EReference.java">EReference</a>. In EMF a proxy is described by a <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.common/src/org/eclipse/emf/common/util/URI.java">URI</a>, which points to the real <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EObject.java">EObject</a>. In the case of lazy linking the stored URI comprises of the context information given at parse time, which is the EObject containing the cross-reference, the actual EReference, the list index (in case it’s a multi-valued cross-reference) and the string which represented the cross-link in the concrete syntax. The latter usually corresponds to the name of the referenced EObject. In EMF a URI consists of information about the resource the EObject is contained in as well as a so called fragment part, which is used to find the EObject within that resource. When an EMF proxy is resolved, the current <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/ResourceSet.java">ResourceSet</a> is asked. The resource set uses the first part to obtain (i.e. load if it is not already loaded) the resource. Then the resource is asked to return the EObject based on the fragment in the URI. The actual cross-reference resolution is done by <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/linking/lazy/LazyLinkingResource.java">LazyLinkingResource.getEObject(String)</a> which receives the fragment and delegates to the implementation of the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/linking/ILinkingService.java">ILinkingService</a>. The default implementation in turn delegates to the <a href="#scoping">scoping API</a>.</p>

 <p>A <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/linking/impl/DefaultLinkingService.java">simple implementation</a> of the linking service is shipped with Xtext and used for any grammar as default. Usually any necessary customization of the linking behavior can best be described using the <a href="#scoping">scoping API</a>.</p>

 <h2 id="scoping">Scoping</h2>

 <p>Using the scoping API one defines which elements are referable by a given reference. For instance, using the entities example a feature contains a cross-reference to a type:</p>

 <pre><code class="language-domainexample">datatype String

 entity HasAuthor {
     author: String
 }
 </code></pre>

 <p>The grammar rule for features looks like this:</p>

 <pre><code class="language-xtext">Feature:
     (many?='many')? name=ID ':' type=[Type];
 </code></pre>

 <p>The grammar declares that for the reference <em>type</em> only instances of the type <em>Type</em> are allowed. However, this simple declaration doesn’t say anything about where to find the type. That is the duty of scopes.</p>

 <p>An <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IScopeProvider.java">IScopeProvider</a> is responsible for providing an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IScope.java">IScope</a> for a given context <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EObject.java">EObject</a> and <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EReference.java">EReference</a>. The returned <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IScope.java">IScope</a> should contain all target candidates for the given object and cross-reference.</p>

 <pre><code class="language-java">public interface IScopeProvider {

 	/**
 	 * Returns a scope for the given context. The scope provides access to the compatible
 	 * visible EObjects for a given reference.
 	 *
 	 * @param context the element from which an element shall be referenced. It doesn't need to be the element
 	 * 		  containing the reference, it is just used to find the most inner scope for given {@link EReference}.
 	 * @param reference the reference for which to get the scope.
 	 * @return {@link IScope} representing the innermost {@link IScope} for the
 	 *         passed context and reference. Note for implementors: The result may not be &lt;code&gt;null&lt;/code&gt;.
 	 *         Return &lt;code&gt;IScope.NULLSCOPE&lt;/code&gt; instead.
 	 */
     IScope getScope(EObject context, EReference reference);

 }
 </code></pre>

 <p>A single <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IScope.java">IScope</a> represents an element of a linked list of scopes. That means that a scope can be nested within an outer scope. Each scope works like a symbol table or a map where the keys are strings and the values are so-called <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescription</a>, which is effectively an abstract description of a real EObject. In order to create IEObjectDescriptions for your model elements, the class <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/Scopes.java">Scopes</a> is very useful.</p>

 <p>To have a concrete example, let’s deal with the following simple grammar.</p>

 <pre><code class="language-xtext">grammar org.xtext.example.mydsl.MyScopingDsl with
                                       org.eclipse.xtext.common.Terminals

 generate myDsl "http://www.xtext.org/example/mydsl/MyScopingDsl"

 Root:
     elements+=Element;

 Element:
     'element' name=ID ('extends' superElement=[Element])?;

 </code></pre>

 <p>If you want to define the scope for the <em>superElement</em> cross-reference, the following Java code is one way to go.</p>

 <pre><code class="language-java">@Override
 public IScope getScope(EObject context, EReference reference) {
     // We want to define the Scope for the Element's superElement cross-reference
     if (context instanceof Element
             &amp;&amp; reference == MyDslPackage.Literals.ELEMENT__SUPER_ELEMENT) {
         // Collect a list of candidates by going through the model
         // EcoreUtil2 provides useful functionality to do that
         // For example searching for all elements within the root Object's tree
         EObject rootElement = EcoreUtil2.getRootContainer(context);
         List&lt;Element&gt; candidates = EcoreUtil2.getAllContentsOfType(rootElement, Element.class);
         // Create IEObjectDescriptions and puts them into an IScope instance
         return Scopes.scopeFor(candidates);
     }
     return super.getScope(context, reference);
 }
 </code></pre>

 <p>There are different useful implementations for IScope shipped with Xtext. We want to mention only some of them here.</p>

 <p>The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/MapBasedScope.java">MapBasedScope</a> comes with the efficiency of a map to look up a certain name. If you prefer to deal with Multimaps the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/MultimapBasedScope.java">MultimapBasedScope</a> should work for you. For situations where some elements should be filtered out of an existing scope, the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/FilteringScope.java">FilteringScope</a> is the right way to go. As scopes can be nested, we strongly recommend to use FilteringScope only for leaf scopes without nested scopes.</p>

 <p>Coming back to our example, one possible scenario for the FilteringScope could be to exclude the context element from the list of candidates as it should not be a super-element of itself.</p>

 <pre><code class="language-java">    @Override
     public IScope getScope(EObject context, EReference reference) {
         if (context instanceof Element
                 &amp;&amp; reference == MyDslPackage.Literals.ELEMENT__SUPER_ELEMENT) {
             EObject rootElement = EcoreUtil2.getRootContainer(context);
             List&lt;Element&gt; candidates = EcoreUtil2.getAllContentsOfType(rootElement, Element.class);
             IScope existingScope = Scopes.scopeFor(candidates);
             // Scope that filters out the context element from the candidates list
             return new FilteringScope(existingScope, (e) -&gt; !Objects.equal(e.getEObjectOrProxy(), context));
         }
         return super.getScope(context, reference);
     }
 </code></pre>

 <h3 id="global-scopes">Global Scopes and Resource Descriptions</h3>

 <p>In the simple scoping example above we don’t have references across model files. Neither is there a concept like a namespace which would make scoping a bit more complicated. Basically, every <em>Element</em> declared in the same resource is visible by its name. However, in the real world things are most likely not that simple: What if you want to reuse certain declared elements across different files and you want to share those as library between different users? You would want to introduce some kind of cross-resource reference.</p>

 <p>Defining what is visible from outside the current resource is the responsibility of global scopes. As the name suggests, global scopes are provided by instances of the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IGlobalScopeProvider.java">IGlobalScopeProvider</a>. The data structures (called index) used to store its elements are described in the next section.</p>

 <h4 id="resource-descriptions">Resource and EObject Descriptions</h4>

 <p>In order to make elements of one file referable from another file, you need to export them as part of a so called <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription</a>.</p>

 <p>An IResourceDescription contains information about the resource itself, which primarily its <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.common/src/org/eclipse/emf/common/util/URI.java">URI</a>, a list of exported <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EObject.java">EObjects</a> in the form of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescription</a>s, as well as information about outgoing cross-references and qualified names it references. The cross-references contain only resolved references, while the list of imported qualified names also contains the names that couldn’t be resolved. This information is leveraged by the indexing infrastructure of Xtext in order to compute the transitive hull of dependent resources.</p>

 <p>For users, and especially in the context of scoping, the most important information is the list of exported EObjects. An IEObjectDescription stores the URI of the actual EObject, its <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/QualifiedName.java">QualifiedName</a>, as well as its <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EClass.java">EClass</a>. In addition one can export arbitrary information using the <em>user data</em> map. The following diagram gives an overview on the description classes and their relationships.</p>

 <p><img src="images/index_datamodel.png" alt="The data model of Xtext's index" /></p>

 <p>A language is configured with default implementations of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription.Manager</a> and <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/impl/DefaultResourceDescriptionStrategy.java">DefaultResourceDescriptionStrategy</a>, which are responsible to compute the list of exported <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescriptions</a>. The Manager iterates over the whole EMF model for each <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java">Resource</a> and asks the ResourceDescriptionStrategy to compute an IEObjectDescription for each EObject. The ResourceDescriptionStrategy applies the <code>getQualifiedName(EObject obj)</code> from <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/IQualifiedNameProvider.java">IQualifiedNameProvider</a> on the object, and if it has a qualified name an IEObjectDescription is created and passed back to the Manager which adds it to the list of exported objects. If an EObject doesn’t have a qualified name, the element is considered to be not referable from outside the resource and consequently not indexed. If you don’t like this behavior, you can implement and bind your own implementation of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IDefaultResourceDescriptionStrategy.java">IDefaultResourceDescriptionStrategy</a>.</p>

 <p>There are two different default implementations of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/IQualifiedNameProvider.java">IQualifiedNameProvider</a>. Both work by looking up an <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EAttribute.java">EAttribute</a> ‘<em>name</em>’. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/SimpleNameProvider.java">SimpleNameProvider</a> simply returns the plain value, while the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/DefaultDeclarativeQualifiedNameProvider.java">DefaultDeclarativeQualifiedNameProvider</a> concatenates the simple name with the qualified name of its parent exported EObject. This effectively simulates the qualified name computation of most namespace-based languages such as Java. It also allows to override the name computation declaratively: Just add methods named <code>qualifiedName</code> in a subclass and give each of them one argument with the type of element you wish to compute a name for.</p>

 <p>As already mentioned, the default implementation strategy exports every model element that the IQualifiedNameProvider can provide a name for. This is a good starting point, but when your models become bigger and you have a lot of them the index will become larger and larger. In most scenarios only a small part of your model should be visible from outside, hence only that small part needs to be in the index. In order to do this, bind a custom implementation of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IDefaultResourceDescriptionStrategy.java">IDefaultResourceDescriptionStrategy</a> and create index representations only for those elements that you want to reference from outside the resource they are contained in. From within the resource, references to those filtered elements are still possible as long as they have a name.</p>

 <p>Beside the exported elements the index contains <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IReferenceDescription.java">IReferenceDescription</a>s that contain the information who is referencing who. They are created through the IResourceDescription.Manager and IDefaultResourceDescriptionStrategy, too. If there is a model element that references another model element, the IDefaultResourceDescriptionStrategy creates an IReferenceDescription that contains the URI of the referencing element (<em>sourceEObjectURI</em>) and the referenced element (<em>targetEObjectURI</em>). These IReferenceDescriptions are very useful to find references and calculate affected resources.</p>

 <p>As mentioned above, in order to compute an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription</a> for a resource the framework asks the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription.Manager</a> which delegates to the IDefaultResourceDescriptionStrategy. To convert between a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/QualifiedName.java">QualifiedName</a> and its String representation you can use the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/naming/IQualifiedNameConverter.java">IQualifiedNameConverter</a>. Here is some Xtend code showing how to do that:</p>

 <pre><code class="language-xtend">@Inject IResourceServiceProvider.Registry rspr
 @Inject IQualifiedNameConverter converter

 def void printExportedObjects(Resource resource) {
     val resServiceProvider = rspr.getResourceServiceProvider(resource.URI)
     val manager = resServiceProvider.getResourceDescriptionManager()
     val description = manager.getResourceDescription(resource)
     for (eod : description.exportedObjects) {
         println(converter.toString(eod.qualifiedName))
     }
 }
 </code></pre>

 <p>In order to obtain a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">Manager</a> it is best to ask the corresponding <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceServiceProvider.java">IResourceServiceProvider</a> as shown above. That is because each language might have a totally different implementation, and as you might refer from one language to a different language you cannot reuse the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">Manager</a> of the first language.</p>

 <p>Now that we know how to export elements to be referable from other resources, we need to learn how those exported <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescriptions</a> can be made available to the referencing resources. That is the responsibility of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IGlobalScopeProvider.java">global scoping</a> which is described in the following section.</p>

 <p>If you would like to see what’s in the index, you could use the ‘Open Model Element’ dialog from the navigation menu entry.</p>

 <h4 id="index-based">Global Scopes Based On External Configuration</h4>

 <p>Instead of explicitly referring to imported resources, another option is to have some kind of external configuration in order to define what is visible from outside a resource. Java for instance uses the notion of the class path to define containers (jars and class folders) which contain referenceable elements. In the case of Java the order of such entries is also important.</p>

 <p>To enable support for this kind of global scoping in Xtext, a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/DefaultGlobalScopeProvider.java">DefaultGlobalScopeProvider</a> has to be bound to the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IGlobalScopeProvider.java">IGlobalScopeProvider</a> interface. By default Xtext leverages the class path mechanism since it is well designed and already understood by most of our users. The available tooling provided by JDT and PDE to configure the class path adds even more value. However, it is just a default: you can reuse the infrastructure without using Java and be independent of the JDT.</p>

 <p>In order to know what is available in the “world”, a global scope provider which relies on external configuration needs to read that configuration in and be able to find all candidates for a certain <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EReference.java">EReference</a>. If you don’t want to force users to have a folder and file name structure reflecting the actual qualified names of the referenceable <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EObject.java">EObjects</a>, you’ll have to load all resources up front and either keep holding them in memory or remember all information which is needed for the resolution of cross-references. In Xtext that information is provided by a so-called <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescription</a>.</p>

 <h5 id="containers">About the Index, Containers and Their Manager</h5>

 <p>Xtext ships with an index which remembers all <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription</a> and their <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescription</a> objects. In the IDE-context (i.e. when running the editor, etc.) the index is updated by an incremental project builder. As opposed to that, in a non-UI context you typically do not have to deal with changes, hence the infrastructure can be much simpler. In both situations the global index state is held by an implementation of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescriptions.java">IResourceDescriptions</a> (note the plural form!). The bound singleton in the UI scenario is even aware of unsaved editor changes, such that all linking happens to the latest possibly unsaved version of the resources. You will find the Guice configuration of the global index in the UI scenario in <a href="https://github.com/eclipse/xtext-eclipse/blob/master/org.eclipse.xtext.ui.shared/src/org/eclipse/xtext/ui/shared/internal/SharedModule.java">SharedModule</a>.</p>

 <p>The index is basically a flat list of instances of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription</a>. The index itself doesn’t know about visibility constraints due to class path restriction. Rather than that, they are defined by the referencing language by means of so called <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IContainer.java">IContainers</a>: While Java might load a resource via <a href="http://docs.oracle.com/javase/8/docs/api/java/lang/ClassLoader.html">ClassLoader.loadResource()</a> (i.e. using the class path mechanism), another language could load the same resource using the file system paths.</p>

 <p>Consequently, the information which container a resource belongs to depends on the referencing context. Therefore an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceServiceProvider.java">IResourceServiceProvider</a> provides another interesting service, which is called <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IContainer.java">IContainer.Manager</a>. For a given <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescription</a>, the Manager provides you the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IContainer.java">IContainer</a> as well as a list of all IContainers which are visible from there. Note that the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescriptions.java">index</a> is globally shared between all languages while the Manager, which adds the semantics of containers, can be very different depending on the language. The following method lists all resources visible from a given <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/resource/Resource.java">Resource</a>:</p>

 <pre><code class="language-xtend">@Inject IContainer.Manager manager

 def void printVisibleResources(Resource resource, IResourceDescriptions index) {
     val descr = index.getResourceDescription(resource.URI)
     for (visibleContainer : manager.getVisibleContainers(descr, index)) {
         for (visibleResourceDesc : visibleContainer.resourceDescriptions) {
             println(visibleResourceDesc.URI)
         }
     }
 }
 </code></pre>

 <p>Xtext ships two implementations of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IContainer.java">IContainer.Manager</a> which are bound with Guice: The default binding is to <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/impl/SimpleResourceDescriptionsBasedContainerManager.java">SimpleResourceDescriptionsBasedContainerManager</a>, which assumes all <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IResourceDescription.java">IResourceDescriptions</a> to be in a single common container. If you don’t care about container support, you’ll be fine with this one. Alternatively, you can bind <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/containers/StateBasedContainerManager.java">StateBasedContainerManager</a> and an additional <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/containers/IAllContainersState.java">IAllContainersState</a> which keeps track of the set of available containers and their visibility relationships.</p>

 <p>Xtext offers a couple of strategies for managing containers: If you’re running an Eclipse workbench, you can define containers based on Java projects and their class paths or based on plain Eclipse projects. Outside Eclipse, you can provide a set of file system paths to be scanned for models. All of these only differ in the bound instance of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/containers/IAllContainersState.java">IAllContainersState</a> of the referring language. These will be described in detail in the following sections.</p>

 <p><img src="images/index_container.png" alt="IContainer Management" /></p>

 <h5 id="jdt-based-containers">JDT-Based Container Manager</h5>

 <p>As JDT is an Eclipse feature, this JDT-based container management is only available in the UI scenario. It assumes so called <a href="http://help.eclipse.org/luna/topic/org.eclipse.jdt.doc.isv/reference/api/org/eclipse/jdt/core/IPackageFragmentRoot.html">IPackageFragmentRoots</a> as containers. An IPackageFragmentRoot in JDT is the root of a tree of Java model elements. It usually refers to</p>

 <ul>
   <li>a source folder of a Java project,</li>
   <li>a referenced jar,</li>
   <li>a class path entry of a referenced Java project, or</li>
   <li>the exported packages of a required PDE plug-in.</li>
 </ul>

 <p>So for an element to be referable, its resource must be on the class path of the caller’s Java project and it must be exported (as described above).</p>

 <p>As this strategy allows to reuse a lot of nice Java things like jars, OSGi, maven, etc. it is part of the default: You should not have to reconfigure anything to make it work. Nevertheless, if you messed something up, make sure you bind</p>

 <pre><code class="language-java">public Class&lt;? extends IContainer.Manager&gt; bindIContainer$Manager() {
     return StateBasedContainerManager.class;
 }
 </code></pre>

 <p>in the runtime module and</p>

 <pre><code class="language-java">public Provider&lt;IAllContainersState&gt; provideIAllContainersState() {
     return org.eclipse.xtext.ui.shared.Access.getJavaProjectsState();
 }
 </code></pre>

 <p>in the Eclipse UI module of the referencing language. The latter looks a bit more difficult than a common binding, as we have to bind a global singleton to a Guice provider. A <a href="https://github.com/eclipse/xtext-eclipse/blob/master/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/containers/StrictJavaProjectsState.java">StrictJavaProjectsState</a> requires all elements to be on the class path, while the default <a href="https://github.com/eclipse/xtext-eclipse/blob/master/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/containers/JavaProjectsState.java">JavaProjectsState</a> also allows models in non-source folders.</p>

 <h5 id="project-based-containers">Eclipse Project-Based Containers</h5>

 <p>If the class path based mechanism doesn’t work for your case, Xtext offers an alternative container manager based on plain Eclipse projects: Each project acts as a container and the project references (<em>Properties → Project References</em>) are the visible containers.</p>

 <p>In this case, your runtime module should use the StateBasedContainerManager as shown above and the Eclipse UI module should bind</p>

 <pre><code class="language-java">public Provider&lt;IAllContainersState&gt; provideIAllContainersState() {
     return org.eclipse.xtext.ui.shared.Access.getWorkspaceProjectsState();
 }
 </code></pre>

 <h5 id="resource-set-containers">ResourceSet-Based Containers</h5>

 <p>If you need an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IContainer.java">IContainer.Manager</a> that is independent of Eclipse projects, you can use the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/containers/ResourceSetBasedAllContainersState.java">ResourceSetBasedAllContainersState</a>. This one can be configured with a mapping of container handles to resource URIs.</p>

 <h3 id="local-scoping">Local Scoping</h3>

 <p>We now know how the outer world of referenceable elements can be defined in Xtext. Nevertheless, not everything is available in all contexts and with a global name. Rather than that, each context can usually have a different scope. As already stated, scopes can be nested, i.e. a scope can contain elements of a parent scope in addition to its own elements. When parent and child scope contain different elements with the same name, the parent scope’s element will usually be <em>shadowed</em> by the element from the child scope.</p>

 <p>To illustrate that, let’s have a look at Java: Java defines multiple kinds of scopes (object scope, type scope, etc.). For Java one would create the scope hierarchy as commented in the following example:</p>

 <pre><code class="language-java">// file contents scope
 import static my.Constants.STATIC;

 public class ScopeExample { // class body scope
   private Object field = STATIC;

   private void method(String param) { // method body scope
     String localVar = "bar";
     innerBlock: { // block scope
       String innerScopeVar = "foo";
       Object field = innerScopeVar;
       // the scope hierarchy at this point would look like this:
       //  blockScope{field,innerScopeVar}-&gt;
       //  methodScope{localVar, param}-&gt;
       //  classScope{field}-&gt; ('field' is shadowed)
       //  fileScope{STATIC}-&gt;
       //  classpathScope{
       //      'all qualified names of accessible static fields'} -&gt;
       //  NULLSCOPE{}
       //
     }
     field.add(localVar);
   }
 }
 </code></pre>

 <p>In fact the class path scope should also reflect the order of class path entries. For instance:</p>

 <pre><code class="language-java">classpathScope{stuff from bin/}
 -&gt; classpathScope{stuff from foo.jar/}
 -&gt; ...
 -&gt; classpathScope{stuff from JRE System Library}
 -&gt; NULLSCOPE{}
 </code></pre>

 <p>Please find the motivation behind this and some additional details in <a href="http://blog.efftinge.de/2009/01/xtext-scopes-and-emf-index.html">this blog post</a>.</p>

 <h3 id="namespace-imports">Imported Namespace Aware Scoping</h3>

 <p>The imported namespace aware scoping is based on qualified names and namespaces. It adds namespace support to your language, which is comparable and similar to namespaces in Scala and C#. Scala and C# both allow to have multiple nested packages within one file, and you can put imports per namespace, such that imported names are only visible within that namespace. See the domain model example: its scope provider extends <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/ImportedNamespaceAwareLocalScopeProvider.java">ImportedNamespaceAwareLocalScopeProvider</a>.</p>

 <h4 id="importing-namespaces">Importing Namespaces</h4>

 <p>The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/impl/ImportedNamespaceAwareLocalScopeProvider.java">ImportedNamespaceAwareLocalScopeProvider</a> looks up <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EAttribute.java">EAttributes</a> with name <em>importedNamespace</em> and interprets them as import statements.</p>

 <pre><code class="language-xtext">PackageDeclaration:
     'package' name=QualifiedName '{'
         (elements+=AbstractElement)*
     '}';

 AbstractElement:
     PackageDeclaration | Type | Import;

 QualifiedName:
     ID ('.' ID)*;

 Import:
     'import' importedNamespace=QualifiedNameWithWildcard;

 QualifiedNameWithWildcard:
     QualifiedName '.*'?;
 </code></pre>

 <p>Qualified names with or without a wildcard at the end are supported. For an import of a qualified name the simple name is made available as we know from e.g. Java, where <code>import java.util.Set</code> makes it possible to refer to <a href="http://docs.oracle.com/javase/8/docs/api/java/util/Set.html">java.util.Set</a> by its simple name <em>Set</em>. Contrary to Java, the import is not active for the whole file, but only for the namespace it is declared in and its child namespaces. That is why you can write the following in the example DSL:</p>

 <pre><code class="language-domainexample">package foo {
     import bar.Foo
     entity Bar extends Foo {
     }
 }

 package bar {
     entity Foo {}
 }
 </code></pre>

 <p>Of course the declared elements within a package are as well referable by their simple name:</p>

 <pre><code class="language-domainexample">package bar {
     entity Bar extends Foo {}
     entity Foo {}
 }
 </code></pre>

 <h2 id="value-converter">Value Converter</h2>

 <p>Value converters are registered to convert the parsed text into a data type instance and vice versa. The primary hook is the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/IValueConverterService.java">IValueConverterService</a> and the concrete implementation can be registered via the runtime <a href="302_configuration.html#guicemodules">Guice module</a>. Simply override the corresponding binding in your runtime module like shown in this example:</p>

 <pre><code class="language-java">public Class&lt;? extends IValueConverterService&gt; bindIValueConverterService() {
     return MySpecialValueConverterService.class;
 }
 </code></pre>

 <p>The easiest way to register additional value converters is to make use of <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/impl/AbstractDeclarativeValueConverterService.java">AbstractDeclarativeValueConverterService</a>, which allows to declaratively register an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/IValueConverter.java">IValueConverter</a> by means of an annotated method.</p>

 <pre><code class="language-java">@ValueConverter(rule = "MyRuleName")
 public IValueConverter&lt;MyDataType&gt; getMyRuleNameConverter() {
     return new MyValueConverterImplementation();
 }
 </code></pre>

 <p>If you use the common terminals grammar <code>org.eclipse.xtext.common.Terminals</code> you should extend the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/common/services/DefaultTerminalConverters.java">DefaultTerminalConverters</a> and override or add value converters by adding the respective methods. In addition to the explicitly defined converters in the default implementation, a delegating converter is registered for each available <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EDataType.java">EDataType</a>. The delegating converter reuses the functionality of the corresponding EMF <a href="https://git.eclipse.org/r/plugins/gitiles/emf/org.eclipse.emf/+/refs/tags/R2_20_0/plugins/org.eclipse.emf.ecore/src/org/eclipse/emf/ecore/EFactory.java">EFactory</a>.</p>

 <p>Many languages introduce a concept for qualified names, i.e. names composed of namespaces separated by a delimiter. Since this is such a common use case, Xtext provides an extensible converter implementation for qualified names. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/impl/QualifiedNameValueConverter.java">QualifiedNameValueConverter</a> handles comments and white spaces gracefully and is capable to use the appropriate value converter for each segment of a qualified name. This allows for individually quoted segments. The domainmodel example shows how to use it.</p>

 <p>The protocol of an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/IValueConverter.java">IValueConverter</a> allows to throw a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/ValueConverterException.java">ValueConverterException</a> if something went wrong. The exception is propagated as a syntax error by the parser or as a validation problem by the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/validation/impl/ConcreteSyntaxValidator.java">ConcreteSyntaxValidator</a> if the value cannot be converted to a valid string. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/impl/AbstractLexerBasedConverter.java">AbstractLexerBasedConverter</a> is useful when implementing a custom value converter. If the converter needs to know about the rule that it currently works with, it may implement the interface <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/IValueConverter.java">RuleSpecific</a>. The framework will set the rule such that the implementation can use it afterwards.</p>

 <h2 id="serialization">Serialization</h2>

 <p>Serialization is the process of transforming an EMF model into its textual representation. Thereby, serialization complements parsing and lexing.</p>

 <p>In Xtext, the process of serialization is split into the following steps:</p>

 <ol>
   <li>Validating the semantic model. This is optional, enabled by default, done by the <a href="#concrete-syntax-validation">concrete syntax validator</a> and can be turned off in the <a href="#save-options">save options</a>.</li>
   <li>Matching the model elements with the grammar rules and creating a stream of tokens. This is done by the <a href="#parse-tree-constructor">parse tree constructor</a>.</li>
   <li>Associating comments with semantic objects. This is done by the <a href="#comment-associater">comment associator</a>.</li>
   <li>Associating existing nodes from the node model with tokens from the token stream.</li>
   <li><a href="#hidden-token-merger">Merging existing white space</a> and line-wraps into the token stream.</li>
   <li>Adding further needed white space or replacing all white space using a <a href="#formatting">formatter</a>.</li>
 </ol>

 <p>Serialization is invoked when calling <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/XtextResource.java">XtextResource.save(..)</a>. Furthermore, the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/Serializer.java">Serializer</a> provides resource-independent support for serialization. Another situation that triggers serialization is applying <a href="310_eclipse_support.html#quick-fixes">quick fixes</a> with semantic modifications. Serialization is <em>not</em> called when a textual editors contents is saved to disk.</p>

 <h3 id="serialization-contract">The Contract</h3>

 <p>The contract of serialization says that a model which is saved (serialized) to its textual representation and then loaded (parsed) again yields a new model that is equal to the original model. Please be aware that this does <em>not</em> imply that loading a textual representation and serializing it back produces identical textual representations. However, the serialization algorithm tries to restore as much information as possible. That is, if the parsed model was not modified in-memory, the serialized output will usually be equal to the previous input. Unfortunately, this cannot be ensured for each and every case. A use case where is is hardly possible, is shown in the following example:</p>

 <pre><code class="language-xtext">MyRule:
   (xval+=ID | yval+=INT)*;
 </code></pre>

 <p>The given <em>MyRule</em> reads <em>ID</em>- and <em>INT</em>-elements which may occur in an arbitrary order in the textual representation. However, when serializing the model all <em>ID</em>-elements will be written first and then all <em>INT</em>-elements. If the order is important it can be preserved by storing all elements in the same list - which may require wrapping the <em>ID</em>- and <em>INT</em>-elements into other objects.</p>

 <h3 id="roles-of-the-semantic-model-and-the-node-model-during-serialization">Roles of the Semantic Model and the Node Model During Serialization</h3>

 <p>A serialized document represents the state of the semantic model. However, if there is a node model available (i.e. the semantic model has been created by the parser), the serializer</p>

 <ul>
   <li>preserves <a href="#hidden-token-merger">existing white spaces</a> from the node model.</li>
   <li>preserves <a href="#comment-associater">existing comments</a> from the node model.</li>
   <li>preserves the representation of cross-references: If a cross-referenced object can be identified by multiple names (i.e. scoping returns multiple <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/IEObjectDescription.java">IEObjectDescriptions</a> for the same object), the serializer tries to keep the name that was used in the input file.</li>
   <li>preserves the representation of values: For values handled by the <a href="#value-converter">value converter</a>, the serializer checks whether the textual representation converted to a value equals the value from the semantic model. If that is true, the textual representation is kept.</li>
 </ul>

 <h3 id="parse-tree-constructor">Parse Tree Constructor</h3>

 <p>The parse tree constructor usually does not need to be customized since it is automatically derived from the <a href="301_grammarlanguage.html">Xtext Grammar</a>. However, it can be helpful to look into it to understand its error messages and its runtime performance.</p>

 <p>For serialization to succeed, the parse tree constructor must be able to <em>consume</em> every non-transient element of the to-be-serialized EMF model. To <em>consume</em> means, in this context, to write the element to the textual representation of the model. This can turn out to be a not-so-easy-to-fulfill requirement, since a grammar usually introduces implicit constraints to the EMF model as explained for the <a href="#concrete-syntax-validation">concrete syntax validator</a>.</p>

 <p>If a model can not be serialized, an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/XtextSerializationException.java">XtextSerializationException</a> is thrown. Possible reasons are listed below:</p>

 <ul>
   <li>A model element can not be consumed. This can have the following reasons/solutions:
     <ul>
       <li>The model element should not be stored in the model.</li>
       <li>The grammar needs an assignment which would consume the model element.</li>
       <li>The <a href="#transient-values">transient value service</a> can be used to indicate that this model element should not be consumed.</li>
     </ul>
   </li>
   <li>An assignment in the grammar has no corresponding model element. The default transient value service considers a model element to be transient if it is <em>unset</em> or <em>equals</em> its default value. However, the parse tree constructor may serialize default values if this is required by a grammar constraint to be able to serialize another model element. The following solution may help to solve such a scenario:
     <ul>
       <li>A model element should be added to the model.</li>
       <li>The assignment in the grammar should be made optional.</li>
     </ul>
   </li>
   <li>The type of the model element differs from the type in the grammar. The type of the model element must be identical to the return type of the grammar rule or the action’s type. Subtypes are not allowed.</li>
   <li><a href="#value-converter">Value conversion</a> fails. The value converter can indicate that a value is not serializable by throwing a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/conversion/ValueConverterException.java">ValueConverterException</a>.</li>
   <li>An enum literal is not allowed at this position. This can happen if the referenced enum rule only lists a subset of the literals of the actual enumeration.</li>
 </ul>

 <p>To understand error messages and performance issues of the parse tree constructor it is important to know that it implements a backtracking algorithm. This basically means that the grammar is used to specify the structure of a tree in which one path (from the root node to a leaf node) is a valid serialization of a specific model. The parse tree constructor’s task is to find this path - with the condition that all model elements are consumed while walking this path. The parse tree constructor’s strategy is to take the most promising branch first (the one that would consume the most model elements). If the branch leads to a dead end (for example, if a model element needs to be consumed that is not present in the model), the parse tree constructor goes back the path until a different branch can be taken. This behavior has two consequences:</p>

 <ul>
   <li>In case of an error, the parse tree constructor has found only dead ends but no leaf. It cannot tell which dead end is actually erroneous. Therefore, the error message lists dead ends of the longest paths, a fragment of their serialization and the reason why the path could not be continued at this point. The developer has to judge on his own which reason is the actual error.</li>
   <li>For reasons of performance, it is critical that the parse tree constructor takes the most promising branch first and detects wrong branches early. One way to achieve this is to avoid having many rules which return the same type and which are called from within the same alternative in the grammar.</li>
 </ul>

 <h3 id="save-options">Options</h3>

 <p><a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/SaveOptions.java">SaveOptions</a> can be passed to <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/XtextResource.java">XtextResource.save(options)</a> and to <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/Serializer.java">Serializer.serialize(..)</a>. Available options are:</p>

 <ul>
   <li><strong>Formatting.</strong> Default: <code>false</code>. If enabled, it is the <a href="#formatting">formatters</a> job to determine all white space information during serialization. If disabled, the formatter only defines white space information for the places in which no white space information can be preserved from the node model. E.g. When new model elements are inserted or there is no node model.</li>
   <li><strong>Validating.</strong> Default: <code>true</code>: Run the <a href="#concrete-syntax-validation">concrete syntax validator</a> before serializing the model.</li>
 </ul>

 <h3 id="comment-associater">Preserving Comments from the Node Model</h3>

 <p>The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/ICommentAssociater.java">ICommentAssociater</a> associates comments with semantic objects. This is important in case an element in the semantic model is moved to a different position and the model is serialized, one expects the comments to be moved to the new position in the document as well.</p>

 <p>Which comment belongs to which semantic object is surely a very subjective issue. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/impl/DefaultCommentAssociater.java">default implementation</a> behaves as follows, but can be customized:</p>

 <ul>
   <li>If there is a semantic token before a comment and in the same line, the comment is associated with this token’s semantic object.</li>
   <li>In all other cases, the comment is associated with the semantic object of the next following object.</li>
 </ul>

 <h3 id="transient-values">Transient Values</h3>

 <p>Transient values are values or model elements which are not persisted (written to the textual representation in the serialization phase). If a model contains model elements which can not be serialized with the current grammar, it is critical to mark them transient using the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/ITransientValueService.java">ITransientValueService</a>, or serialization will fail. The default implementation marks all model elements transient, which are <code>eStructuralFeature.isTransient()</code> or not <code>eObject.eIsSet(eStructuralFeature)</code>. By default, EMF returns <code>false</code> for <code>eIsSet(..)</code> if the value equals the default value.</p>

 <h3 id="unassigned-text">Unassigned Text</h3>

 <p>If there are calls of data type rules or terminal rules that do not reside in an assignment, the serializer by default doesn’t know which value to use for serialization.</p>

 <p>Example:</p>

 <pre><code class="language-xtext">PluralRule:
   'contents:' count=INT Plural;

 terminal Plural:
   'item' | 'items';
 </code></pre>

 <p>Valid models for this example are <code>contents 1 item</code> or <code>contents 5 items</code>. However, it is not stored in the semantic model whether the keyword <em>item</em> or <em>items</em> has been parsed. This is due to the fact that the rule call <em>Plural</em> is unassigned. However, the <a href="#parse-tree-constructor">parse tree constructor</a> needs to decide which value to write during serialization. This decision can be be made by customizing the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/ITokenSerializer.java">IValueSerializer.serializeUnassignedValue(EObject, RuleCall, INode)</a>.</p>

 <h3 id="cross-reference-serializer">Cross-Reference Serializer</h3>

 <p>The cross-reference serializer specifies which values are to be written to the textual representation for cross-references. This behavior can be customized by implementing <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/ITokenSerializer.java">ICrossReferenceSerializer</a>. The default implementation delegates to various other services such as the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/scoping/IScopeProvider.java">IScopeProvider</a> or the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/linking/impl/LinkingHelper.java">LinkingHelper</a> each of which may be the better place for customization.</p>

 <h3 id="hidden-token-merger">Merge White Space</h3>

 <p>After the <a href="#parse-tree-constructor">parse tree constructor</a> has done its job to create a stream of tokens which are to be written to the textual representation, and the <a href="#comment-associater">comment associator</a> has done its work, the existing white spaces from the node model are merged into the stream.</p>

 <p>The strategy is as follows: If two tokens follow each other in the stream and the corresponding nodes in the node model follow each other as well, then the white space information in between is kept. In all other cases it is up to the <a href="#formatting">formatter</a> to calculate new white space information.</p>

 <h3 id="token-stream">Token Stream</h3>

 <p>The <a href="#parse-tree-constructor">parse tree constructor</a> and the <a href="#formatting">formatter</a> use an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/ITokenStream.java">ITokenStream</a> for their output, and the latter for its input as well. This allows for chaining the two components. Token streams can be converted to a <a href="http://docs.oracle.com/javase/8/docs/api/java/lang/String.html">String</a> using the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/impl/TokenStringBuffer.java">TokenStringBuffer</a> and to a <a href="http://docs.oracle.com/javase/8/docs/api/java/io/Writer.html">Writer</a> using the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/reconstr/impl/WriterTokenStream.java">WriterTokenStream</a>.</p>

 <pre><code class="language-java">public interface ITokenStream {

   void flush() throws IOException;
   void writeHidden(EObject grammarElement, String value);
   void writeSemantic(EObject grammarElement, String value);
 }
 </code></pre>

 <h2 id="formatting">Formatting</h2>

 <p>Formatting (aka. pretty printing) is the process of rearranging the text in a document to improve the readability without changing the semantic value of the document. Therefore a formatter is responsible for arranging line-wraps, indentation, whitespace, etc. in a text to emphasize its structure, but it is not supposed to alter a document in a way that impacts the semantic model.</p>

 <p>The actual formatting is done by constructing a list of text replacements. A text replacement describes a new text which should replace an existing part of the document, described by offset and length. Applying the text replacements turns the unformatted document into a formatted document.</p>

 <p>To invoke the formatter programmatically, you need to instantiate a <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/FormatterRequest.java">request</a> and pass it to the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/IFormatter2.java">formatter</a>. The formatter will return a list of text replacements. The document modification itself can be performed by an <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/regionaccess/ITextRegionRewriter.java">ITextRegionRewriter</a>.</p>

 <p>Implementors of a formatter should extend <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/AbstractFormatter2.java">AbstractFormatter2</a> and add dispatch methods for the model elements that should be formatted. The format routine has to be invoked recursively if the children of an object should be formatted, too.</p>

 <p>The following example illustrates that pattern. An instance of PackageDeclaration is passed to the format method along with the current <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/IFormattableDocument.java">formattable document</a>. In this scenario, the package name is surrounded by a single space, the curly brace is followed by a new line and increased indentation etc. All elements within that package should be formatted, too, thus <code>format(..)</code> is invoked on these as well.</p>

 <pre><code class="language-xtend">def dispatch void format(PackageDeclaration p, extension IFormattableDocument doc) {
     p.regionFor.feature(PACKAGE_DECLARATION__NAME).surround[oneSpace]
     interior(
         p.regionFor.keyword('{').append[newLine],
         p.regionFor.keyword('}'),
         [indent]
     )
     for (element : p.elements) {
         format(element, doc)
         element.append[setNewLines(1, 1, 2)]
     }
 }
 </code></pre>

 <p>The API is designed in a way that allows to describe the formatting in a declarative way by calling methods on the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/IHiddenRegionFormatter.java">IHiddenRegionFormatter</a> which is made available inside invocations of <code>prepend</code>, <code>surround</code> or <code>append</code> to specify the formatting rules. This can be done in arbitrary order – the infrastructure will reorder all the configurations to execute them from top to bottom of the document. If the configuration-based approach is not sufficient for a particular use case, the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/IFormattableDocument.java">document</a> also accepts imperative logic that is associated with a given range. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/ITextReplacer.java">ITextReplacer</a> that can be added directly to the document allows to perform all kinds of modifications to the text in the region that it is associated with.</p>

 <p>More detailed information about the API is available as <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/formatting2/package-info.java">JavaDoc on the org.eclipse.xtext.formatting2 package</a>.</p>

 <h2 id="encoding">Character Encoding</h2>

 <p>Encoding, aka. <em>character set</em>, describes the way characters are encoded into bytes and vice versa. Famous standard encodings are <em>UTF-8</em> or <em>ISO-8859-1</em>. The list of available encodings can be determined by calling <a href="http://docs.oracle.com/javase/8/docs/api/java/nio/charset/Charset.html">Charset.availableCharsets()</a>. There is also a list of encodings and their canonical Java names in the <a href="http://download.oracle.com/javase/1.5.0/docs/guide/intl/encoding.doc.html">API docs</a>.</p>

 <p>Unfortunately, each platform and/or spoken language tends to define its own native encoding, e.g. <em>Cp1258</em> on Windows in Vietnamese or <em>MacIceland</em> on Mac OS X in Icelandic.</p>

 <p>In an Eclipse workspace, files, folders, projects can have individual encodings, which are stored in the hidden file <em>.settings/org.eclipse.core.resources.prefs</em> in each project. If a resource does not have an explicit encoding, it inherits the one from its parent recursively. Eclipse chooses the native platform encoding as the default for the workspace root. You can change the default workspace encoding in the Eclipse preferences <em>Preferences → Workspace → Default text encoding</em>. If you develop on different platforms, you should consider choosing an explicit common encoding for your text or code files, especially if you use special characters.</p>

 <p>While Eclipse allows to define and inspect the encoding of a file, your file system usually doesn’t. Given an arbitrary text file there is no general strategy to tell how it was encoded. If you deploy an Eclipse project as a jar (even a plug-in), any encoding information not stored in the file itself is lost, too. Some languages define the encoding of a file explicitly, as in the first processing instruction of an XML file. Most languages don’t. Others imply a fixed encoding or offer enhanced syntax for character literals, e.g. the unicode escape sequences <em>\uXXXX</em> in Java.</p>

 <p>As Xtext is about textual modeling, it allows to tweak the encoding in various places.</p>

 <h3 id="encoding-at-language-design-time">Encoding at Language Design Time</h3>

 <p>The plug-ins created by the <em>New Xtext Project</em> wizard are by default encoded in the workspace standard encoding. The same holds for all files that Xtext generates in there. If you want to change that, e.g. because your grammar uses/allows special characters, you should manually set the encoding in the properties of these projects after their creation. Do this before adding special characters to your grammar or at least make sure the grammar reads correctly after the encoding change. To tell the Xtext generator to generate files in the same encoding, set the encoding property in the workflow, e.g.</p>

 <pre><code class="language-mwe2">component = XtextGenerator {
     configuration = {
         code = {
             encoding = "UTF-8"
         }
     ...
 </code></pre>

 <h3 id="encoding-at-language-runtime">Encoding at Language Runtime</h3>

 <p>As each language could handle the encoding problem differently, Xtext offers a service here. The <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parser/IEncodingProvider.java">IEncodingProvider</a> has a single method <code>getEncoding(URI)</code> to define the encoding of the resource with the given URI. Users can implement their own strategy, but keep in mind that this is not intended to be a long running method. If the encoding is stored within the model file itself, it should be extractable in an easy way, like from the first line in an XML file. The default implementation returns the default Java character set in a standalone scenario.</p>

 <p>In the Eclipse UI scenario, when there is a workspace, users will expect the encoding of the model files to be settable the same way as for other files in the workspace. The default implementation of the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parser/IEncodingProvider.java">IEncodingProvider</a> in the Eclipse context therefore returns the file’s workspace encoding for files in the workspace and delegates to the runtime implementation for all other resources, e.g. models in a jar or from a deployed plug-in. Keep in mind that you are going to lose the workspace encoding information as soon as you leave this workspace, e.g. deploy your project.</p>

 <p>Unless you want to enforce a uniform encoding for all models of your language, we advise to override the runtime service only. It is bound in the runtime module using the binding annotation <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/service/DispatchingProvider.java">@Runtime</a>:</p>

 <pre><code class="language-java">@Override
 public void configureRuntimeEncodingProvider(Binder binder) {
     binder.bind(IEncodingProvider.class)
         .annotatedWith(DispatchingProvider.Runtime.class)
         .to(MyEncodingProvider.class);
 }
 </code></pre>

 <p>For a uniform encoding, bind the plain <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parser/IEncodingProvider.java">IEncodingProvider</a> to the same implementation in all modules. In the Eclipse UI module you can use similar code as above, but with <em>DispatchingProvider.Ui</em> instead of <em>Runtime</em>.</p>

 <h3 id="encoding-of-an-xtextresource">Encoding of an XtextResource</h3>

 <p>An <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/resource/XtextResource.java">XtextResource</a> uses the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext/src/org/eclipse/xtext/parser/IEncodingProvider.java">IEncodingProvider</a> of your language by default. You can override that by passing an option on load and save, e.g.</p>

 <pre><code class="language-xtend">myXtextResource.load(#{XtextResource.OPTION_ENCODING -&gt; "UTF-8"})
 </code></pre>

 <p>or</p>

 <pre><code class="language-xtend">myXtextResource.save(#{XtextResource.OPTION_ENCODING -&gt; "ISO-8859-1"})
 </code></pre>

 <h2 id="testing">Unit Testing</h2>

 <p>Automated tests are crucial for the maintainability and the quality of a software product. That is why it is strongly recommended to write unit tests for your language, too. The Xtext project wizard creates test projects for that purpose, which simplify the setup procedure for the basic language implementation as well as platform-specific integrations. It supports an option to either create your tests for JUnit 4 or JUnit 5. Depending on your choice your test layout will vary in some details.</p>

 <h3 id="creating-a-simple-test-class">Creating a Simple Test Class</h3>

 <p>The core of the test infrastructure for JUnit 4 is the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/XtextRunner.java">XtextRunner</a> and the language specific <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/IInjectorProvider.java">IInjectorProvider</a>. Both have to be provided by means of class annotations. Your test cases should be annotated with <a href="https://github.com/junit-team/junit4/blob/master/src/main/java/org/junit/Test.java">org.junit.Test</a>. A static import <a href="https://github.com/junit-team/junit4/blob/master/src/main/java/org/junit/Assert.java">org.junit.Assert</a> makes your tests more readable.</p>

 <pre><code class="language-xtend">import org.eclipse.xtext.testing.InjectWith
 import org.eclipse.xtext.testing.XtextRunner
 import org.junit.runner.RunWith
 import org.junit.Test
 import static org.junit.Assert.*
 import org.example.domainmodel.DomainmodelInjectorProvider

 @InjectWith(DomainmodelInjectorProvider)
 @RunWith(XtextRunner)
 class ParserTest {
     @Test def void simple() {
         assertTrue(true)
     }
 }
 </code></pre>

 <p>This configuration will make sure that you can use dependency injection in your test class, and that the global EMF registries are properly populated before and cleaned up after each test.</p>

 <p>A test class for JUnit 5 looks quite similar. Instead of runners JUnit 5 has a notion of <a href="https://junit.org/junit5/docs/current/user-guide/#extensions">Extensions</a>. While there can only be one runner per test class for JUnit 4 there could be multiple extensions for JUnit 5. The replacement for the XtextRunner is the new <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/extensions/InjectionExtension.java">InjectionExtension</a>. Still needed is the language specific <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/IInjectorProvider.java">IInjectorProvider</a>. Instead of <code>org.junit.Test</code> you have to annotate your cases with <a href="https://github.com/junit-team/junit5/blob/master/junit-jupiter-api/src/main/java/org/junit/jupiter/api/Test.java">org.junit.jupiter.api.Test</a> and import the methods from <a href="https://github.com/junit-team/junit5/blob/master/junit-jupiter-api/src/main/java/org/junit/jupiter/api/Assertions.java">org.junit.jupiter.api.Assertions</a>. A simple test class for JUnit 5 will then look like this:</p>

 <pre><code class="language-xtend">import org.eclipse.xtext.testing.InjectWith
 import org.eclipse.xtext.testing.extensions.InjectionExtension
 import org.junit.jupiter.api.^extension.ExtendWith
 import org.junit.jupiter.api.Test
 import static org.junit.jupiter.api.Assertions.*
 import org.example.domainmodel.DomainmodelInjectorProvider

 @InjectWith(DomainmodelInjectorProvider)
 @ExtendWith(InjectionExtension)
 class ParserTest {
     @Test def void simple() {
         assertTrue(true)
     }
 }
 </code></pre>

 <h3 id="testing-the-parser">Testing the Parser</h3>

 <p>The class <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/util/ParseHelper.java">ParseHelper</a> allows to parse an arbitrary string into an AST model. The AST model itself can be traversed and checked afterwards.</p>

 <pre><code class="language-xtend">import org.eclipse.xtext.testing.util.ParseHelper
 ...

 @Inject ParseHelper&lt;Domainmodel&gt; parser

 @Test
 def void parseDomainmodel() {
     val model = parser.parse('''
         entity MyEntity {
             parent: MyEntity
         }
     ''')
     val entity = model.elements.head as Entity
     assertSame(entity, entity.features.head.type)
 }
 </code></pre>

 <h3 id="test-validators">Testing the Validator</h3>

 <p>Testing your validation is very simple with the <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/validation/ValidationTestHelper.java">ValidationTestHelper</a>:</p>

 <pre><code class="language-xtend">...

 @Inject extension ParseHelper
 @Inject extension ValidationTestHelper

 @Test
 def void testLowercaseName() {
     val model = "entity foo {}".parse
     model.assertWarning(DomainmodelPackage.Literals.ENTITY, null,
             "Name should start with a capital")
 }
 </code></pre>

 <p>See the various <em>assert</em> methods in that helper class to explore the testing capabilities. You can either assert that a given model has specific issues as in the example above, or assert that it has no issues.</p>

 <h3 id="testing-multiple-languages">Testing Multiple Languages</h3>

 <p>If in addition to the main language your tests require using other languages for references from/to your main language, you’ll have to parse and load dependent resources into the same ResourceSet first for cross-reference resolution to work.</p>

 <p>As the default generated <a href="https://github.com/eclipse/xtext-core/blob/master/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/IInjectorProvider.java">IInjectorProvider</a> of your main language (e.g. DomainmodelInjectorProvider) does not know about any other dependent languages, they must be initialized explicitly. The recommended pattern for this is to create a new subclass of the generated <em>MyLanguageInjectorProvider</em> in your <em>*.test</em> project and make sure the dependent language is intizialized properly. Then you can use this new injector provider instead of the original one in your test’s <em>@InjectWith</em>:</p>

 <pre><code class="language-java">public class MyLanguageWithDependenciesInjectorProvider extends MyLanguageInjectorProvider {
     @Override
     protected Injector internalCreateInjector() {
         MyOtherLangLanguageStandaloneSetup.doSetup();
         return super.internalCreateInjector();
     }
 }

 // @RunWith(XtextRunner.class) // JUnit 4
 @ExtendWith(InjectionExtension.class) // JUnit 5
 @InjectWith(MyLanguageWithDependenciesInjectorProvider.class)
 public class YourTest {
     ...
 }
 </code></pre>

 <p>You should not put injector creation for referenced languages in your standalone setup. Note that for the headless code generation use case, the Maven plug-in is configured with multiple setups, so usually there is no problem there.</p>

 <p>You may also need to initialize imported Ecore models that are not generated by your Xtext language. This should be done by using an explicit <em>MyModelPackage.eINSTANCE.getName();</em> in the <em>doSetup()</em> method of your respective language’s StandaloneSetup class. Note that it is strongly recommended to follow this pattern instead of just using <em>@Before</em> methods in your *Test class, as due to internal technical reasons that won’t work anymore as soon as you have more than just one <em>@Test</em>.</p>

 <pre><code class="language-java">public class MyLanguageStandaloneSetup extends MyLanguageStandaloneSetupGenerated {

     public static void doSetup() {
         if (!EPackage.Registry.INSTANCE.containsKey(MyPackage.eNS_URI))
             EPackage.Registry.INSTANCE.put(MyPackage.eNS_URI, MyPackage.eINSTANCE);
         new MyLanguageStandaloneSetup().createInjectorAndDoEMFRegistration();
     }

 }
 </code></pre>

 <p>This only applies to referencing dependencies to imported Ecore models and languages based on them which may be used in the test. The inherited dependencies from mixed-in grammars are automatically listed in the generated super class already, and nothing needs to be done for those.</p>

 <p>The Xtext example projects (<em>File → New → Example → Xtext Examples</em>) contain further unit test cases, e.g. testing the formatter, serializer, compiler, …etc . Feel free to study the corresponding <code>org.eclipse.xtext.example.&lt;language&gt;.tests</code> projects to get some inspirations on how to implement automated unit test cases for your Xtext-based language.</p>

 <hr />

 <p><strong><a href="305_xbase.html">Next Chapter: Integration with Java</a></strong></p>

           </div>
         </div>
       </div>
     </div>
   </div>
 </div>

     <footer class="site-footer">
   <div id="extra">
     <div class="inner">
       <div class="container">
         <div class="row">
           <div class="span6">
             <h3 class="footer-links-header">Quick Links</h3>
             <ul class="footer-links clearfix">
               <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li>
               <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li>
               <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li>
               <li><a href="http://www.eclipse.org/legal/">Legal</a></li>
             </ul>
             <ul class="footer-links clearfix">
               <li><a href="http://www.eclipse.org">Eclipse Home</a></li>
               <li><a href="http://marketplace.eclipse.org/">Market Place</a></li>
               <li><a href="http://www.planeteclipse.org/">Eclipse Planet</a></li>
               <li><a href="https://www.eclipse.org/forums/index.php/f/27/">Xtext Forum</a></li>
             </ul>
           </div>
           <div class="span6">
             <!-- Social Media Links -->
             <h3 class="footer-links-header"">Social Media</h3>
             <ul class="footer-links clearfix">
               <li>
                 <a href="https://twitter.com/xtext"><img src="/Xtext/images/Twitter-bird-darkgray.png" class="img-responsive" style="margin-right: 5px;height: 1em;" alt="Twitter icon">@xtext on Twitter</a>
               </li>
             </ul>
           </div>
         </div>
       </div>
     </div>
   </div>
   <a href="#" class="scrollup fadeOutRight animated" style="display: none;">ScrollUp</a>
   <!-- Le javascript
       ================================================== -->
   <!-- Placed at the end of the document so the pages load faster -->

   <script src="/Xtext/js/jquery-1.11.3.min.js"></script>
   <script src="/Xtext/js/bootstrap.min.js"></script>
   <script src="/Xtext/js/jquery.easing.1.3.js" type="text/javascript"></script>
   <script src="/Xtext/js/jquery.prettyPhoto.js" type="text/javascript"></script>
   <script src="/Xtext/js/prettify.js" type="text/javascript"></script>
   <script src="/Xtext/js/lang-xtend.js" type="text/javascript"></script>
   <script src="/Xtext/js/lang-common.js" type="text/javascript"></script>
   <script src="/Xtext/js/custom.js" type="text/javascript"></script>
   <!--script src="https://apis.google.com/js/platform.js" async defer></script-->
   <!--script type="text/javascript">
     var _gaq = _gaq || [];
       _gaq.push([ '_setAccount', 'UA-2429174-3' ]);
     _gaq.push([ '_trackPageview' ]);
     (function() {
       var ga = document.createElement('script');
       ga.type = 'text/javascript';
       ga.async = true;
       ga.src = ('https:' == document.location.protocol ? 'https://ssl'
           : 'http://www')
           + '.google-analytics.com/ga.js';
       var s = document.getElementsByTagName('script')[0];
       s.parentNode.insertBefore(ga, s);
     })();
   </script-->
   <script src="/Xtext/js/coverflow.min.js" type="text/javascript"></script>
   <script>
       $(function() {
         $('#coverflow').coverflow({
           active : 1,
           visibleAside: 2,
           overlap : 0.5,
           scale : 0.9,
           angle : 20,
           trigger : {
             "itemfocus" : true,
             "swipe" : true,
             "mousewheel" : false
           }
         });
         $('#coverflow :hidden').toggle();
         $(window).resize(function() {
           $('#coverflow').coverflow();
         });
       });

   </script>
 </footer>


   </body>

 </html>