blob: 73a30a8138047efaa460f6faf7c39d7bbad49235 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge"><![endif]-->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="generator" content="Asciidoctor 1.5.5">
<meta name="author" content="2019-08-08 13:15:33 CEST">
<title>N4JS IDE Specification</title>
<link rel="stylesheet" href="styles/spec.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css">
<!-- ************* docinfo ******************************************************************* -->
<!-- ************* Favicon ************-->
<link rel="icon" href="images/favicon.ico" />
<!-- ************* Back-to-top JQuery ************* -->
<script src="https://code.jquery.com/jquery-1.12.4.js"></script>
<script src="https://code.jquery.com/ui/1.12.0/jquery-ui.js"></script>
<link href="styles/prism.min.css" rel="stylesheet" />
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/MathJax.js?config=TeX-MML-AM_CHTML">
</script>
<!-- ************* Styles ************* -->
<link rel="stylesheet" type="text/css" href="styles/n4jsspec-adoc.css">
<!-- ****************** NavBar ****************** -->
<div id="menubar">
<div class="banner">
<a href="https://www.eclipse.org/n4js/#"><img id="logo" src="images/n4js-logo.png" alt="Eclipse N4JS"></a>
</div>
<ul>
<li><a href="index.html">Index</a></li>
</ul>
</div>
<!-- ************* docinfo ******************************************************************* -->
<style>
.admonitionblock td.icon .icon-todo:before{content:"\f249";color:#f4ee42}
</style>
</head>
<body class="book toc2 toc-left">
<div id="header">
<h1>N4JS IDE Specification</h1>
<div class="details">
<span id="author" class="author">2019-08-08 13:15:33 CEST</span><br>
<span id="revnumber">version 0.9</span>
</div>
<div id="toc" class="toc2">
<div id="toctitle">Table of Contents</div>
<ul class="sectlevel1">
<li><a href="views.html#sec:Views">1. Views</a>
<ul class="sectlevel2">
<li><a href="views.html#_eclipse-standard-views">1.1. Eclipse Standard Views</a>
<ul class="sectlevel3">
<li><a href="views.html#sec:Project_Explorer_View">1.1.1. Project Explorer</a></li>
<li><a href="views.html#sec:Outline_View">1.1.2. Outline</a></li>
<li><a href="views.html#sec:Problems_View">1.1.3. Problems</a></li>
<li><a href="views.html#sec:Console_View">1.1.4. Console</a></li>
<li><a href="views.html#sec:History_View">1.1.5. History</a></li>
<li><a href="views.html#sec:Error_Log_View">1.1.6. Error Log</a></li>
</ul>
</li>
<li><a href="views.html#_n4js-specific-views">1.2. N4JS Specific Views</a>
<ul class="sectlevel3">
<li><a href="views.html#_test-results">1.2.1. Test Results</a></li>
<li><a href="views.html#_source-graphs">1.2.2. Source Graphs</a></li>
<li><a href="views.html#_api-compare">1.2.3. API Compare</a></li>
<li><a href="views.html#_performance-graphs">1.2.4. Performance Graphs</a></li>
<li><a href="views.html#_source-mapping">1.2.5. Source Mapping</a></li>
<li><a href="views.html#_xpect-view">1.2.6. Xpect View</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="navigation.html#_navigation">2. Navigation</a>
<ul class="sectlevel2">
<li><a href="navigation.html#sec:Outline_Navigation">2.1. Outline</a>
<ul class="sectlevel3">
<li><a href="navigation.html#_quick-outline">2.1.1. Quick Outline</a></li>
<li><a href="navigation.html#_normal-outline">2.1.2. Normal Outline</a></li>
</ul>
</li>
<li><a href="navigation.html#_navigation-commands">2.2. Navigation Commands</a>
<ul class="sectlevel3">
<li><a href="navigation.html#sec:Navigate_to_Declaration">2.2.1. Navigate to Declaration</a></li>
<li><a href="navigation.html#sec:find_by_references">2.2.2. Find by References</a></li>
<li><a href="navigation.html#sec:Open_Type_Declaration">2.2.3. Open Type Declaration</a></li>
</ul>
</li>
<li><a href="navigation.html#sec:Working_Sets">2.3. Working Sets</a>
<ul class="sectlevel3">
<li><a href="navigation.html#sec:Working_Set_Managers">2.3.1. Working Set Managers</a></li>
<li><a href="navigation.html#sec:Working_Set_Constraints">2.3.2. Working Set Constraints</a></li>
<li><a href="navigation.html#sec:Manual_Association_Working_Set_Manager_UI_Features">2.3.3. Manual Association Working Set Manager - UI Features</a></li>
<li><a href="navigation.html#sec:Project_Name_Filter_Working_Set_Manager">2.3.4. Project Name Filter Working Set Manager</a></li>
<li><a href="navigation.html#sec:Git_Repository_Working_Set_Manager">2.3.5. Git Repository Working Set Manager</a></li>
<li><a href="navigation.html#sec:Project_Location_Working_Set_Manager">2.3.6. Project Location Working Set Manager</a></li>
<li><a href="navigation.html#sec:N4JS_Project_Type_Working_Set_Manager">2.3.7. N4JS Project Type Working Set Manager</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="assistance.html#_assistance">3. Assistance</a>
<ul class="sectlevel2">
<li><a href="assistance.html#sec:Content_Assist">3.1. Content Assist</a>
<ul class="sectlevel3">
<li><a href="assistance.html#sec:Complete_Keywords">3.1.1. Complete Keywords</a></li>
<li><a href="assistance.html#sec:Complete_Annotations">3.1.2. Complete Annotations</a></li>
<li><a href="assistance.html#sec:Complete_Identifier_Reference">3.1.3. Complete Identifier Reference</a></li>
<li><a href="assistance.html#sec:Complete_Member_Overrides">3.1.4. Complete Member Overrides</a></li>
<li><a href="assistance.html#sec:Constructor_Completion">3.1.5. Constructor Completion</a></li>
<li><a href="assistance.html#sec:Complete_Function_Expression">3.1.6. Complete Function Expression with Known Type</a></li>
<li><a href="assistance.html#sec:Complete_Variable_and_Parameter_Names">3.1.7. Complete Variable and Parameter Names</a></li>
</ul>
</li>
<li><a href="assistance.html#sec:Quick_Fixes">3.2. Quick Fixes</a>
<ul class="sectlevel3">
<li><a href="assistance.html#sec:N4JS_Issue_Properties">3.2.1. N4JS Issue User data</a></li>
<li><a href="assistance.html#sec:N4JS_Issue_Fixes">3.2.2. N4JS Issue Fixes</a>
<ul class="sectlevel4">
<li><a href="assistance.html#sec:Linking_Issues">3.2.2.1. Linking Issues</a></li>
<li><a href="assistance.html#sec:Import_Issues">3.2.2.2. Import Issues</a></li>
<li><a href="assistance.html#sec:Visibility_Issues">3.2.2.3. Visibility Issues</a></li>
<li><a href="assistance.html#sec:Classifier_Issues">3.2.2.4. Classifier Issues</a></li>
<li><a href="assistance.html#sec:Function_Issues">3.2.2.5. Function Issues</a></li>
<li><a href="assistance.html#sec:Syntax_Issues">3.2.2.6. Syntax Issues</a></li>
<li><a href="assistance.html#sec:Conversion_Issues">3.2.2.7. Conversion Issues</a></li>
<li><a href="assistance.html#sec:Type_Issues">3.2.2.8. Type Issues</a></li>
<li><a href="assistance.html#sec:Expression_Issues">3.2.2.9. Expression Issues</a></li>
<li><a href="assistance.html#sec:Super_Keyword_Issues">3.2.2.10. Super Keyword Issues</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="assistance.html#sec:Quick_Assist">3.3. Quick Assist</a></li>
<li><a href="assistance.html#sec:Cleanup_Operations">3.4. Cleanup Operations</a>
<ul class="sectlevel3">
<li><a href="assistance.html#sec:Formatting">3.4.1. Formatting</a></li>
<li><a href="assistance.html#sec:Organize_Imports">3.4.2. Organize Imports</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="wizards.html#_wizards">4. Wizards</a>
<ul class="sectlevel2">
<li><a href="wizards.html#sec:N4JS_Project_Wizard">4.1. N4JS Project Wizard</a></li>
<li><a href="wizards.html#sec:Empty_N4JS_File">4.2. Empty N4JS File</a></li>
<li><a href="wizards.html#sec:Empty_JS_File">4.3. Empty JS File</a>
<ul class="sectlevel3">
<li><a href="wizards.html#sec:N4JS_Class_File_Wizard">4.3.1. N4JS Class Wizard</a></li>
<li><a href="wizards.html#field-properties">4.3.2. Field Properties</a></li>
<li><a href="wizards.html#visibility-issues-or-final-super-classes">4.3.3. Visibility issues or <code>@Final</code> super classes</a></li>
<li><a href="wizards.html#generation-1">4.3.4. Generation</a></li>
<li><a href="wizards.html#preview-1">4.3.5. Preview</a></li>
</ul>
</li>
<li><a href="wizards.html#sec:N4JS_Interface_Wizard">4.4. Interface Wizard</a>
<ul class="sectlevel3">
<li><a href="wizards.html#field-properties-1">4.4.1. Field Properties</a></li>
<li><a href="wizards.html#visibility-issues">4.4.2. Visibility Issues</a></li>
<li><a href="wizards.html#import-naming-conflicts">4.4.3. Import naming conflicts</a></li>
<li><a href="wizards.html#generation-2">4.4.4. Generation</a></li>
<li><a href="wizards.html#preview-2">4.4.5. Preview</a></li>
</ul>
</li>
<li><a href="wizards.html#sec:N4JS_Enum_File_Wizard">4.5. Enum Wizard</a>
<ul class="sectlevel3">
<li><a href="wizards.html#field-properties-2">4.5.1. Field Properties</a></li>
<li><a href="wizards.html#generation-3">4.5.2. Generation</a></li>
<li><a href="wizards.html#preview-3">4.5.3. Preview</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="execution_support.html#_execution-support">5. Execution Support</a>
<ul class="sectlevel2">
<li><a href="execution_support.html#sec:Non_UI_Execution">5.1. Non-UI Execution</a>
<ul class="sectlevel3">
<li><a href="execution_support.html#sec:Non_UI_Debugging">5.1.1. Non-UI Debugging</a></li>
</ul>
</li>
<li><a href="execution_support.html#sec:UI_Execution">5.2. UI Execution</a></li>
</ul>
</li>
<li><a href="#_test-support">6. Test Support</a>
<ul class="sectlevel2">
<li><a href="#sec:N4JS_Mangelhaft_support">6.1. N4JS Mangelhaft support</a>
<ul class="sectlevel3">
<li><a href="#sec:Asynchronous_Tests">6.1.1. Asynchronous Tests, Test Isolation and Timeouts</a></li>
<li><a href="#sec:Supported_xUnit_API">6.1.2. Supported xUnit API</a>
<ul class="sectlevel4">
<li><a href="#sec:Test_Group">6.1.2.1. Test Group</a></li>
<li><a href="#sec:Test_Method">6.1.2.2. Test Method</a></li>
<li><a href="#sec:BeforeAll">6.1.2.3. BeforeAll Setup</a></li>
<li><a href="#sec:Before_Setup">6.1.2.4. Before Setup</a></li>
<li><a href="#sec:After_Teardown">6.1.2.5. After Teardown</a></li>
<li><a href="#sec:AfterAll_Teardown">6.1.2.6. AfterAll Teardown</a></li>
<li><a href="#sec:Test_Ignore">6.1.2.7. Test Ignore</a></li>
<li><a href="#sec:Timeout">6.1.2.8. Timeout</a></li>
<li><a href="#sec:Description">6.1.2.9. Description</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#sec:Test_Reporting">6.2. Test Reporting</a>
<ul class="sectlevel3">
<li><a href="#sec:Test_Messages">6.2.1. Test Messages</a>
<ul class="sectlevel4">
<li><a href="#_test-case-ids">6.2.1.1. Test Case IDs</a></li>
<li><a href="#sec:Start_Session">6.2.1.2. Start Session</a></li>
<li><a href="#sec:Ping_Session">6.2.1.3. Ping Session</a></li>
<li><a href="#sec:End_Session">6.2.1.4. End Session</a></li>
<li><a href="#sec:Start_Test">6.2.1.5. Start Test</a></li>
<li><a href="#sec:End_Test">6.2.1.6. End Test</a></li>
<li><a href="#sec:Ping_Test">6.2.1.7. Ping Test</a></li>
<li><a href="#sec:Test_Catalog">6.2.1.8. Test Catalog</a></li>
<li><a href="#sec:Test_Session_Example">6.2.1.9. Test Session Example</a></li>
</ul>
</li>
<li><a href="#sec:Test_Runtime_Configuration">6.2.2. Test Runtime Configuration</a></li>
<li><a href="#sec:Test_Plan">6.2.3. Test Plan</a></li>
<li><a href="#sec:Test_Environment_Configuration">6.2.4. Test Environment Configuration</a></li>
<li><a href="#sec:Test_Environment_Configuration_Example">6.2.5. Test Environment Configuration Example</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="help_system.html#_help-system">7. Help System</a>
<ul class="sectlevel2">
<li><a href="help_system.html#sec:Built_In_Help">7.1. Built-In Help</a></li>
<li><a href="help_system.html#sec:Context_Sensitive_Help">7.2. Context Sensitive Help</a></li>
<li><a href="help_system.html#sec:Cheat_Sheets">7.3. Cheat Sheets</a></li>
<li><a href="help_system.html#sec:JSDoc">7.4. JSDoc</a></li>
<li><a href="help_system.html#sec:Hovering">7.5. Hovering</a>
<ul class="sectlevel3">
<li><a href="help_system.html#sec:Show_Type_Information_of_Selection">7.5.1. Show Type Information of Selection</a></li>
</ul>
</li>
<li><a href="help_system.html#sec:Example_Projects_and_Files">7.6. Example Projects and Files</a></li>
</ul>
</li>
<li><a href="bug_management.html#_bug-management">8. Bug Management</a>
<ul class="sectlevel2">
<li><a href="bug_management.html#sec:Built_In_Xpect_Support">8.1. Built-In Xpect Support</a>
<ul class="sectlevel3">
<li><a href="bug_management.html#sec:Report_View">8.1.1. Report View</a></li>
<li><a href="bug_management.html#sec:Generate_Bug_Report">8.1.2. Generate Bug Report</a></li>
<li><a href="bug_management.html#sec:Supported_Xpect_Tests">8.1.3. Supported Xpect Tests</a>
<ul class="sectlevel4">
<li><a href="bug_management.html#sec:XPECT_N4JS_Errors">8.1.3.1. Errors, Warnings, Infos, Issues</a></li>
<li><a href="bug_management.html#sec:XPECT_N4JS_Noerrors">8.1.3.2. Noerrors</a></li>
<li><a href="bug_management.html#sec:XPECT_N4JS_Output">8.1.3.3. Output, OutputRegex</a></li>
<li><a href="bug_management.html#sec:XPECT_N4JS_Type_Of">8.1.3.4. Type Of</a></li>
<li><a href="bug_management.html#sec:XPECT_Advanced_Methods">8.1.3.5. Advanced methods</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li><a href="cli.html#_cli">9. CLI</a>
<ul class="sectlevel2">
<li><a href="cli.html#sec:Headless_Compiler">9.1. Headless Compiler</a></li>
<li><a href="cli.html#sec:Headless_Dependencies">9.2. Headless Dependnecies</a></li>
<li><a href="cli.html#sec:Headless_Execution">9.3. Headless Execution</a>
<ul class="sectlevel3">
<li><a href="cli.html#sec:Cleaning_Headlessly">9.3.1. Cleaning Headlessly</a></li>
<li><a href="cli.html#sec:Running_Headlessly">9.3.2. Running Headlessly</a></li>
<li><a href="cli.html#sec:Information_about_running_headlessly">9.3.3. Information about running headlessly</a></li>
<li><a href="cli.html#sec:Testing_Headlessly">9.3.4. Testing Headlessly</a></li>
<li><a href="cli.html#sec:Information_about_testing_headlessly">9.3.5. Information about testing headlessly</a></li>
<li><a href="cli.html#_testresults">9.3.6. TestResults</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="appendix_a_license.html#sec:License">Appendix A: License</a></li>
<li><a href="appendix_b_acronyms.html#sec:Acronyms">Appendix B: Acronyms</a></li>
<li><a href="appendix_c_bibliography.html#_bibliography">Appendix C: Bibliography</a></li>
</ul>
</div>
</div>
<div id="content"><div class="sect1">
<h2 id="_test-support"><a class="anchor" href="#_test-support"></a><a class="link" href="#_test-support">6. Test Support</a></h2>
<div class="sectionbody">
<div class="paragraph">
<p>N4IDE provides tests support by allowing different <em>TestRunner</em>s to extend it with test specific functionality. This allows to support specialized and very different from each other test requirements (e.g. nodejs based tests, browser based interactive ui tests, server integration tests).</p>
</div>
<div id="fig:test_support_diagram" class="imageblock">
<div class="content">
<img src="chapters/09_testsupport/fig/cd_testsupport.png" alt="cd testsupport">
</div>
<div class="title">Figure 10. Test Support Diagram</div>
</div>
<div class="paragraph">
<p>Explanation of the main components:</p>
</div>
<div class="dlist">
<dl>
<dt class="hdlist1"><em>User Project</em></dt>
<dd>
<div class="ulist">
<ul>
<li>
<p>a project with production code (e.g. <code>src</code> folder), and test code (e.g. test folder)</p>
</li>
<li>
<p>test code may contain special language features contributed by <em>Test Library</em></p>
</li>
</ul>
</div>
</dd>
<dt class="hdlist1"><em>N4IDE</em></dt>
<dd>
<div class="ulist">
<ul>
<li>
<p>manage user project, including all test related parts (e.g. support test related code, validate some test code constraints)</p>
</li>
<li>
<p>host runner, allow its UI contributions</p>
</li>
</ul>
</div>
</dd>
<dt class="hdlist1"><em>Test Runner</em></dt>
<dd>
<div class="ulist">
<ul>
<li>
<p>contribute to N4IDE necessary elements (test results view, user test selection, test start/stop actions)</p>
</li>
<li>
<p>use N4IDE mechanisms to access user project test fragment (e.g. discover tests)</p>
</li>
<li>
<p>configure <em>Test Execution Environment</em></p>
</li>
<li>
<p>manage test runtime (e.g. start/stop execution environment)</p>
</li>
</ul>
</div>
</dd>
<dt class="hdlist1"><em>Test Execution Environment</em></dt>
<dd>
<div class="ulist">
<ul>
<li>
<p>hosts (directly or indirectly) js engine in which tests are executed</p>
</li>
<li>
<p>executes test library logic</p>
</li>
<li>
<p>is responsible for some tests execution aspects (e.g. test isolation)</p>
</li>
<li>
<p>deals with some execution errors (e.g. no callback from async test, infinite loop in test)</p>
</li>
</ul>
</div>
</dd>
<dt class="hdlist1"><em>Test Library</em></dt>
<dd>
<div class="ulist">
<ul>
<li>
<p>provides test api that user can use in his project</p>
</li>
<li>
<p>coordinates scheduling and test code execution (e.g. order of tests, execution of setups / teardowns)</p>
</li>
<li>
<p>creates test results</p>
</li>
</ul>
</div>
</dd>
</dl>
</div>
<div class="paragraph">
<p>Below picture and listings depicts the components of the Test Runner in the IDE:</p>
</div>
<div id="fig:test_runner_components" class="imageblock" style="text-align: center">
<div class="content">
<img src="chapters/09_testsupport/fig/test_runner_components.png" alt="test runner components" width="75%">
</div>
<div class="title">Figure 11. Test Runner Components</div>
</div>
<div class="hdlist">
<table>
<tr>
<td class="hdlist1">
<em>Test Delegate</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>After the test discovery it starts and stops the test session via the Test Facade.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test Facade</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>Ensures that an embedded HTTP server is running to receive messages from the Test Execution Environment. Registers a test session into the IDE side via the Test Finite State Machine Registry and triggers the actual test running at Test Execution Environment.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>HTTP Server</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>HTTP server is listening for HTTP requests from the Test Execution Environment via its RESTful API.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Resource Router Servlet</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>This servlet is deployed into the servlet container of the HTTP Server. The servlet percepts the HTTP requests from the Test Execution Environment and delegates the request body to the corresponding REST Endpoint Logic.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>REST Endpoint Logic</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>Parses the HTTP request bodies, creates special events and sends them to all subscribers via the Test Event Bus.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test Event Bus</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>This component is used to asynchronously deliver messages between the main test runner components in a decoupled fashion.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test Finite State Machine Registry</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>This registry is used to cache test sessions represented as test trees and Test Finite State Machines. Percepts all messages initially sent by the Test Execution Environment and delegates them to the corresponding subscribers.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test Finite State Machine</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>Ensures the lifecycle of a test session. Handles timeouts duo to possible communication errors between the HTTP server and the Test Execution Environment.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test Tree Registry</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>Registry to handle the state of a test session. Responsible for updating a test tree associated with a test session with the received test results.</p>
</li>
</ul>
</div>
</td>
</tr>
<tr>
<td class="hdlist1">
<em>Test UI</em>
</td>
<td class="hdlist2">
<div class="ulist">
<ul>
<li>
<p>This UI component provides feedback about the running test session to the end-user.</p>
</li>
</ul>
</div>
</td>
</tr>
</table>
</div>
<div class="sect2 language-n4js">
<h3 id="sec:N4JS_Mangelhaft_support"><a class="anchor" href="#sec:N4JS_Mangelhaft_support"></a><a class="link" href="#sec:N4JS_Mangelhaft_support">6.1. N4JS Mangelhaft support</a></h3>
<div id="fig:xUnitSupportDesign" class="imageblock center" style="text-align: center">
<div class="content">
<img src="chapters/09_testsupport/fig/xUnitSupportDesign.png" alt="xUnitSupportDesign" width="75%">
</div>
<div class="title">Figure 12. xUnit Support Design</div>
</div>
<div class="paragraph">
<p>In this section and subsections we specify N4IDE support for testing with Mangelhaft.</p>
</div>
<div class="paragraph">
<p>Mangelhaft is N4JS <em>Test Library</em>. It is focused more on a xUnit tests than other forms of testing (BDD, Acceptance Testing, Functional UI Testing).</p>
</div>
<div class="paragraph">
<p>The following test scenarios are supported on different <em>Test Execution Environment</em>s:</p>
</div>
<table id="tab:Test_Scenarios" class="tableblock frame-all grid-all spread">
<caption class="title">Table 2. Test Scenarios</caption>
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-center valign-top">Test</th>
<th class="tableblock halign-center valign-top">Node</th>
<th class="tableblock halign-center valign-top">Browser</th>
<th class="tableblock halign-center valign-top">Wrapper</th>
</tr>
</thead>
<tbody>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">Plain</p></th>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>yes</code></p></td>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>yes</code></p></td>
<td class="tableblock halign-center valign-top"></td>
</tr>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">DOM</p></th>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>-</code></p></td>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>yes</code></p></td>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>-</code></p></td>
</tr>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">non-interactive UI</p></th>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>-</code></p></td>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
</tr>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">interactive UI (iUI)</p></th>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
</tr>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">(non UI) Server</p></th>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
</tr>
<tr>
<th class="tableblock halign-left valign-top"><p class="tableblock">iUI Server</p></th>
<td class="tableblock halign-center valign-top"><p class="tableblock"><code>-</code></p></td>
<td class="tableblock halign-center valign-top"></td>
<td class="tableblock halign-center valign-top"></td>
</tr>
</tbody>
</table>
<div class="sect3">
<h4 id="sec:Asynchronous_Tests"><a class="anchor" href="#sec:Asynchronous_Tests"></a><a class="link" href="#sec:Asynchronous_Tests">6.1.1. Asynchronous Tests, Test Isolation and Timeouts</a></h4>
<div class="paragraph">
<p>A special problem about JavaScript tests is to control asynchronous tests and non-terminating tests.</p>
</div>
<div class="paragraph">
<p>Performance and test isolation are conflicting goals: a perfect isolation would mean to run every tests by a separate JavaScript engine, which is not performant. For that reason, all tests are run by the same JS-engine in general. A test has to notify the test runner when it has been finished (successfully or with failure). If it does not finish in a defined time (timeout), <em>Test Execution Environment</em> or <em>Manglehaft</em> needs to handle that (e.g. restart node vm in which code is executed)&#8230;&#8203;</p>
</div>
<div class="paragraph">
<p>Main concerns with running test in parallel on js side are:</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>Timeouts Mangelhaft is supposed to track test timeout. If tests are running in fake parallel mode achieved by cooperative multitasking, then one test running eats up time for other test. This can cause tests to timeout when running in parallel, while succeed when running in sequential mode.</p>
</li>
<li>
<p>Mutability on client. Tests running in parallel can affect each other by mutating global state in which they operate. When they run in sequential mode this can happen too, but it is much less likely to.</p>
</li>
<li>
<p>Mutable state on the server. Tests running on the same session/login are prone to affecting each other through server interaction (and or mutating data on the server).</p>
</li>
</ol>
</div>
</div>
<div class="sect3">
<h4 id="sec:Supported_xUnit_API"><a class="anchor" href="#sec:Supported_xUnit_API"></a><a class="link" href="#sec:Supported_xUnit_API">6.1.2. Supported xUnit API</a></h4>
<div class="paragraph">
<p>xUnit API is user facing API for defining tests. It allows test developer to define tests and configure some test execution aspects. N4IDE (via <em>Test Runner</em> extension) supports defined API by :</p>
</div>
<div class="ulist">
<ul>
<li>
<p>gathering information via AST analysis and reflection</p>
</li>
<li>
<p>presenting user available actions, based on gathered information</p>
</li>
<li>
<p>gathering user input and configurations for test execution</p>
</li>
<li>
<p>generating proper data for test infrastructure, based on user actions</p>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="sec:Test_Group"><a class="anchor" href="#sec:Test_Group"></a><a class="link" href="#sec:Test_Group">6.1.2.1. Test Group</a></h5>
<div class="paragraph">
<p>A test group is a logical collection of tests. It is created by grouping <code>N4ClassDeclarations</code> that contain test methods or test methods directly (see <a href="#sec:Test_Method">Test Method</a>). Those classes or individual methods can be assigned to a <em>Group</em> by annotating them with <code>@Group</code> annotation. This annotation takes non empty list of strings as parameter. Passed strings are used as category name (which is like its id).</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@Group'
(' $group+=$STRING ')?
AnnotatedElement
;
AnnnotatedElement:
N4JSClassDeclaration | N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@Group</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>Group</code></p>
</li>
<li>
<p>targets &#8594; N4Method, N4Class</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; YES</p>
</li>
<li>
<p>repeatable &#8594; YES</p>
</li>
<li>
<p>arguments &#8594; <em>String</em>s</p>
</li>
<li>
<p>arguments are optional &#8594; NO</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="sec:Test_Method"><a class="anchor" href="#sec:Test_Method"></a><a class="link" href="#sec:Test_Method">6.1.2.2. Test Method</a></h5>
<div class="paragraph">
<p><em>Test Method</em> marks procedure that has to be executed by <em>Test Library</em>.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@Test'
AnnotatedElement
;
AnnnotatedElement:
N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@Test</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>Test</code></p>
</li>
<li>
<p>targets &#8594; N4Method</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; NO</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; none</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>Additional <em>TestMethod</em> constraints:</p>
</div>
<div class="paragraph">
<p>Test Method <a id="cnst:Test_Method"></a></p>
</div>
<div class="ulist">
<ul>
<li>
<p>allowed only <code>N4ClassDeclarations</code> in project test fragment</p>
</li>
<li>
<p>method must be public</p>
</li>
<li>
<p>method takes no parameters</p>
</li>
<li>
<p>method return type is <code>Promise?</code></p>
</li>
<li>
<p>method must not be referenced by other owning class members or other classes (also no <em>@override</em>)</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="sec:BeforeAll"><a class="anchor" href="#sec:BeforeAll"></a><a class="link" href="#sec:BeforeAll">6.1.2.3. BeforeAll Setup</a></h5>
<div class="paragraph">
<p><code>@BeforeAll</code> marks method that will be executed once before <strong>all</strong> tests in a given test class will be executed.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@BeforeAll'
AnnotatedElement
;
AnnnotatedElement:
N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@BeforeAll</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>BeforeAll</code></p>
</li>
<li>
<p>targets &#8594; N4Method</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; NO</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; none</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>The same constraints apply as for the test method, see <a href="#cnst:Test_Method">Test Method Constraints</a>.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:Before_Setup"><a class="anchor" href="#sec:Before_Setup"></a><a class="link" href="#sec:Before_Setup">6.1.2.4. Before Setup</a></h5>
<div class="paragraph">
<p><code>@Before</code> marks method that will be executed once before <strong>each</strong> tests in a given test class will be executed.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@Before'
AnnotatedElement
;
AnnnotatedElement:
N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@Before</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>Before</code></p>
</li>
<li>
<p>targets &#8594; N4Method</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; NO</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; none</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>The same constraints apply as for the test method, see <a href="#cnst:Test_Method">Test Method Constraints</a>.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:After_Teardown"><a class="anchor" href="#sec:After_Teardown"></a><a class="link" href="#sec:After_Teardown">6.1.2.5. After Teardown</a></h5>
<div class="paragraph">
<p><code>@After</code> marks method that will be executed once after <strong>each</strong> tests in a given test class will be executed.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@After'
AnnotatedElement
;
AnnnotatedElement:
N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@After</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>After</code></p>
</li>
<li>
<p>targets &#8594; N4Method</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; NO</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; none</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>The same constraints apply as for the test method, see <a href="#cnst:Test_Method">Test Method Constraints</a>.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:AfterAll_Teardown"><a class="anchor" href="#sec:AfterAll_Teardown"></a><a class="link" href="#sec:AfterAll_Teardown">6.1.2.6. AfterAll Teardown</a></h5>
<div class="paragraph">
<p><code>@AfterAll</code> marks method that will be executed once after <strong>all</strong> tests in a given test class will be executed.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@After'
AnnotatedElement
;
AnnnotatedElement:
N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="ulist Test Fixture][cnst:Test_Fixture">
<ul class="Test Fixture][cnst:Test_Fixture">
<li>
<p>allowed only in class marked with <em>@TestClass</em></p>
</li>
<li>
<p>method must be public</p>
</li>
<li>
<p>method takes no parameters</p>
</li>
<li>
<p>method return type is <code>void</code></p>
</li>
<li>
<p>method must not be referenced by other owning class members</p>
</li>
</ul>
</div>
<div class="paragraph">
<p><code>@AfterAll</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>AfterAll</code></p>
</li>
<li>
<p>targets &#8594; N4Method</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; NO</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; none</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>The same constraints apply as for the test method, see <a href="#cnst:Test_Method">Test Method Constraints</a>.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:Test_Ignore"><a class="anchor" href="#sec:Test_Ignore"></a><a class="link" href="#sec:Test_Ignore">6.1.2.7. Test Ignore</a></h5>
<div class="hdlist">
<table>
<tr>
<td class="hdlist1">
name
</td>
<td class="hdlist2">
<p>@Ignore</p>
</td>
</tr>
<tr>
<td class="hdlist1">
targets
</td>
<td class="hdlist2">
<p>N4Method, N4Class</p>
</td>
</tr>
<tr>
<td class="hdlist1">
retention policy
</td>
<td class="hdlist2">
<p>RUNTIME</p>
</td>
</tr>
<tr>
<td class="hdlist1">
transitive
</td>
<td class="hdlist2">
<p>YES</p>
</td>
</tr>
<tr>
<td class="hdlist1">
repeatable
</td>
<td class="hdlist2">
<p>NO</p>
</td>
</tr>
<tr>
<td class="hdlist1">
arguments
</td>
<td class="hdlist2">
<p>String reason</p>
</td>
</tr>
<tr>
<td class="hdlist1">
arguments are optional
</td>
<td class="hdlist2">
<p>&#8594; Yes</p>
</td>
</tr>
</table>
</div>
<div class="paragraph">
<p><em>Test Ignore</em> allows to mark tests that should be skipped during the test execution. That is the preferred way to temporarily disable tests without removing them (or commenting them out). Test developers may provide reason for skipping to make reason/intentions clearer.</p>
</div>
<div class="paragraph">
<p>This annotation is <em>transitive</em>, which means that: <em>Test Method</em> is considered as marked with <em>Test Skip</em></p>
</div>
<div class="ulist">
<ul>
<li>
<p>explicitly when it is directly marked or</p>
</li>
<li>
<p>implicitly, when container of a <em>Test Method</em> is marked.</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>If a class is marked as <code>@Ignore</code>, then all its contained test methods will be ignored.<br>
When <code>@Ignore</code> occurs at class level in a test class hierarchy chain, then the following rules are applied. Assume the following test classes:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">export public class A {
@Test
public aTest(): void {
console.log('A#aTest');
}
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">import { A } from "A"
@Ignore('Class B is ignored.')
export public class B extends A {
@Test
public b1Test(): void {
console.log('B#b1Test');
}
@Ignore("Method B#b2Test is ignored.")
@Test
public b2Test(): void {
console.log("B#b2Test");
}
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">import { B } from "B"
export public class C extends B {
@Test
public cTest(): void {
console.log('C#cTest');
}
}</code></pre>
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>When module <em>A</em> is being tested, then it is obvious that all the test methods of <code>A</code> will be tested. No methods will be skipped at all.</p>
</li>
<li>
<p>When module <em>B</em> is being tested, then although the inherited members of class <code>A</code> will be included in the test tree, all methods, including the inherited ones (from class <code>A</code> from module <em>A</em>) will be skipped. Nothing will be tested.</p>
</li>
<li>
<p>When module <em>C</em> is being tested, then all inherited members from class <code>B</code> and class <code>A</code> will be collected an included in the test tree. The <code>@Ignore</code> annotation declared at class level at <code>B</code> will be ignored but the <code>@Ignore</code> at method level in class <code>B</code> will be considered. In a nutshell, the following methods will be executed:</p>
<div class="ulist">
<ul>
<li>
<p><code>A#aTest</code></p>
</li>
<li>
<p><code>B#b1Test</code></p>
</li>
<li>
<p><code>C#cTest</code></p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="paragraph">
<p>The above described behavior is identical to the behavior of <em>JUnit 4</em> with respect to the <code>@Ignore</code> annotation handling in case of test class inheritance.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:Timeout"><a class="anchor" href="#sec:Timeout"></a><a class="link" href="#sec:Timeout">6.1.2.8. Timeout</a></h5>
<div class="paragraph">
<p><em>Timeout</em> allows test developer to set custom timeout when executing given test code. This can be used to set timeout for both <em>Test Method</em>s or <em>Test Fixtures</em></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@Timeout'
($timoeout+=$INT)?
AnnotatedElement
;
AnnnotatedElement:
N4JSClassDeclaration | N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@Timeout</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>Timeout</code></p>
</li>
<li>
<p>targets &#8594; N4Method, N4Class</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; YES</p>
</li>
<li>
<p>repeatable &#8594; NO</p>
</li>
<li>
<p>arguments &#8594; Number</p>
</li>
<li>
<p>arguments are optional &#8594; NO</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="sec:Description"><a class="anchor" href="#sec:Description"></a><a class="link" href="#sec:Description">6.1.2.9. Description</a></h5>
<div class="paragraph">
<p><em>Description</em> allows test developer provide string describing given test or test class that <em>can</em> be used in IDE test view or in the test report.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-xtext" data-lang="xtext">Annotation:
'@Description'
($desc+=$STRING)?
AnnotatedElement
;
AnnnotatedElement:
N4JSClassDeclaration | N4JSMethodDeclaration
;</code></pre>
</div>
</div>
<div class="paragraph">
<p><code>@Description</code> properties</p>
</div>
<div class="ulist">
<ul>
<li>
<p>name &#8594; <code>Description</code></p>
</li>
<li>
<p>targets &#8594; N4Method, N4Class</p>
</li>
<li>
<p>retention policy &#8594; RUNTIME</p>
</li>
<li>
<p>transitive &#8594; YES</p>
</li>
<li>
<p>arguments &#8594; String</p>
</li>
<li>
<p>arguments are optional &#8594; NO</p>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="sec:Test_Reporting"><a class="anchor" href="#sec:Test_Reporting"></a><a class="link" href="#sec:Test_Reporting">6.2. Test Reporting</a></h3>
<div class="paragraph">
<p><em>Test Runtime Environment</em> communicates with <em>Test Runner</em> over HTTP. Defined communication is based on protocol used between lupenrein and old ide. It is used to send the information about test execution progress from the <em>Test Runtime</em> to <em>Test Runner</em>. Information send by this protocol is not equivalent to test results. <em>Test Runner</em> interprets progress it receives and based on gathered information it generates test results. Under specific conditions <em>Test Runner</em> may change reported test status PASS to test result FAILED and put this information to the test report e.g. when timeout happens (see note on timeouts below).</p>
</div>
<div id="fig:sm_TestListener" class="imageblock center" style="text-align: center">
<div class="content">
<img src="chapters/09_testsupport/fig/sm_TestListener.png" alt="sm TestListener" width="25%">
</div>
<div class="title">Figure 13. TestListener</div>
</div>
<div class="paragraph">
<p><a href="#fig:sm_TestListener">Test Listener</a> shows Communication flow expected by the <em>Test Runner</em>. When the <em>Test Runner</em> is started first it waits for <em>Start Session</em> message. Next <em>Test Tree</em> message is expected. This describes list of all tests that are expected to be executed. For all tests in the list <em>Test Runner</em> expects <em>Test Start</em> and <em>Test End</em> message to be received. <em>End Session</em> is expected to be last message in the test session. <em>Ping</em> message can be send multiple times in between other messages to manage synchronization issues between <em>Test Runner</em> and <em>Test Runtime</em> (see below).</p>
</div>
<div class="paragraph">
<p>Since all communication is asynchronous, IDE <em>Test Runner</em> must assume some timeout values that will define standard wait time during communication:</p>
</div>
<div class="ulist">
<ul>
<li>
<p>Initial 90s timeout to wait for the <em>Start Session</em> message. It may be fixed or adjusted to given environment (local/remote) and project (library/application).</p>
</li>
<li>
<p>Default timeout between all other test messages is 10 seconds. <em>Test Runtime</em> may notify IDE <em>Test Runner</em> that it should wait longer with <em>Ping</em> <em>test message</em>. This is one time thing, as soon as another command is received the default timeout will have to be reused again.</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>Do to the asynchronous nature of the tests, status updates can be given out of order by the Test Runtime Environment. The only sure thing is that all tests begin with <em>SessionStart</em> and ends with a <em>SessionEnd</em>. Furthermore a <em>TestStart</em> will be send before the <em>TestEnd</em> for a particular test.</p>
</div>
<div class="sect3">
<h4 id="sec:Test_Messages"><a class="anchor" href="#sec:Test_Messages"></a><a class="link" href="#sec:Test_Messages">6.2.1. Test Messages</a></h4>
<div class="paragraph">
<p>IDE <em>Test Runner</em> will be waiting for specific messages from <em>Test Runtime</em>. We assume that communication will be done over HTTP protocol. <em>Test Execution Environement</em> should be configured by the <em>Test Runner</em> in a way that <em>Test Runtime</em> knows address where it has to send messages (see
<a href="#sec:Test_Runtime_Configuration">Test Runtime Configuration</a>). <em>Test Runner</em> exposes RESTful API allowing him to receive messages. Below we define parts of that api that enable specific messages to be communicated.</p>
</div>
<div class="paragraph">
<p>When defining <em>Test Message</em>s we assume following model of tests:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">TestTree {
ID sessionId,
Array&lt;TestSuite&gt;? testSuites
}
TestSuite {
string name,
Array&lt;TestCase&gt;? testCases,
Array&lt;TestSuite&gt;? children
}
TestCase {
ID id,
string className,
string origin,
string name,
string displayName,
TestResult? result
}
TestResult {
TestStatus teststatus,
number elapsed,
string? expected,
string? actual,
string? message,
array&lt;string&gt;? trace
}
enum TestStatus {
PASSED, SKIPPED, FAILED, ERROR
}
ID {
string value
}</code></pre>
</div>
</div>
<div class="sect4">
<h5 id="_test-case-ids"><a class="anchor" href="#_test-case-ids"></a><a class="link" href="#_test-case-ids">6.2.1.1. Test Case IDs</a></h5>
<div class="paragraph">
<p>The ID of a test case in the following specifications is referred to as <code>testID</code>.
This ID is of the following structure:</p>
</div>
<div class="listingblock">
<div class="content">
<pre>testID: fqn '#' methodName</pre>
</div>
</div>
<div class="paragraph">
<p>When used as part of the URL the testID is percent-escaped as defined in <a href="https://tools.ietf.org/html/rfc3986#section-2.1">RFC3986 Section 2.1</a>. This is necessarry to circumvent the fact that the N4JS FQN delimiter <code>/</code> is a reserved character in URLs and cannot be used in its original form.</p>
</div>
</div>
<div class="sect4">
<h5 id="sec:Start_Session"><a class="anchor" href="#sec:Start_Session"></a><a class="link" href="#sec:Start_Session">6.2.1.2. Start Session</a></h5>
<div class="paragraph">
<p>Signals start of the test session. When user triggers test execution, configures <em>IDETestRunnerCtrl</em>, afterwards IDE <em>Listener</em> waits for this message from <em>TestRunner</em>.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">StartSession :
uri : /n4js/testing/sessions/{sessionID}/start
method : POST
contentType : application/vnd.n4js.start_session_req.tm+json
accept: application/json
responses:
200:
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>Start session request object MIME type <em>application/vnd.n4js.start_session_req.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
map&lt;string, string&gt;? properties
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:Ping_Session"><a class="anchor" href="#sec:Ping_Session"></a><a class="link" href="#sec:Ping_Session">6.2.1.3. Ping Session</a></h5>
<div class="paragraph">
<p>Signals that test runner is still busy doing things, and will report later to the listener.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">PingSession :
uri : /n4js/testing/sessions/{sessionID}/ping
method : POST
contentType : application/vnd.n4js.ping_session_req.tm+json
accept: application/json
responses:
200:
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>Ping session request object MIME type <em>application/vnd.n4js.ping_session_req.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
number timeout,
string? comment
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:End_Session"><a class="anchor" href="#sec:End_Session"></a><a class="link" href="#sec:End_Session">6.2.1.4. End Session</a></h5>
<div class="paragraph">
<p>Signals end of test session Notifies IDE <em>Listener</em> that session is finished and no further related <em>TestMessage</em>s are expected. IDE, can stop listening and proceed with its own tasks (e.g. create summary test report ).</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">EndSession :
uri : /n4js/testing/sessions/{sessionID}/end
method : POST
responses:
200:
400:</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:Start_Test"><a class="anchor" href="#sec:Start_Test"></a><a class="link" href="#sec:Start_Test">6.2.1.5. Start Test</a></h5>
<div class="paragraph">
<p>Signals that a test run has started. Updates the state of the test reported with the <em>tree</em> .</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">StartTest :
uri : /n4js/testing/sessions/{sessionID}/tests/{testID}/start
method : POST
contentType : application/vnd.n4js.start_test_req.tm+json
accept: application/json
responses:
200:
contentType : application/vnd.n4js.start_test_res.tm+json
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>Start test request object MIME type <em>application/vnd.n4js.start_test_req.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
number timeout,
map&lt;string, string&gt;? properties
}</code></pre>
</div>
</div>
<div class="paragraph">
<p>Start test response object MIME type <em>application/vnd.n4js.start_test_res.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
links : [
{
rel: "ping test",
uri: "/n4js/testing/sessions/{sessionID}/tests/{testID}/ping"
},
{
rel: "end test",
uri: "/n4js/testing/sessions/{sessionID}/tests/{testID}/end"
}
]
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:End_Test"><a class="anchor" href="#sec:End_Test"></a><a class="link" href="#sec:End_Test">6.2.1.6. End Test</a></h5>
<div class="paragraph">
<p>Signals that a test run has ended. Updates the state of the test reported with the <em>tree</em> .</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">EndTest :
uri : /n4js/testing/sessions/{sessionID}/tests/{testID}/end
method : POST
contentType : application/vnd.n4js.end_test_req.tm+json
accept: application/json
responses:
200:
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>End test request object MIME type <em>application/vnd.n4js.end_test_req.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
TestResult result
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:Ping_Test"><a class="anchor" href="#sec:Ping_Test"></a><a class="link" href="#sec:Ping_Test">6.2.1.7. Ping Test</a></h5>
<div class="paragraph">
<p>Notifies IDE that <em>TestRunner</em> is doing something (e.g. test setup/teardown code, long running test). Without this notification IDE might interpret long pause in received messages as timeout, <em>TestRunner</em> crash or other issues (in consequence it might terminate whole test execution environment).</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">PingTest :
uri : /n4js/testing/sessions/{sessionID}/tests/{testID}/ping
method : POST
contentType : application/vnd.n4js.ping_test_req.tm+json
accept: application/json
responses:
200:
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>Ping test request object MIME type <em>application/vnd.n4js.ping_test_req.tm+json</em>:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
number timeout,
string? comment
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:Test_Catalog"><a class="anchor" href="#sec:Test_Catalog"></a><a class="link" href="#sec:Test_Catalog">6.2.1.8. Test Catalog</a></h5>
<div class="paragraph">
<p>Assembles and returns with the test catalog representing all the tests available in the underlying <em>IN4JSCore</em> specific workspace. The content of the test catalog is calculated dynamically. The test catalog calculation depends on the current built state of the workspace. If the workspace was cleaned and not built yet, then a test catalog containing zero test suites (and test cases) will be provided as a response. If the workspace is built and in consistent state, then a catalog containing all test cases will be sent as the response body. The provided test catalog format complies to the Mangelhaft reporters.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">TestCatalog :
uri : /n4js/testing/sessions/testcatalog
method : GET
contentType : application/vnd.n4js.assemble_test_catalog_req.tm+json
accept: application/json
responses:
200:
400:</code></pre>
</div>
</div>
<div class="paragraph">
<p>Below listings represents an example of the test catalog format:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
"endpoint": "http://localhost:9415",
"sessionId": "fc3a425c-b675-47d7-8602-8877111cf909",
"testDescriptors": [
{
"origin": "SysProjectA-0.0.1",
"fqn": "T/T",
"testMethods": [
"t"
]
},
{
"origin": "TestProjectA-0.0.1",
"fqn": "A/A",
"testMethods": [
"a"
]
},
{
"origin": "TestProjectA-0.0.1",
"fqn": "B/B",
"testMethods": [
"b1",
"b2"
]
},
{
"origin": "TestProjectB-0.0.1",
"fqn": "CSub1/CSub1",
"testMethods": [
"c1",
"c2"
]
},
{
"origin": "TestProjectB-0.0.1",
"fqn": "CSub2/CSub2",
"testMethods": [
"c1",
"c2",
"c3"
]
}
]
}</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="sec:Test_Session_Example"><a class="anchor" href="#sec:Test_Session_Example"></a><a class="link" href="#sec:Test_Session_Example">6.2.1.9. Test Session Example</a></h5>
<div class="paragraph">
<p>Below example demonstrates what are the expected HTTP requests and JSON structures for a simple test group.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">class A {
@Test
public void foo() {}
@Test
@Ignore
public void bar() {}
}
class B {
@Test
public void baz() {}
}
class C {
@Test
public void qux() {}
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/start/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.start_session_req.tm+json; charset=ISO-8859-1</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FC%23qux/start/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.start_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 1000
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FB%23baz/start/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.start_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 1000
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FA%23bar/start/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.start_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 1000
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FA%23foo/start/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.start_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 1000
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FA%23bar/ping
Headers: Accept=*/*
Content-Type=application/vnd.n4js.ping_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 1000
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FC%23qux/ping/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.ping_test_req.tm+json; charset=ISO-8859-1
Body:
{
"timeout": 2000
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FB%23baz/end/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.end_test_req.tm+json; charset=ISO-8859-1
Body:
{
"message": "Some optional message.",
"trace": [
"trace_element_1",
"trace_element_2",
"trace_element_3"
],
"expected": "1",
"testStatus": "FAILED",
"elapsedTime": 100,
"actual": "2"
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FC%23qux/end/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.end_test_req.tm+json; charset=ISO-8859-1
Body:
{
"message": "Some failure message.",
"trace": [
"trace_element_1",
"trace_element_2",
"trace_element_3"
],
"expected": "4",
"testStatus": "FAILED",
"elapsedTime": 50,
"actual": "3"
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2F%23foo/end/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.end_test_req.tm+json; charset=ISO-8859-1
Body:
{
"expected": "2",
"testStatus": "PASSED",
"elapsedTime": 60,
"actual": "power of 2 for 2"
}
Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/tests/Test%2FA%23bar/end/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.end_test_req.tm+json; charset=ISO-8859-1
Body:
{
"testStatus": "SKIPPED",
"elapsedTime": 0,
}</code></pre>
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">Request method: POST
Request path: http://localhost:9415/n4js/testing/sessions/19f47a37-c1d1-4cb7-a514-1e131f26ab13/end/
Headers: Accept=*/*
Content-Type=application/vnd.n4js.end_session_req.tm+json; charset=ISO-8859-1</code></pre>
</div>
</div>
</div>
</div>
<div class="sect3">
<h4 id="sec:Test_Runtime_Configuration"><a class="anchor" href="#sec:Test_Runtime_Configuration"></a><a class="link" href="#sec:Test_Runtime_Configuration">6.2.2. Test Runtime Configuration</a></h4>
<div class="paragraph">
<p><em>Test Runner</em> must gather relevant information and send it to <em>Test Environment</em> to allow proper test execution:</p>
</div>
<div class="ulist">
<ul>
<li>
<p>gathering user input and test options</p>
</li>
<li>
<p>gathering information about user project test code</p>
</li>
<li>
<p>maintaining proper name mappings (e.g. if project is minimized test names/references must be mapped correctly)</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="sec:Test_Plan"><a class="anchor" href="#sec:Test_Plan"></a><a class="link" href="#sec:Test_Plan">6.2.3. Test Plan</a></h4>
<div class="paragraph">
<p><em>Test Runner</em> uses N4IDE infrastructure to obtain information about test fragment of the user project. Based on that information and user input in UI (e.g. triggering test execution on whole project) IDE can determine <em>Test Method</em>s that should be executed. Such test list or <em>Test Plan</em> is send to <em>Test Environment</em> and is expected to be executed by a <em>Test Library</em>.</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">TestPlan {
Array&lt;TestProcedure&gt; procedures
}
TestProcedure {
string functionName,
string functionType,
string functionContainer,
string containerModule
}</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="sec:Test_Environment_Configuration"><a class="anchor" href="#sec:Test_Environment_Configuration"></a><a class="link" href="#sec:Test_Environment_Configuration">6.2.4. Test Environment Configuration</a></h4>
<div class="paragraph">
<p>Additionally <em>Test Runner</em> sends to <em>Test Environment</em> other configuration options:</p>
</div>
<div class="ulist">
<ul>
<li>
<p><em>Test Runner</em> test communication protocol base url (<em>baseURL</em>)</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="sec:Test_Environment_Configuration_Example"><a class="anchor" href="#sec:Test_Environment_Configuration_Example"></a><a class="link" href="#sec:Test_Environment_Configuration_Example">6.2.5. Test Environment Configuration Example</a></h4>
<div class="paragraph">
<p>For example assuming that user selects <em>ProjectX</em> to test that contains only one test class in <em>src/test/n4js/core</em> path like:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-n4js" data-lang="n4js">class MyTestClass{
@BeforeAll
public void someOneTimeSetup(){ /* setup code */}
@Test
public void testA(){ /* some test code*/ }
@Test
public void testB(){ /* some test code*/ }
@Test
public void testC(){ /* some test code*/ }
@After
public void afterCleanup(){ /* setup code */}
}</code></pre>
</div>
</div>
<div class="paragraph">
<p>Configuration sent for <em>Test Execution Environment</em> would look like:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="highlight"><code class="language-json" data-lang="json">{
"baseURL" : "http://localhost:1234/",
"testPlan":
[
{
"functionName": "someOneTimeSetup",
"functionType": "@BeforeAll",
"functionContainer": "MyTestClass",
"containerModule": "test/n4js/core/MyTestClass",
},
{
"functionName": "testA",
"functionType": "@Test",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
},
{
"functionName": "afterCleanup",
"functionType": "@After",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
},
{
"functionName": "testB",
"functionType": "@Test",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
},
{
"functionName": "afterCleanup",
"functionType": "@After",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
},
{
"functionName": "testC",
"functionType": "@Test",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
},
{
"functionName": "afterCleanup",
"functionType": "@After",
"functionContainer": "MyTestClassA",
"containerModule": "test/n4js/core/MyTestClassA",
}
]
}</code></pre>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div id="footer">
<div id="footer-text">
Version 0.9<br>
Last updated 2019-08-08 13:15:33 CEST
</div>
</div>
<!-- ************* docinfo-footer *************************************************************** -->
<div class="Grid social" style="color:#d5dfea">
<div class="Cell Cell--2-12 m-Cell--withMargin">
<h2>Quick Links</h2>
<ul>
<li><a href="../downloads.html">Download</a></li>
<li><a href="../userguides/index.html">Documentation</a></li>
<li><a href="https://github.com/eclipse/n4js/">Source</a></li>
<li><a href="https://github.com/eclipse/n4js/issues">Issues</a></li>
</ul>
</div>
<div class="Cell Cell--2-12 m-Cell--withMargin">
<br/><br/>
<ul>
<li><a href="https://www.eclipse.org/forums/index.php/f/365/">Forum</a></li>
<li><a href="http://n4js.blogspot.de/">Blog</a></li>
<li><a href="https://dev.eclipse.org/mailman/listinfo/n4js-dev">Mailing List</a></li>
<li><a href="https://projects.eclipse.org/projects/technology.n4js">Eclipse Project Page</a></li>
<li><a href="https://twitter.com/n4jsdev">Tweets by n4jsdev</a></li>
</ul>
</div>
<div class="Cell Cell--2-12 m-Cell--withMargin">
<br/><br/>
<ul>
<li><a href="http://www.eclipse.org/">Eclipse Home</a></li>
<li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li>
<li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li>
<li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li>
<li><a href="http://www.eclipse.org/legal/">Legal</a></li>
</ul>
</div>
<div style="clear: both; height: 0; overflow: hidden;"></div>
</div>
<!-- ************* UI Scripts ************* -->
<script type="text/javascript" src="scripts/back-to-top.js"></script>
<script type="text/javascript" src="scripts/treeview.js"></script>
<script type="text/javascript" src="scripts/toc.js"></script>
<!-- ************* Prism.js Syntax Highlighting ************* -->
<script src="scripts/prism.js"></script>
<script type="text/javascript">
// Add the 'toclist' id for search function
$(".toc2 > ul").attr('id', 'toclist');
// Generate a Search input form
$("#toclist > li:first-of-type").before('<input type="text" id="pagesearch" onkeyup="search()" placeholder="Search for section...">');
$("#toclist > li:first-of-type").before('<i id="clear" class="fa fa-times-circle-o"></i>');
$("#clear").click(function(){
$("#pagesearch").val('');
search();
$('.toc2 > ul').treeView('collapseAll');
});
// intialize Treeview.js
$(".toc2 > ul").treeView();
// Initialize Scrollspy
</script>
<!-- ************* docinfo-footer *************************************************************** -->
</body>
</html>