| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Documentation/TaskManager,SMILA/Documentation/Bundle org.eclipse.smila.clusterconfig.simple,SMILA/Documentation/JobManager,SMILA/Documentation/WorkerManager,SMILA/Glossary" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Documentation/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Documentation/TaskManager - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "http://wiki.eclipse.org/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Documentation/TaskManager"; |
| var wgTitle = "SMILA/Documentation/TaskManager"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "32482"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "326629"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <style type="text/css">/*<![CDATA[*/ |
| .source-javascript {line-height: normal; font-size: medium;} |
| .source-javascript li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for javascript |
| * CSS class: source-javascript, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-javascript .de1, .source-javascript .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-javascript {} |
| .source-javascript .head {} |
| .source-javascript .foot {} |
| .source-javascript .imp {font-weight: bold; color: red;} |
| .source-javascript .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-javascript li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-javascript li.li2 {font-weight: bold;} |
| .source-javascript .kw1 {color: #000066; font-weight: bold;} |
| .source-javascript .kw2 {color: #003366; font-weight: bold;} |
| .source-javascript .kw3 {color: #000066;} |
| .source-javascript .co1 {color: #009900; font-style: italic;} |
| .source-javascript .coMULTI {color: #009900; font-style: italic;} |
| .source-javascript .es0 {color: #000099; font-weight: bold;} |
| .source-javascript .br0 {color: #66cc66;} |
| .source-javascript .st0 {color: #3366CC;} |
| .source-javascript .nu0 {color: #CC0000;} |
| .source-javascript .me1 {color: #006600;} |
| .source-javascript .sc0 {} |
| .source-javascript .sc1 {} |
| .source-javascript .sc2 {} |
| .source-javascript .sc3 {} |
| .source-javascript .re0 {color: #0066FF;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "http://wiki.eclipse.org/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><link rel="stylesheet" type="text/css" href="TaskManager.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Documentation_TaskManager"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/TaskManager">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Documentation/TaskManager">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Documentation/TaskManager">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/TaskManager&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/TaskManager&oldid=326629">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="TaskManager.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Documentation/TaskManager&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/TaskManager&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/TaskManager&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/TaskManager"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Documentation/TaskManager</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Documentation.1.html" title="SMILA/Documentation">Documentation</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="TaskManager.html#column-one">navigation</a>, <a href="TaskManager.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="TaskManager.html#TaskManager"><span class="tocnumber">1</span> <span class="toctext">TaskManager</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="TaskManager.html#Configuration"><span class="tocnumber">1.1</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="TaskManager.html#External_REST_API"><span class="tocnumber">1.2</span> <span class="toctext">External REST API</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="TaskManager.html#TaskManager_State_Overview"><span class="tocnumber">1.2.1</span> <span class="toctext">TaskManager State Overview</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#Worker_Overview"><span class="tocnumber">1.2.2</span> <span class="toctext">Worker Overview</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#Task_Content"><span class="tocnumber">1.2.3</span> <span class="toctext">Task Content</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-2"><a href="TaskManager.html#Tasks"><span class="tocnumber">1.3</span> <span class="toctext">Tasks</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="TaskManager.html#JSON_format_of_task_.281.29"><span class="tocnumber">1.3.1</span> <span class="toctext">JSON format of task (1)</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#JSON_format_of_qualifier_condition_.282.29"><span class="tocnumber">1.3.2</span> <span class="toctext">JSON format of qualifier condition (2)</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#JSON_format_of_result_description_.283.29"><span class="tocnumber">1.3.3</span> <span class="toctext">JSON format of result description (3)</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-2"><a href="TaskManager.html#Internal_REST_API"><span class="tocnumber">1.4</span> <span class="toctext">Internal REST API</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="TaskManager.html#Get_task_information"><span class="tocnumber">1.4.1</span> <span class="toctext">Get task information</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#Get_initial_task_information"><span class="tocnumber">1.4.2</span> <span class="toctext">Get initial task information</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#Get_task_for_scaling_worker"><span class="tocnumber">1.4.3</span> <span class="toctext">Get task for scaling worker</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#TaskManager_State_Overview_2"><span class="tocnumber">1.4.4</span> <span class="toctext">TaskManager State Overview</span></a></li> |
| <li class="toclevel-3"><a href="TaskManager.html#Finish_or_keep_alive_task"><span class="tocnumber">1.4.5</span> <span class="toctext">Finish or keep alive task</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="TaskManager"></a><h1> <span class="mw-headline"> TaskManager </span></h1> |
| <p>The TaskManager is a component for administrating and delivering <a href="../Glossary.html#T" title="SMILA/Glossary">tasks</a> during a <a href="../Glossary.html#J" title="SMILA/Glossary">job run</a>. Tasks are stored in internal queues. They are consumed by <a href="../Glossary.html#W" title="SMILA/Glossary">workers</a> pulling them for processing. They are produced as initial tasks or as follow-up tasks when workers finish their processing of their current task. |
| </p><p>The TaskManager guarantees that tasks are delivered at least once, but tasks may be delivered more than once (e.g. when IOErrors, process crashes or timeouts occur). A client must be able to handle a task multiple times. But redelivery of tasks is the exception to the rule, usually during task processing we do not expect errors to happen. There are some rare cases in which a task is successfully processed (i.e. output data and a possible follow up task are created successfully) but the task is redelivered nonetheless. This could happen, if: |
| </p> |
| <ul><li> the worker process crashes exactly after that step OR |
| </li><li> the TTL of the task times out exactly at the time when the worker is just finishing, in which case the the task is rolled back inside the TaskManager automatically. |
| </li></ul> |
| <a name="Configuration"></a><h2> <span class="mw-headline"> Configuration </span></h2> |
| <p>The TaskManager is configured via the <a href="Bundle_org.eclipse.smila.clusterconfig.simple.html" title="SMILA/Documentation/Bundle org.eclipse.smila.clusterconfig.simple">ClusterConfig service</a>. With the "simple" ClusterConfig service, it uses the properties in the "taskmanager" section of <code>clusterconfig.json</code>: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-javascript"><span class="br0">{</span> |
| ... |
| <span class="st0">"taskmanager"</span>: <span class="br0">{</span> |
| <span class="st0">"maxScaleUp"</span> : <span class="nu0">16</span>, |
| <span class="st0">"maxRetries"</span>: <span class="nu0">10</span>, |
| <span class="st0">"timeToLive"</span>: <span class="nu0">300</span>, |
| <span class="st0">"resumeJobs"</span>: <span class="kw2">false</span> |
| <span class="br0">}</span>, |
| ... |
| <span class="br0">}</span></pre></div> |
| <ul><li> maxScaleUp: Maximum number of scale-up tasks (tasks for worker that set the "workerHost" property in the Get-Task request, see below) allowed on a single node. Usually this covers tasks for workers that are not marked as "runAlways". If a worker tries to get a task when the limit is reached, the taskmanager will answer as if there was no task available currently. Meaning, if the maxScaleUp value of the taskmanager is less or equal to the sum of all the maxScaleUp values of the workers and pipelets no asynchronous work will be done. See <a href="WorkerManager.html#ScaleUp" title="SMILA/Documentation/WorkerManager">WorkerManager</a>, too. |
| </li><li> timeToLive: Time in seconds before an in-progress task is finished with a recoverable error result when it is not kept alive by the worker. Default is 300 seconds. Whether the task is repeated, committed or fails completely depends on the worker description and job manager configuration. |
| </li><li> maxRetries: Used by the JobManager to decide how often a task should be retried that has failed with an RECOVERABLE_ERROR, either because the "timeToLive" was exceeded or the worker itself reported such an error. If the retry limit is reached, the task will finally fail with a FATAL_ERROR. (Technically, this property is used by the <a href="JobManager.html" title="SMILA/Documentation/JobManager">Jobmanager</a>, not the TaskManager itself) |
| </li><li> resumeJobs: By default, all information about running jobs and current tasks is removed when the SMILA process is stopped or killed and restarted, no jobs are running then and no tasks are in todo or in progress. Setting the "resumeJobs" flag to true causes SMILA to resume all jobs and continue to process the tasks leftover from the previous process. See more details on resuming jobs below. |
| </li></ul> |
| <a name="External_REST_API"></a><h2> <span class="mw-headline"> External REST API </span></h2> |
| <p>The external ReST API provides a number of endpoints for use by the administrator to watch the state of the taskmanager. This includes getting numbers of tasks in the various task pipes and being able to look into the content of tasks. |
| </p> |
| <a name="TaskManager_State_Overview"></a><h3> <span class="mw-headline"> TaskManager State Overview </span></h3> |
| <p>Use a GET request to get information about taskmanager state. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information about taskmanager task state. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/smila/tasks</tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution. |
| </li><li>500 INTERNAL SERVER ERROR: If the taskmanager is not operational anymore because too many nodes are already dead. |
| </li></ul> |
| </li></ul> |
| <p>The taskmanager state consists of three parts: |
| </p> |
| <ul><li>The failsafety value is the maximum number of nodes that may fail before the taskmanager stops working. Nodes that are already down at this moment are not subtracted from the number. |
| </li><li>After that the workers sections with counters for tasks todo or in inprogress for each worker and links to get more details about one type of tasks in a single worker follows. It always includes a _finishingTasks worker. This is an internal worker that picks up real worker tasks that have been finished by the worker or the taskmanager (in case of missing keep-alives or canceled workflow run) and sends them to the JobManager for really finishing them (aggregating counters, computing follow-up tasks, etc.) |
| </li><li>Finally there is an overview on the scale-up situation: first "maxScaleUp" describes the configured maximum number of tasks to be processed by scaling workers at the same time. A value of -1 means that there is no limit configured. Then "scaleUp" gives an array of maps each containing a host name (as in cluster.ini) and a counter describing the number of currently working scaling worker on this host. "runAlways" workers do not contribute to this counter, and a host on which no scaling worker has yet started to work will not occur in the list. |
| </li></ul> |
| <a name="Worker_Overview"></a><h3> <span class="mw-headline"> Worker Overview </span></h3> |
| <p>Use a GET request to get information about a specific worker. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/smila/tasks/<worker>/<type>/</tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET: shows the number of tasks and links to the first 100 tasks for a given worker and type. The <type> can currently be: |
| <ul><li> todo: tasks not yet fetched by any worker (or retried) |
| </li><li> inprogress: tasks currently worked on by a worker. |
| </li></ul> |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution. |
| </li><li>404 NOT FOUND: bad URL path, e.g. bad worker or type |
| </li><li>500 INTERNAL SERVER ERROR: If the taskmanager not available or is not operational anymore because too many nodes are already dead. |
| </li></ul> |
| </li></ul> |
| <a name="Task_Content"></a><h3> <span class="mw-headline"> Task Content </span></h3> |
| <p>Shows the content of a task in JSON format. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information about specific task. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/smila/tasks/<worker>/<type>/<taskname>/</tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET: Shows the content of a task in JSON format. Additionally the creation and modification timestamps are given. For todo tasks they should be identical, for inprogress tasks the modification timestamp is the timestamp of the last keepAlive request that was successfully executed. The <taskname> is not necessarily the task ID, but an internal name for the task as created by the /smila/tasks/<worker>/<type>/ handler. In addition to todo and inprogress, <type> can be todo_part in partitioned pipes. Tasks that are inprogress have an additional property inProgressBy that indicates which thread currently processes the task. |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution. |
| </li><li>404 NOT FOUND: bad URL path, e.g. bad worker or type |
| </li><li>500 INTERNAL SERVER ERROR: If the taskmanager not available or is not operational anymore because too many nodes are already dead. |
| </li></ul> |
| </li></ul> |
| <a name="Tasks"></a><h2> <span class="mw-headline"> Tasks </span></h2> |
| <p>The rest of this page is only of interest for you if you want to create a worker without using the <a href="WorkerManager.html" title="SMILA/Documentation/WorkerManager">WorkerManager</a> framework and need to implement the interaction with the TaskManager on your own. This is the case if you need to create a worker that cannot run inside of SMILA for whatever reason, so you would need the HTTP API of the TaskManager to get, keep alive and finish tasks. First we describe the JSON representation of tasks and then the REST URLs for communication with the TaskManager. |
| </p> |
| <a name="JSON_format_of_task_.281.29"></a><h3> <span class="mw-headline"> JSON format of task (1) </span></h3> |
| <pre> |
| { |
| "taskId": "...", |
| "workerName": "...", |
| <"qualifier": "...">, |
| "properties": { |
| "name": "value", ... |
| } |
| "parameters": { |
| "name": "value", ... |
| } |
| "input": { |
| "slot1": [ { "bucket": "...", "store": "...", "id": "..." } , ... ], |
| "slot2": [ { "bucket": "...", "store": "...", "id": "..." } , ... ], |
| ... |
| } |
| "output": { |
| "slot1": [ { "bucket": "...", "store": "...", "id": "..." } , ... ], |
| "slot2": [ { "bucket": "...", "store": "...", "id": "..." } , ... ], |
| ... |
| } |
| } |
| </pre> |
| <p>A task consists of |
| </p> |
| <ul><li> task ID, workerName, qualifier: needed by the taskmanager to manage the task. |
| </li><li> properties: used by the jobmanager to associate the task to its job/workflow run. The taskmanager may also add properties. |
| <ul><li> "recoverable": if set to 'false' this task will not be retried after a timeout or after a recoverable worker error |
| </li><li> "workerHost": the host where the worker is running that requested the task (only set if worker sets host as request parameter |
| </li><li> "createdTime", "startTime", "taskAge": The time the task was created, retrieved by the worker and the difference in milliseconds. Apart from statistical purposes this can for example be used by a worker to decide that a task must be processed instead of being postponed because some age limit has been reached. |
| </li></ul> |
| </li><li> parameters: worker parameters as defined in workflow and job definition. |
| </li><li> input: Lists of bulk infos associated the worker's input slots describing which data has to be processed to complete this task. May be empty for "initial tasks" (e.g. BulkBuilder). |
| </li><li> output: Lists of bulk infos associated to the worker's output slots describing where to put the results of the task (currently it's always a single bulk info per slot). May be empty completely (e.g. for the HSSI record deleter worker). |
| </li></ul> |
| <p>The "bucket" name in the bulk info is usually irrelevant to the worker, it just needs to read the "store" and "id" to be able to find and create data objects. |
| </p> |
| <a name="JSON_format_of_qualifier_condition_.282.29"></a><h3> <span class="mw-headline"> JSON format of qualifier condition (2) </span></h3> |
| <pre> |
| { |
| "qualifier": [ "parts/abcd", "parts/1234", ... ] |
| <, "workerHost": "..."> |
| } |
| </pre> |
| <p>The qualifier is used in POST requests to <tt>/taskmanager/<worker></tt> to get only tasks that have one of the given set as the "qualifier" field. This is used for example by the HSSI record delete worker to receive only tasks for certain partitions. The qualifier can be set in a task by adding the mode "qualifier" to the the input slot of a worker in which case the "id" value of the bulk in this slot is used as the qualifier. Note that the qualifier will be the complete object ID path, not only the UUID part. The workerHost must be set by scaling workers so that taskmanager can control the maximum number of tasks delivered to such workers. |
| </p> |
| <a name="JSON_format_of_result_description_.283.29"></a><h3> <span class="mw-headline"> JSON format of result description (3) </span></h3> |
| <pre> |
| { |
| "status": < "SUCCESSFUL", "FATAL_ERROR", "RECOVERABLE_ERROR", "POSTPONE" >, |
| "errorCode": "...", |
| "errorMessage": "...", |
| "counters": { |
| "name": numbervalue, |
| ... |
| } |
| } |
| </pre> |
| <p>The result description is added to the "task finished" request so that the JobManager can decide on what to do next based on the result of the task. It consists of: |
| </p> |
| <ul><li> status: one of |
| <ul><li> SUCCESSFUL: the task has been processed completely and successfully. Follow-up tasks can be generated. |
| </li><li> FATAL_ERROR: the task cannot be processed and should not be repeated, e.g. input data is corrupt. This leads to cancelling the complete associated workflow run as failed, so no further tasks can be created in this workflow run. (The job run as a whole is continued, of course). |
| </li><li> RECOVERABLE_ERROR: the task could not be processed <b>now</b>, e.g. because input data was temporary not available. Usually the job manager will repeat this task until a configured retry limit is reached. However, special workers may specify that the task should not be repeated in this case but follow-up tasks should be created nethertheless. |
| </li><li> POSTPONE: the worker cannot yet perform this task for some reason (no error, just waiting for a condition in the system) but it should be processed later. The task will be readded to the todo queue for this worker and redelivered later (but very soon, usually). There is no limit to retrying a task after postponing it. |
| </li></ul> |
| </li><li> errorCode/errorMessage: will currently be logged only. Could be merged to the job run result data in later versions to give an overview about errors that happened during the run. |
| </li><li> counters: Integer or floating point numbers giving statistical information about the task processing. Not evaluated by task/job manager currently, but may be aggregated in the job run data in a later version. |
| </li></ul> |
| <a name="Internal_REST_API"></a><h2> <span class="mw-headline"> Internal REST API </span></h2> |
| <a name="Get_task_information"></a><h3> <span class="mw-headline"> Get task information </span></h3> |
| <p>Use a GET request to get information about a specific task. Use POST for getting information with qualifier condition. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information about task. |
| </li><li>POST: get information about task with qualifier condition. Qualifier condition is provided with JSON body (2). |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/taskmanager/<workername></tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET |
| </li><li>POST |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK + JSON Body (1): Upon successful execution. |
| </li><li>204 NO CONTENT: No task available. |
| </li></ul> |
| </li></ul> |
| <a name="Get_initial_task_information"></a><h3> <span class="mw-headline"> Get initial task information </span></h3> |
| <p>Use a GET request to get initial task for given job. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get initial task for given job. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/taskmanager/<workername>/initialTask/<jobname></tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK + JSON Body (1): Upon successful execution. |
| </li><li>204 NO CONTENT: No task available. |
| </li></ul> |
| </li></ul> |
| <a name="Get_task_for_scaling_worker"></a><h3> <span class="mw-headline"> Get task for scaling worker </span></h3> |
| <p>Use a GET request to get task for scaling worker. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get task for scaling worker. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/taskmanager/<workername>?workerHost=<hostname></tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET== External REST API == |
| </li></ul> |
| </li></ul> |
| <p>The external ReST API provides a number of endpoints for use by the administrator to watch the state of the taskmanager. This includes getting numbers of tasks in the various task pipes and being able to look into the content of tasks. |
| </p> |
| <a name="TaskManager_State_Overview_2"></a><h3> <span class="mw-headline"> TaskManager State Overview </span></h3> |
| <p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information about taskmanager task state. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/smila/tasks/<worker>/<type>/</tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET: shows the number of tasks and links to the first 100 tasks for a given worker and type. The <type> can currently be: |
| </li><li> todo: tasks not yet fetched by any worker (or retried) |
| </li><li> inprogress: tasks currently worked on by a worker. |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution. |
| </li><li>404 NOT FOUND: bad URL path, e.g. bad worker or type |
| </li><li>500 INTERNAL SERVER ERROR: If the taskmanager not available or is not operational anymore because too many nodes are already dead. |
| </li></ul> |
| </li></ul> |
| <p>Use a GET request to get information about taskmanager state. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>GET: get information about taskmanager task state. |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/smila/tasks</tt>. |
| </li><li>Allowed methods: |
| <ul><li>GET |
| </li></ul> |
| </li><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution. |
| </li><li>500 INTERNAL SERVER ERROR: If the taskmanager is not operational anymore because too many nodes are already dead. |
| </li></ul> |
| </li></ul> |
| <p>The taskmanager state consists of three parts: |
| </p> |
| <ul><li>The failsafety value is the maximum number of nodes that may fail before the taskmanager stops working. Nodes that are already down at this moment are not subtracted from the number. |
| </li><li>Follows the workers sections with counters for tasks todo or in inprogress for each worker and links to get more details about one type of tasks in a single worker. It always includes a _finishingTasks worker. This is an internal worker that picks up real worker tasks that have been finished by the worker or the taskmanager (in case of missing keep-alives or canceled workflow run) and sends them to the JobManager for really finishing them (aggregating counters, computing follow-up tasks, etc.) |
| </li><li>Finally there is an overview on the scale-up situation: first "maxScaleUp" describes the configured maximum number of tasks to be processed by scaling workers at the same time. A value of -1 means that there is no limit configured. Then "scaleUp" gives an array of maps each containing a host name (as in cluster.ini) and a counter describing the number of currently working scaling worker on this host. "runAlways" workers do not contribute to this counter, and a host on which no scaling worker has yet started to work will not occur in the list. |
| </li><li>Response status codes: |
| <ul><li>200 OK + JSON Body (1): Upon successful execution. |
| </li><li>204 NO CONTENT: No task available. |
| </li></ul> |
| </li></ul> |
| <a name="Finish_or_keep_alive_task"></a><h3> <span class="mw-headline"> Finish or keep alive task </span></h3> |
| <p>Use a POST request and JSON body (3) to finish a task or use POST and empty body to send a keep-alive signal for a task in progress. |
| </p><p><b>Supported operations:</b> |
| </p> |
| <ul><li>POST: with body finish task, without body send keep-alive |
| </li></ul> |
| <p><b>Usage:</b> |
| </p> |
| <ul><li>URL: <tt>http://<hostname>:8080/taskmanager/<workername>/<taskId></tt>. |
| </li><li>Allowed methods: |
| <ul><li>POST |
| </li></ul> |
| </li></ul> |
| <ul><li>Response status codes: |
| <ul><li>200 OK: Upon successful execution for finishing a task with result description. |
| </li><li>202 ACCEPTED: task is kept alive during progress. |
| </li></ul> |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 67/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:32482-0!1!0!!en!2!edit=0 and timestamp 20130416060946 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="TaskManager.html">http://wiki.eclipse.org/SMILA/Documentation/TaskManager</a>"</div> |
| <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2013 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 09:16, 16 January 2013 by <a href="http://wiki.eclipse.org/index.php?title=User:Dhaenssgen.brox.de&action=edit" class="new" title="User:Dhaenssgen.brox.de">Daniel Hänßgen</a>. Based on work by <a href="http://wiki.eclipse.org/index.php?title=User:Juergen.schumacher.attensity.com&action=edit" class="new" title="User:Juergen.schumacher.attensity.com">Juergen Schumacher</a>, <a href="http://wiki.eclipse.org/index.php?title=User:Nadine.auslaender.attensity.com&action=edit" class="new" title="User:Nadine.auslaender.attensity.com"> </a> and <a href="http://wiki.eclipse.org/User:Andreas.schank.attensity.com" title="User:Andreas.schank.attensity.com">A. Schank</a> and <a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/TaskManager&action=credits" title="SMILA/Documentation/TaskManager">others</a>.</p> |
| <p id="footerviews">This page has been accessed 1,604 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.055 secs. --></body></html> |
