| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Documentation/Worker/PipeletProcessorWorker,SMILA/Development Guidelines/How to write a Pipelet,SMILA/Documentation/SesameOntologyManager,SMILA/Glossary" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Documentation/Worker/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Documentation/Worker/PipeletProcessorWorker - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "http://wiki.eclipse.org/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Documentation/Worker/PipeletProcessorWorker"; |
| var wgTitle = "SMILA/Documentation/Worker/PipeletProcessorWorker"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "33414"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "285843"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <link rel="stylesheet" type="text/css" href="PipeletProcessorWorker.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Documentation_Worker_PipeletProcessorWorker"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/Worker/PipeletProcessorWorker">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Documentation/Worker/PipeletProcessorWorker">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Documentation/Worker/PipeletProcessorWorker">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/Worker/PipeletProcessorWorker&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/Worker/PipeletProcessorWorker&oldid=285843">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="PipeletProcessorWorker.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Documentation/Worker/PipeletProcessorWorker&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/Worker/PipeletProcessorWorker&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/Worker/PipeletProcessorWorker&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/Worker/PipeletProcessorWorker"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Documentation/Worker/PipeletProcessorWorker</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../../SMILA.html" title="SMILA">SMILA</a> | <a href="../../Documentation.1.html" title="SMILA/Documentation">Documentation</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="PipeletProcessorWorker.html#column-one">navigation</a>, <a href="PipeletProcessorWorker.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="PipeletProcessorWorker.html#PipeletProcessorWorker_.28bundle_org.eclipse.smila.processing.worker.29"><span class="tocnumber">1</span> <span class="toctext">PipeletProcessorWorker (bundle org.eclipse.smila.processing.worker)</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="PipeletProcessorWorker.html#JavaDoc"><span class="tocnumber">1.1</span> <span class="toctext">JavaDoc</span></a></li> |
| <li class="toclevel-2"><a href="PipeletProcessorWorker.html#Configuration"><span class="tocnumber">1.2</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="PipeletProcessorWorker.html#PipeletProcessorWorker_definition_in_workers.json"><span class="tocnumber">1.3</span> <span class="toctext">PipeletProcessorWorker definition in workers.json</span></a></li> |
| <li class="toclevel-2"><a href="PipeletProcessorWorker.html#Access_task_parameters_in_pipelets"><span class="tocnumber">1.4</span> <span class="toctext">Access task parameters in pipelets</span></a></li> |
| <li class="toclevel-2"><a href="PipeletProcessorWorker.html#Error_handling"><span class="tocnumber">1.5</span> <span class="toctext">Error handling</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="PipeletProcessorWorker_.28bundle_org.eclipse.smila.processing.worker.29"></a><h1> <span class="mw-headline"> PipeletProcessorWorker (bundle org.eclipse.smila.processing.worker) </span></h1> |
| <p>The PipeletProcessorWorker is a <a href="../../Glossary.html#W" title="SMILA/Glossary">worker</a> designed to execute a single pipelet directly, without pipeline overhead. |
| </p> |
| <a name="JavaDoc"></a><h2> <span class="mw-headline"> JavaDoc </span></h2> |
| <p>This page gives only a rough overview of the service. Please refer to the <a href="http://build.eclipse.org/rt/smila/javadoc/current/index.html?org/eclipse/smila/processing/worker/PipeletProcessorWorker.html" class="external text" title="http://build.eclipse.org/rt/smila/javadoc/current/index.html?org/eclipse/smila/processing/worker/PipeletProcessorWorker.html" rel="nofollow">JavaDoc</a> for detailed information about the Java components. |
| </p> |
| <a name="Configuration"></a><h2> <span class="mw-headline"> Configuration </span></h2> |
| <p>The PipeletProcessorWorker is configured via incoming task parameters. These parameters could have been set e.g. in a <a href="../../Glossary.html#J" title="SMILA/Glossary">job</a> definition. |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Parameter |
| </th><th> Description |
| </th><th> Default value |
| </th></tr> |
| <tr> |
| <td> <tt>pipeletName</tt> |
| </td><td> Name of the pipelet to execute |
| </td><td> --- |
| </td></tr> |
| <tr> |
| <td> <tt>keepAttachmentsInMemory</tt> |
| </td><td> By default, attachments on processed records are kept in memory. If you don't have much memory or very large attachments it may be useful to set this parameter to false and have a BinaryStorage service activated. Then attachments are stored in BinaryStorage while processing and less memory is used. However, processing will probably be slower in this case. After processing the attachments will be removed from BinaryStorage again. If no BinaryStorage service is active, all attachments will be kept in memory and this parameter will not have any effect. |
| </td><td> true |
| </td></tr> |
| <tr> |
| <td> <tt>writeAttachmentsToOutput</tt> |
| </td><td> By default, attachments on incoming records are also added to the output records (if any are written). If this parameter is set to <tt>false</tt>, only record metadata is written to the output bulk. This can save a lot of IO if attachments are not needed anymore in the workflow after this worker. |
| </td><td> true |
| </td></tr> |
| </table> |
| <p>Sample job definition that sets the parameters: |
| </p> |
| <pre> |
| { |
| "name":"myJob", |
| "parameters":{ |
| "pipeletName": "MySamplePipelet", |
| ... |
| }, |
| "workflow":"myWorkflow" |
| } |
| </pre> |
| <a name="PipeletProcessorWorker_definition_in_workers.json"></a><h2> <span class="mw-headline"> PipeletProcessorWorker definition in workers.json </span></h2> |
| <pre> |
| GET /smila/jobmanager/workers/pipeletProcessor/ |
| |
| HTTP/1.x 200 OK |
| |
| { |
| "name" : "pipeletProcessor", |
| "readOnly" : true, |
| "parameters" : [ |
| { |
| "name" : "pipeletName" |
| }, |
| { |
| "name" : "<pipeletParameters>", |
| "optional" : true, |
| "type" : "any" |
| } |
| ], |
| "input" : [ { |
| "name" : "input", |
| "type" : "recordBulks" |
| } ], |
| "output" : [ { |
| "name" : "output", |
| "type" : "recordBulks", |
| "modes" : [ "optional" ] |
| } ] |
| } |
| </pre> |
| <p>The output bucket of the worker is optional, hence in an <a href="../../Glossary.html#W" title="SMILA/Glossary">asynchronous workflow</a> the worker does not need to have a successor. If the output bucket is not defined, the result records of the pipeline processing are not persisted to a bulk, but thrown away. This makes sense if the pipelet stores the records somewhere itself, e.g. adds them to an index. |
| </p> |
| <a name="Access_task_parameters_in_pipelets"></a><h2> <span class="mw-headline"> Access task parameters in pipelets </span></h2> |
| <p>The worker adds all task parameters to a map in attribute <tt>_parameters</tt> in each record before giving it to the pipelet which can access them. The helper class <tt>org.eclipse.smila.processing.parameters.ParameterAccesssor</tt> supports this by checking for requested parameters first in this <tt>_parameters</tt> map, then at the top-level of a record and then in the pipelet configuration. Therefore it's possible to override properties from the pipelet configuration by setting them as task parameters, if the pipelet uses the ParameterAccessor to access parameters in records and configuration. This is done for example by the <a href="../SesameOntologyManager.html" title="SMILA/Documentation/SesameOntologyManager">Sesame</a> pipelets. |
| </p><p>If the internal parameter <tt>_failOnError</tt> was not set before, the worker will set the parameter to "false". This means that the called pipelet should continue processing records and not stop when processing defect records. The pipelet itself must implement this behavior. How to achieve this is explained in <a href="../../Development_Guidelines/How_to_write_a_Pipelet.html" class="mw-redirect" title="SMILA/Development Guidelines/How to write a Pipelet">How to write a Pipelet</a>. |
| </p> |
| <a name="Error_handling"></a><h2> <span class="mw-headline"> Error handling </span></h2> |
| <p>The following errors may occur when a <a href="../../Glossary.html#T" title="SMILA/Glossary">task</a> for the PipeletProcessorWorker is processed: |
| </p> |
| <ul><li> Pipelet parameter missing or invalid parameter |
| <ul><li> If the given pipelet parameter is not set (or invalid) the task will fail with a non-recoverable error. |
| </li></ul> |
| </li><li> ProcessingException while processing a bunch of parallel records. |
| <ul><li> Recoverable <tt>ProcessingException</tt>: The current task will fail with a recoverable error, so the whole task (with all records) will be repeated. |
| </li><li> Non-recoverable <tt>ProcessingException</tt>: An error will be logged and the worker will continue with the next bunch of records. The records of the current bunch will be lost. (This is implemented in a way as to <i>not</i> fail the whole task with all its input records in case of a single record defect.) |
| </li></ul> |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 25/1000000 |
| Post-expand include size: 174/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:33414-0!1!0!!en!2!edit=0 and timestamp 20130416060951 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="PipeletProcessorWorker.html">http://wiki.eclipse.org/SMILA/Documentation/Worker/PipeletProcessorWorker</a>"</div> |
| <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2013 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 15:54, 23 January 2012 by <a href="http://wiki.eclipse.org/index.php?title=User:Juergen.schumacher.attensity.com&action=edit" class="new" title="User:Juergen.schumacher.attensity.com">Juergen Schumacher</a>. Based on work by <a href="http://wiki.eclipse.org/User:Andreas.Weber.empolis.com" title="User:Andreas.Weber.empolis.com">Andreas Weber</a>, <a href="http://wiki.eclipse.org/index.php?title=User:Nadine.auslaender.attensity.com&action=edit" class="new" title="User:Nadine.auslaender.attensity.com"> </a> and <a href="http://wiki.eclipse.org/User:Drazen.cindric.attensity.com" title="User:Drazen.cindric.attensity.com">Drazen Cindric</a>.</p> |
| <p id="footerviews">This page has been accessed 1,146 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.073 secs. --></body></html> |
