| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Documentation/QueueWorker,SMILA/Documentation/QueueWorker,SMILA/Documentation/Record Storage,SMILA/Documentation/Usage of Blackboard Service" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Documentation/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Documentation/QueueWorker - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Documentation/QueueWorker"; |
| var wgTitle = "SMILA/Documentation/QueueWorker"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "15400"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "247283"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <style type="text/css">/*<![CDATA[*/ |
| .source-java {line-height: normal; font-size: medium;} |
| .source-java li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for java |
| * CSS class: source-java, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-java .de1, .source-java .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-java {} |
| .source-java .head {} |
| .source-java .foot {} |
| .source-java .imp {font-weight: bold; color: red;} |
| .source-java .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-java li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-java li.li2 {font-weight: bold;} |
| .source-java .kw1 {color: #7F0055; font-weight: bold;} |
| .source-java .kw2 {color: #7F0055; font-weight: bold;} |
| .source-java .kw3 {color: #000000; font-weight: normal} |
| .source-java .kw4 {color: #7F0055; font-weight: bold;} |
| .source-java .co1 {color: #3F7F5F; font-style: italic;} |
| .source-java .co2 {color: #3F7F5F;} |
| .source-java .co3 {color: #3F7F5F; font-style: italic; font-weight: bold;} |
| .source-java .coMULTI {color: #3F5FBF; font-style: italic;} |
| .source-java .es0 {color: #000000;} |
| .source-java .br0 {color: #000000;} |
| .source-java .st0 {color: #2A00ff;} |
| .source-java .nu0 {color: #000000;} |
| .source-java .me1 {color: #000000;} |
| .source-java .me2 {color: #000000;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><style type="text/css">/*<![CDATA[*/ |
| .source-xml {line-height: normal; font-size: medium;} |
| .source-xml li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for xml |
| * CSS class: source-xml, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-xml .de1, .source-xml .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-xml {} |
| .source-xml .head {} |
| .source-xml .foot {} |
| .source-xml .imp {font-weight: bold; color: red;} |
| .source-xml .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-xml li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-xml li.li2 {font-weight: bold;} |
| .source-xml .coMULTI {color: #808080; font-style: italic;} |
| .source-xml .es0 {color: #000099; font-weight: bold;} |
| .source-xml .br0 {color: #66cc66;} |
| .source-xml .st0 {color: #ff0000;} |
| .source-xml .nu0 {color: #cc66cc;} |
| .source-xml .sc0 {color: #00bbdd;} |
| .source-xml .sc1 {color: #ddbb00;} |
| .source-xml .sc2 {color: #339933;} |
| .source-xml .sc3 {color: #009900;} |
| .source-xml .re0 {color: #000066;} |
| .source-xml .re1 {font-weight: bold; color: black;} |
| .source-xml .re2 {font-weight: bold; color: black;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><style type="text/css">/*<![CDATA[*/ |
| .source-text {line-height: normal; font-size: medium;} |
| .source-text li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for text |
| * CSS class: source-text, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-text .de1, .source-text .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-text {} |
| .source-text .head {} |
| .source-text .foot {} |
| .source-text .imp {font-weight: bold; color: red;} |
| .source-text .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-text li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-text li.li2 {font-weight: bold;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><link rel="stylesheet" type="text/css" href="QueueWorker.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Documentation_QueueWorker"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/QueueWorker">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Documentation/QueueWorker">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Documentation/QueueWorker">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/QueueWorker&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/QueueWorker&oldid=247283">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="QueueWorker.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Documentation/QueueWorker&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/QueueWorker&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/QueueWorker&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/QueueWorker"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Documentation/QueueWorker</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Documentation.html" title="SMILA/Documentation">Documentation</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="QueueWorker.html#column-one">navigation</a>, <a href="QueueWorker.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="QueueWorker.html#Introduction"><span class="tocnumber">1</span> <span class="toctext">Introduction</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#QueueWorker_Bundle"><span class="tocnumber">1.1</span> <span class="toctext">QueueWorker Bundle</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#JMS_Broker"><span class="tocnumber">2</span> <span class="toctext">JMS Broker</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#JMS"><span class="tocnumber">2.1</span> <span class="toctext">JMS</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#JMS_Queues_and_Topics"><span class="tocnumber">2.2</span> <span class="toctext">JMS Queues and Topics</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#JMS_Broker_2"><span class="tocnumber">2.3</span> <span class="toctext">JMS Broker</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Broker_Configuration"><span class="tocnumber">2.4</span> <span class="toctext">Broker Configuration</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#Broker_Connection"><span class="tocnumber">3</span> <span class="toctext">Broker Connection</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Configuration"><span class="tocnumber">3.1</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Example"><span class="tocnumber">3.2</span> <span class="toctext">Example</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#JMS_Message_Properties"><span class="tocnumber">4</span> <span class="toctext">JMS Message Properties</span></a></li> |
| <li class="toclevel-1"><a href="QueueWorker.html#Router"><span class="tocnumber">5</span> <span class="toctext">Router</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Configuration_2"><span class="tocnumber">5.1</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Example_2"><span class="tocnumber">5.2</span> <span class="toctext">Example</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#Listener"><span class="tocnumber">6</span> <span class="toctext">Listener</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Configuration_3"><span class="tocnumber">6.1</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Example_3"><span class="tocnumber">6.2</span> <span class="toctext">Example</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#RecordRecycler"><span class="tocnumber">7</span> <span class="toctext">RecordRecycler</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Configuration_4"><span class="tocnumber">7.1</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Usage_of_JMX_Management_Agent"><span class="tocnumber">7.2</span> <span class="toctext">Usage of JMX Management Agent</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#In_Detail:_QueueWorker_Rules.2C_Conditions_and_Tasks"><span class="tocnumber">8</span> <span class="toctext">In Detail: QueueWorker Rules, Conditions and Tasks</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Rule_attributes"><span class="tocnumber">8.1</span> <span class="toctext">Rule attributes</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Source"><span class="tocnumber">8.2</span> <span class="toctext">Source</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Condition"><span class="tocnumber">8.3</span> <span class="toctext">Condition</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Tasks"><span class="tocnumber">8.4</span> <span class="toctext">Tasks</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="QueueWorker.html#Process"><span class="tocnumber">8.4.1</span> <span class="toctext">Process</span></a></li> |
| <li class="toclevel-3"><a href="QueueWorker.html#Send"><span class="tocnumber">8.4.2</span> <span class="toctext">Send</span></a> |
| <ul> |
| <li class="toclevel-4"><a href="QueueWorker.html#RecordFilter"><span class="tocnumber">8.4.2.1</span> <span class="toctext">RecordFilter</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#Tips"><span class="tocnumber">9</span> <span class="toctext">Tips</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="QueueWorker.html#Persistent_Queues"><span class="tocnumber">9.1</span> <span class="toctext">Persistent Queues</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Redelivery_and_Dead_Letter_Queue"><span class="tocnumber">9.2</span> <span class="toctext">Redelivery and Dead Letter Queue</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Automatic_reconnect"><span class="tocnumber">9.3</span> <span class="toctext">Automatic reconnect</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Preprocessing_by_using_JMS_properties"><span class="tocnumber">9.4</span> <span class="toctext">Preprocessing by using JMS properties</span></a></li> |
| <li class="toclevel-2"><a href="QueueWorker.html#Preprocessing_by_using_several_queues"><span class="tocnumber">9.5</span> <span class="toctext">Preprocessing by using several queues</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="QueueWorker.html#Useful_Links"><span class="tocnumber">10</span> <span class="toctext">Useful Links</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Introduction"></a><h2> <span class="mw-headline"> Introduction </span></h2> |
| <p>The QueueWorker components provide a bridge between Connectivity and OSGi-Runtime Processes that are responsible for processing the crawled content. |
| The Connectivity is responsible for interacting with crawlers and agents, which in turn gather raw data information from various data sources and pack them into records. Data Flow Processes (DFP) allow manipulating or post processing the crawled information in those records. In most cases the processed data is stored in some form of a store or an index at the end of a DFP. |
| Because not all data is equal (e.g. post processed the same way, except in the case of most simple of setups), the QueueWorker components are used to determine which record needs to be feed into which DFP. It thus acts as a router between the raw record and its post processing. |
| To decide which DFP is the right one for a records, the QW investigates the message properties and assigns the DFP based on configured rules. |
| The QueueWorker components are developed for the usage with JMS (Java Message Service). SMILA is distributed with an embedded JMS provider: Apache ActiveMQ. |
| </p><p><a href="http://wiki.eclipse.org/Image:Workflow.jpg" class="image" title="Workflow"><img alt="Workflow" src="http://wiki.eclipse.org/images/3/35/Workflow.jpg" width="584" height="121" border="0" /></a> |
| </p> |
| <a name="QueueWorker_Bundle"></a><h3> <span class="mw-headline"> QueueWorker Bundle </span></h3> |
| <p>The QueueWorker contains different components/features to take records in and out of queues. They are located in bundle org.eclipse.smila.connectivity.queue.worker.jms. |
| </p><p>The Router is needed to receive records from the Connectivity Framework, filter them and push them into queues. The Listener component of the QueueWorker is used as part of an OSGi runtime that processes the crawled information. It takes entries from the queue and calls one or more processing pipelines. Optionally, the processed records may be resend to a queue again. The QueueWorker contains also a Recycler component that can take records (which usually contain crawled or manipulated information) from storages (usually a <a href="Record_Storage.html" title="SMILA/Documentation/Record Storage">RecordStorage</a>) and send them back to queues. All components are configured with “Rules” which execute “Tasks”. |
| </p> |
| <a name="JMS_Broker"></a><h2> <span class="mw-headline"> JMS Broker </span></h2> |
| <a name="JMS"></a><h3> <span class="mw-headline"> JMS </span></h3> |
| <p>JMS stands for Java Massage Service. The API is a messaging standard which allows applications written in Java to create, send, receive and read messages. An implementation of the JMS API is called a <i>JMS Provider</i>. SMILA embeds <a href="http://activemq.apache.org" class="external text" title="http://activemq.apache.org" rel="nofollow">ActiveMQ</a> as default JMS provider. |
| </p> |
| <a name="JMS_Queues_and_Topics"></a><h3> <span class="mw-headline"> JMS Queues and Topics </span></h3> |
| <p>A <i>JMS Queue</i> is a container for messages that should be received by exactly one consumer. Messages are delivered in the order they were sent. A message is removed from the queue once it has been read successfully. |
| </p><p>In opposite to JMS Queues, so called <i>JMS Topics</i> implement a publish and subscribe semantic: When you publish a message it goes to all the subscribers who are interested - so zero to many subscribers will receive a copy of the message. |
| </p><p><b>Note</b>: JMS Topics are not used/supported in SMILA at the current state of implementation |
| </p><p><b>Note on ActiveMQ</b>: In ActiveMQ you do not have to create/configure destinations (= queues/topics) up front before you can use them. The ActiveMQ broker auto-creates the physical resources associated with a destination on demand (i.e. when messages are sent to a new destination on a broker). |
| </p> |
| <a name="JMS_Broker_2"></a><h3> <span class="mw-headline"> JMS Broker </span></h3> |
| <p>The JMS Message Broker is the main component on the server side of a JMS provider. It maintains queues and connections, routes messages to queues, acknowledges messages and handles transactions. In SMILA, an ActiveMQ Broker is embedded in the application per default. |
| </p> |
| <a name="Broker_Configuration"></a><h3> <span class="mw-headline"> Broker Configuration </span></h3> |
| <p>In SMILA there is a <b>jms.properties</b> file to configure the embedded JMS broker. It is located in configuration/org.eclipse.smila.connectivity.queue.broker.main |
| </p><p>Default Configuration: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-java">java.<span class="me1">naming</span>.<span class="me1">factory</span>.<span class="me1">initial</span>=org.<span class="me1">apache</span>.<span class="me1">activemq</span>.<span class="me1">jndi</span>.<span class="me1">ActiveMQInitialContextFactory</span> |
| java.<span class="me1">naming</span>.<span class="me1">provider</span>.<span class="me1">url</span>=vm:<span class="br0">(</span>broker:<span class="br0">(</span>tcp:<span class="co1">//localhost:61616?jms.useAsyncSend=true)?persistent=true)?marshal=false</span></pre></div> |
| <p>The <i>java.naming.provider.url</i> parameter in this example defines a broker to which a JMS client can connect via TCP transport protocol. |
| </p><p><b>Note on ActiveMQ:</b> If you deactivate the org.eclipse.smila.connectivity.queue.broker.main bundle on SMILA startup, an internal broker will be created automatically by ActiveMQ. This broker can only be connected from a client running in the same SMILA instance by using the VM protocol (vm://localhost). So, if you need to connect to a broker (resp. its queues) via TCP from a different machine, the org.eclipse.smila.connectivity.queue.broker.main bundle should be activated (which is default in SMILA) and jms.properties allowing a TCP connection have to be set - like it is done in the default jms.properties shown above. |
| </p><p><br /> |
| </p> |
| <a name="Broker_Connection"></a><h2> <span class="mw-headline"> Broker Connection </span></h2> |
| <p>A Broker Connection provides a cached connection pool to JMS brokers. The QueueWorker components use Broker Connections to communicate with different Queues in different JMS brokers. |
| </p> |
| <a name="Configuration"></a><h3> <span class="mw-headline"> Configuration </span></h3> |
| <p>The different components of the QueueWorker can use connections to different brokers. The connections are defined in a special configuration file. |
| A configuration has a unique Id and contains a list of specific JMS properties like: |
| </p> |
| <ol><li> URL to broker |
| </li><li> user and password for login |
| </li><li> the connection type (factory) of specific broker |
| </li></ol> |
| <p>Below is an example of the default configuration file. It is located in configuration/org.eclipse.smila.queue.worker/QueueWorkerConnectionConfig.xml: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerConnectionsConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><ConnectionConfig</span> <span class="re0">Id</span>=<span class="st0">"broker1"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><URL<span class="re2">></span></span></span>vm://localhost<span class="sc3"><span class="re1"></URL<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><User<span class="re2">></span></span></span>any<span class="sc3"><span class="re1"></User<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Password<span class="re2">></span></span></span>any<span class="sc3"><span class="re1"></Password<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><ConnectionFactory<span class="re2">></span></span></span>org.apache.activemq.ActiveMQConnectionFactory<span class="sc3"><span class="re1"></ConnectionFactory<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></ConnectionConfig<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerConnectionsConfig<span class="re2">></span></span></span></pre></div> |
| <p>The schema for this configuration file is located at org.eclipse.smila.connectivity.queue.worker/schemas/QueueWorkerConfig.xsd. |
| </p> |
| <a name="Example"></a><h3> <span class="mw-headline"> Example </span></h3> |
| <p>This example uses two completely independent brokers. Keep in mind that there should be a unique Id for every connection. |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerConnectionsConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><ConnectionConfig</span> <span class="re0">Id</span>=<span class="st0">"broker1"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><URL<span class="re2">></span></span></span>tcp://localhost:61616<span class="sc3"><span class="re1"></URL<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><User<span class="re2">></span></span></span>any<span class="sc3"><span class="re1"></User<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Password<span class="re2">></span></span></span>any<span class="sc3"><span class="re1"></Password<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><ConnectionFactory<span class="re2">></span></span></span>org.apache.activemq.ActiveMQConnectionFactory<span class="sc3"><span class="re1"></ConnectionFactory<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></ConnectionConfig<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"><ConnectionConfig</span> <span class="re0">Id</span>=<span class="st0">"broker2"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><URL<span class="re2">></span></span></span>tcp://10.0.0.1:61617<span class="sc3"><span class="re1"></URL<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><User<span class="re2">></span></span></span>any1<span class="sc3"><span class="re1"></User<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Password<span class="re2">></span></span></span>any1<span class="sc3"><span class="re1"></Password<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><ConnectionFactory<span class="re2">></span></span></span>com.ibm.mq.jms.MQQueueConnectionFactory<span class="sc3"><span class="re1"></ConnectionFactory<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><ConnectionProperty</span> <span class="re0">Name</span>=<span class="st0">"useExponentialBackOff"</span><span class="re2">></span></span>true<span class="sc3"><span class="re1"></ConnectionProperty<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></ConnectionConfig<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerConnectionsConfig<span class="re2">></span></span></span></pre></div> |
| <p><br /> |
| </p> |
| <a name="JMS_Message_Properties"></a><h2> <span class="mw-headline"> JMS Message Properties </span></h2> |
| <p>Each JMS message may contain properties. In SMILA the following properties are predefined and are automatically included with every message: |
| </p> |
| <ul><li> <b>DataSourceID</b> - the data source id |
| </li><li> <b>Operation</b> - the kind of operation (<tt>ADD</tt>, <tt>DELETE</tt> or <tt>NONE</tt> |
| </li></ul> |
| <p>SMILA also supports user defined message properties. These are simply added to each record by named values of the predefined Annotation <tt>MessageProperties</tt>. If a record contains the annotation <tt>MessageProperties</tt> each named value of it is converted to a String message property. |
| </p><p>It's also possible to add additional properties with static values in the <tt>Send</tt> task of your Router or Listener configuration (see <a href="QueueWorker.html#Send" title="">Send taskbelow</a>). |
| </p><p><br /> |
| </p> |
| <a name="Router"></a><h2> <span class="mw-headline"> Router </span></h2> |
| <p>The Router is one of the QueueWorker services. The main goal is to put record into one or more JMS queues (which are hosted/provided by a JMS broker). This happens within the <a href="QueueWorker.html#Send" title="SMILA/Documentation/QueueWorker">Send task</a>. The Router looks for every record that was crawled by a rule that fits with its <a href="QueueWorker.html#Condition" title="SMILA/Documentation/QueueWorker">condition</a>. |
| But it is also possible to configure the Router only with a <a href="QueueWorker.html#Process" title="SMILA/Documentation/QueueWorker">Process task</a>, which will effectively make the MQ obsolete which can be useful in small and resource-limited environments. In that case the router will directly put the record into the pipeline. |
| </p> |
| <div class="messagebox" style="background-color: #def3fe; border: 1px solid #c5d7e0; color: black; padding: 5px; margin: 1ex 0; min-height: 35px; padding-left: 45px;"> |
| <div style="float: left; margin-left: -40px;"><a href="http://wiki.eclipse.org/Image:Note.png" class="image" title="Note.png"><img alt="" src="http://wiki.eclipse.org/images/c/cc/Note.png" width="35" height="35" border="0" /></a></div> |
| <div><b>todo: elaborate</b><br /> |
| <ul><li> are tasks executed in the order given? |
| </li><li> are the tasks executed synchronously or asynchronously?</div> |
| </li></ul> |
| </div> |
| <a name="Configuration_2"></a><h3> <span class="mw-headline"> Configuration </span></h3> |
| <p>The configuration file is located in configuration/org.eclipse.smila.connectivity.queue.worker.jms/QueueWorkerRouterConfig.xml. |
| It simply consists of a list of routing rules. |
| </p><p>Default configuration: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerRouterConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Default Route Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span><span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerRouterConfig<span class="re2">></span></span></span></pre></div> |
| <p>The schema is located in org.eclipse.smila.connectivity.queue.worker/schemas/QueueWorkerConfig.xsd. |
| </p> |
| <a name="Example_2"></a><h3> <span class="mw-headline"> Example </span></h3> |
| <p>This example routes each record to more than one queue by defining several Send tasks: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerRouterConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Double Send Route Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span><span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"queue1"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker2"</span> <span class="re0">Queue</span>=<span class="st0">"queue2"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerRouterConfig<span class="re2">></span></span></span></pre></div> |
| <p>A detailed description of the Routing Rules can be found <a href="QueueWorker.html#In_Detail:_QueueWorker_Rules.2C_Conditions_and_Tasks" title="SMILA/Documentation/QueueWorker">here</a>. |
| </p> |
| <a name="Listener"></a><h2> <span class="mw-headline"> Listener </span></h2> |
| <p>The Listener is another QueueWorker component. This component can be used to “listen” for specific records on specific queues. If a record matching the condition of a rule exists the Listener takes the record from the queue and pushed it to BPEL pipelines. Generally the Listener works with the <a href="QueueWorker.html#Process" title="SMILA/Documentation/QueueWorker">Process task</a>, but it’s also possible to use <a href="QueueWorker.html#Send" title="SMILA/Documentation/QueueWorker">Send tasks</a> maybe for resending a record into a new JMS queue after the pipeline has finished. |
| </p> |
| <a name="Configuration_3"></a><h3> <span class="mw-headline"> Configuration </span></h3> |
| <p>The configuration is nearly the same as the Router Configuration. There are only these following add-ons: |
| </p> |
| <ul><li> <i>Source</i> element: a source reference to the used broker (BrokerId should be specified in Broker Connection configuration) |
| </li><li> <i>WaitMassageTime</i> attribute: timeout (in seconds) for attempts to pull JMS message from queue, default value is 1 second. |
| </li><li> <i>MaxMessageBlockSize</i> attribute: maximum number of messages to receive from the queue in one session. If less messages are available those are consumed without waiting for more messages. Default value is 1 message. |
| </li><li> <i>Threads</i> attribute: startup number of threads which listen to the queue under this rule, default value is 1 thread. |
| </li></ul> |
| <p>Default configuration: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerListenerConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"ADD Rule"</span> <span class="re0">WaitMessageTimeout</span>=<span class="st0">"10"</span> <span class="re0">Threads</span>=<span class="st0">"4"</span> <span class="re0">MaxMessageBlockSize</span>=<span class="st0">"20"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD' and NOT(DataSourceID LIKE '%feeds%')<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Delete Rule"</span> <span class="re0">WaitMessageTimeout</span>=<span class="st0">"10"</span> <span class="re0">Threads</span>=<span class="st0">"2"</span> <span class="re0">MaxMessageBlockSize</span>=<span class="st0">"20"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='DELETE'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"DeletePipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"ADD Feed Rule"</span> <span class="re0">WaitMessageTimeout</span>=<span class="st0">"10"</span> <span class="re0">Threads</span>=<span class="st0">"2"</span> <span class="re0">MaxMessageBlockSize</span>=<span class="st0">"20"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD' and DataSourceID LIKE '%feeds%'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddFeedPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerListenerConfig<span class="re2">></span></span></span></pre></div> |
| <p>There is a schema located in org.eclipse.smila.connectivity.queue.worker/schemas/QueueWorkerConfig.xsd. |
| </p> |
| <a name="Example_3"></a><h3> <span class="mw-headline"> Example </span></h3> |
| <p>An example for a preprocessing workflow using more than one queue can be found in the <a href="QueueWorker.html#Tips" title="SMILA/Documentation/QueueWorker">Tips section</a>. |
| </p><p>A detailed description of the Listener Rules can be found <a href="QueueWorker.html#In_Detail:_QueueWorker_Rules.2C_Conditions_and_Tasks" title="SMILA/Documentation/QueueWorker">here</a>. |
| </p><p><br /> |
| </p> |
| <a name="RecordRecycler"></a><h2> <span class="mw-headline"> RecordRecycler </span></h2> |
| <p>A RecordRecycler is a special QueueWorker component which won't be needed in a SMILA standard scenario. You can use it if you need to reprocess (re-index) your data without accessing the original data sources again. |
| </p><p>The current RecordRecycler is identical to the Router with the exception that RecordRecycler gets records from RecordStorage while the router gets them from the Connectivity. Thus it can be used to take records from the RecordStore and processes them (again) without having to recrawl the data sources. |
| </p> |
| <a name="Configuration_4"></a><h3> <span class="mw-headline"> Configuration </span></h3> |
| <p>The RecordRecycler is high configurable with the xml configuration files in configuration/org.eclipse.smila.connectivity.queue.worker.jms/recyclers. The RecordRecycler is able to execute any Router/Listener specific task. |
| </p><p>Default Configuration (recyclers1.xml): |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerRecordRecyclerJob</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"../schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"First recycle Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>DataSourceID='myDataSource'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Second recycle Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>NOT(DataSourceID='myDataSource')<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">Name</span>=<span class="st0">"doAdd"</span><span class="re2">></span></span>no<span class="sc3"><span class="re1"></SetProperty<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Send<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerRecordRecyclerJob<span class="re2">></span></span></span></pre></div> |
| <p>There is a schema located at org.eclipse.smila.connectivity.queue.worker/schemas/QueueWorkerConfig.xsd. |
| </p><p>A detailed description of the Rules can be found <a href="QueueWorker.html#In_Detail:_QueueWorker_Rules.2C_Conditions_and_Tasks" title="SMILA/Documentation/QueueWorker">here</a>. |
| </p> |
| <a name="Usage_of_JMX_Management_Agent"></a><h3> <span class="mw-headline"> Usage of JMX Management Agent </span></h3> |
| <p>The JMX Agent name is org.eclipse.smila.connectivity.queue.worker.jms.RecordRecycler. |
| The following commands are available to be executed: |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Signature |
| </th><th> Description |
| </th><th> Example |
| </th></tr> |
| <tr> |
| <td> startRecycle (String configurationID, String dataSourceID) |
| </td><td> starts the recycle process |
| </td><td> startRecycle (recycler1, file) |
| </td></tr> |
| <tr> |
| <td> stopRecycle (String dataSource ID) |
| </td><td> stops the recycle process |
| </td><td> stopRecycle (file) |
| </td></tr> |
| <tr> |
| <td> getRecordsRecycled (String dataSourceID) |
| </td><td> gets the number of recycled records |
| </td><td> getRecrodsRecycled (dataSourceID) |
| </td></tr> |
| <tr> |
| <td> getStatus (String dataSourceID) |
| </td><td> shows the current status of recycle process |
| </td><td> getStatus (file) |
| </td></tr> |
| <tr> |
| <td> getConfigurations() |
| </td><td> lists all available recycler configurations |
| </td><td> |
| </td></tr></table> |
| <p><br /> |
| </p> |
| <a name="In_Detail:_QueueWorker_Rules.2C_Conditions_and_Tasks"></a><h2> <span class="mw-headline"> In Detail: QueueWorker Rules, Conditions and Tasks </span></h2> |
| <p>QueueWorker Rules (used by Router-, Listener- and Recycler configuration) consist of: |
| </p> |
| <ul><li> Rule attributes |
| </li><li> Source (Listener configuration only) |
| </li><li> Condition |
| </li><li> Tasks (Send/Process) |
| </li></ul> |
| <p>Here's an example of a Listener Rule: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"ADD Rule"</span> <span class="re0">WaitMessageTimeout</span>=<span class="st0">"10"</span> <span class="re0">Threads</span>=<span class="st0">"2"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span></pre></div> |
| <p>Its elements will be explained in detail in the following sections. |
| </p> |
| <a name="Rule_attributes"></a><h3> <span class="mw-headline"> Rule attributes </span></h3> |
| <p>In the Rule element you can specify the following attributes: |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Attribute |
| </th><th> Description |
| </th></tr> |
| <tr> |
| <td> Name |
| </td><td> (unique) name of the rule |
| </td></tr> |
| <tr> |
| <td> WaitMessageTimeout(Listener only) |
| </td><td> timeout (in seconds) for attempts to pull JMS message from queue, default value is 1 second. |
| </td></tr> |
| <tr> |
| <td> Threads (Listener only) |
| </td><td> startup number of threads which listen to the queue under this rule, default value is 2 threads. |
| </td></tr></table> |
| <a name="Source"></a><h3> <span class="mw-headline"> Source </span></h3> |
| <p>The Source element is only valid for the Listener configuration. It specifies a reference to the used broker (queue) connection. The given BrokerId must exist in the JMS Broker Connection configuration. |
| </p> |
| <a name="Condition"></a><h3> <span class="mw-headline"> Condition </span></h3> |
| <p>Conditions are elements of rules which are used by Router, Listener and RecordRecycler. A condition is a string which syntax is based on a subset of the SQL92 conditional expression syntax as ref'ed in the "Message Selector Syntax" section of [<a href="http://java.sun.com/products/jms/docs.html|JMS" class="external text" title="http://java.sun.com/products/jms/docs.html|JMS" rel="nofollow">specification</a>], and is applied against the JMS properties of an MQ entry. Within SMILA basically the two JMS properties, Operation and DataSourceID are supported, but generally every JMS property can be tested, including your own. |
| An empty condition element represents an condition that is always true. |
| </p><p>Router and Listener use the rules and their conditions differently: |
| </p><p><b> Router / RecordRecycler </b><br /> |
| The primary role of the router is to put JMS records into the desired MQ. The target MQ is defines in the Send-Task, which record is defined by the conditions of the rule. Other than the Listener the router will consult all rules sequentially from top bottom for any given record and execute the task for the first matching rule. It is therefore a good idea to define as a last rule with an empty condition and send records that did not match any rules this far into a special queue, such as the dead letter queue. |
| </p><p><b> Listener </b><br /> |
| The primary role of the Listener is to take records from the MQ and carry out the associated task. Hence the conditions are used as a "select clause" on all records of the queue. The task is then executed on one arbitrary record that matches the condition. Note that a listener creates at least one thread for each rule as defined by the Threads attribute at the rule. Hence the rules are executed concurrently. If therefore the conditions for diff. rules (and thus tasks) are not disjunctive, it is not deterministic which task is executed for which record. This is usually not desired so care must be taken when designing the rules. |
| </p><p><b> Examples </b><br /> |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Condition sample |
| </th><th> Description |
| </th></tr> |
| <tr> |
| <td> <empty> |
| </td><td> empty conditions (<Condition></Condition>) are always true |
| </td></tr> |
| <tr> |
| <td> Operation='ADD' |
| </td><td> if Operation equals to ADD |
| </td></tr> |
| <tr> |
| <td> DataSourceID LIKE 'web%' |
| </td><td> if dataSourceId begins with web |
| </td></tr> |
| <tr> |
| <td> NOT(DataSourceID LIKE 'web%' |
| </td><td> if dataSourceId does not start with web |
| </td></tr></table> |
| <a name="Tasks"></a><h3> <span class="mw-headline"> Tasks </span></h3> |
| <p>Tasks are specified within a Task element. |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Attribute |
| </th><th> Description |
| </th></tr> |
| <tr> |
| <td> BlackboardSync |
| </td><td> boolean value, determines if records should be synched(stored) via Blackboard |
| </td></tr></table> |
| <p>The BlackboardSync attribute has the following impacts: |
| </p> |
| <ul><li> true: a <i>persistent</i> blackboard will be used |
| </li><li> false: a <i>transient</i> blackboard will be used |
| </li></ul> |
| <p>A description of the differences between persistent and transient blackboards can be found <a href="Usage_of_Blackboard_Service.html#Creation_and_Lifecycle_of_Blackboard" title="SMILA/Documentation/Usage of Blackboard Service"> here</a>. |
| </p><p>Example: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><Task</span> <span class="re0">BlackboardSync</span>=<span class="st0">"false"</span><span class="re2">></span></span> // false -> we use a transient blackboard here |
| ... // <span class="sc3"><span class="re1"><Process<span class="re2">></span></span></span>/<span class="sc3"><span class="re1"><Send<span class="re2">></span></span></span> sub elements |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span></pre></div> |
| <p>There are two pre-defined tasks <Send> and <Process>. The QueueWorker components Router and Listener use these tasks to do their job. The Router usually uses the <Send> task to send records (which contain one crawled entry) to a queue. The Listener uses the <Process> task to take a Record from a queue and feed it into a BPEL workflow for processing this record. At the end of the process it may create a <Send> task again for putting the record result (back) into a queue. The Queue Worker provides all tasks and every component has the possibility to execute each of these tasks. |
| </p> |
| <a name="Process"></a><h4> <span class="mw-headline"> Process </span></h4> |
| <p>The Process task is only a plain tag with the attribute “Workflow”. This task takes a record from JMS broker and pushes it to a BPEL pipeline. |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Attribute |
| </th><th> Description |
| </th></tr> |
| <tr> |
| <td> Workflow |
| </td><td> the BPEL pipeline/workflow to execute. |
| </td></tr></table> |
| <p>Example: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddPipeline"</span><span class="re2">/></span></span></pre></div> |
| <a name="Send"></a><h4> <span class="mw-headline"> Send </span></h4> |
| <p>The Send task is used to send a Record to a JMS queue. The <Send> element may have the following attributes: |
| </p> |
| <table border="1px solid #CCC" cellpadding="5px" style="border:1px solid #CCC; border-collapse:collapse; font:small/1.5 Tahoma, Bitstream Vera Sans, Verdana, Helvetica, sans-serif;"> |
| <tr> |
| <th> Attribute |
| </th><th> Requirement |
| </th><th> Default value |
| </th><th> Description |
| </th></tr> |
| <tr> |
| <td> BrokerId |
| </td><td> required |
| </td><td> |
| </td><td> Id of the broker specified in JMS Broker Connection |
| </td></tr> |
| <tr> |
| <td> Queue |
| </td><td> required |
| </td><td> |
| </td><td> target JMS queue |
| </td></tr> |
| <tr> |
| <td> RecordFilter |
| </td><td> optional |
| </td><td> |
| </td><td> attribute to set the record filter used to downsize the record (see description below). If no RecordFilter attribute is set, the full record (including the attachments) will be send to the queue). |
| </td></tr> |
| <tr> |
| <td> PersistentDelivery |
| </td><td> optional |
| </td><td> true |
| </td><td> attribute correspondent to JMS delivery mode. If it is set to true (by default), message delivery will be persistent, otherwise not |
| </td></tr> |
| <tr> |
| <td> WithAttachments |
| </td><td> optional |
| </td><td> false |
| </td><td> used to get attachments from binary storage back into record after filtering them out. This might be needed in clustering scenarios where the MQ is used to transport the records to diff. nodes. By default this att. it's false (means that attachments are not attached to the Record) |
| </td></tr></table> |
| <p>The Send element may contain any number of SetProperty elements. Each SetProperty element will add a same named and valued JMS Property to the MQ Entry of the record being put into the queue. These are in addition to any predefined or dynamically set JMS properties (see <a href="QueueWorker.html#JMS_Message_Properties" title="">JMS Message Properties</a>). |
| </p><p>Example: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"filterName"</span> <span class="re0">PersistentDelivery</span>=<span class="st0">"false"</span> <span class="re0">WithAttachments</span>=<span class="st0">"true"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">Name</span>=<span class="st0">"doSomething"</span><span class="re2">></span></span>some value<span class="sc3"><span class="re1"></SetProperty<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">Name</span>=<span class="st0">"doSomethingElse"</span><span class="re2">></span></span>some other value<span class="sc3"><span class="re1"></SetProperty<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Send<span class="re2">></span></span></span></pre></div> |
| <a name="RecordFilter"></a><h5> <span class="mw-headline"> RecordFilter </span></h5> |
| <p>A Record Filter, as the name suggest, filters attributes of a record when sending it to a queue. |
| Often a record is complicated object with many attributes holding the metadata but also the binary content of the crawled data. As a consequence the serialized binary size of the record can be very large and would need to be handled (stored!) by the MQ. Further, most attributes are only needed once or twice during processing which makes it less desirable to always carrying them around in the MQ. |
| To lighten the record of such less needed data the records can be stripped of these, containing only the relevant information. However, keep in mind that stripped data must be fetched from the record or binary store when not held in memory which incurs some performance penalty. |
| A Record Filter will always cut the attachment off from a record. (If you dont use a RecordFilter (see Router/Listener Configuration), the attachments will stay) |
| </p><p>The filters are defined in an xml config file which is is located in <span style="font-family:monospace;">configuration/org.eclipse.smila.blackboard/RecordFilter.xml</span>, because the RecordFilter is invoked via <a href="Usage_of_Blackboard_Service.html" title="SMILA/Documentation/Usage of Blackboard Service">Blackboard service</a>. |
| </p><p>Each filter is identified by its name attribute and referenced from the Send Task's RecordFilter attribute. Every filter specifies <b>copy-rules</b> via the nested Attribute elements. I.e. these items are not removed from the record but actually put into the record of that message going to the destination. |
| </p><p>Default Configuration: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><RecordFilters</span> <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/RecordFilters.xsd"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"workflow-object"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"MimeType"</span> <span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"only-attributes"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"*"</span> <span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"no-filter"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"*"</span> <span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"nothing"</span> <span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></RecordFilters<span class="re2">></span></span></span></pre></div> |
| <p>Only the Send-task may use a RecordFilter and is selected by its name. The given attribute tags decide which attributes will be copied to the record pushed into the MQ. The name attribute, is a selector for the attribute and supports '*' as a wildcard. A RecordFilterNotFoundException will be thrown if no filter is chosen or a given filter is not found. |
| </p><p>Example: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><RecordFilters<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"only-attributes"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"*"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="coMULTI"><!-- all attributes like MimeType, Size and so on but without sub-annotations. --></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"no-filter"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"*"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="coMULTI"><!-- a full copy of the record will be created. --></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="coMULTI"><!-- a record only containing the id is copied. --></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"only-attribute1"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"attribute1"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="coMULTI"><!-- only a specified attributes is copied. --></span></span> |
| <span class="sc3"><span class="re1"><Filter</span> <span class="re0">name</span>=<span class="st0">"filter-single-and-datetime"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"single value"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Element</span> <span class="re0">name</span>=<span class="st0">"datetime value"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Filter<span class="re2">></span></span></span> |
| <span class="sc3"><span class="coMULTI"><!-- only single and datetime values are copied. --></span></span> |
| <span class="sc3"><span class="re1"></RecordFilters<span class="re2">></span></span></span></pre></div> |
| <p>There is a schema available at org.eclipse.smila.datamodel.tools/schemas/RecordFilters.xsd |
| </p> |
| <a name="Tips"></a><h2> <span class="mw-headline"> Tips </span></h2> |
| <a name="Persistent_Queues"></a><h3> <span class="mw-headline"> Persistent Queues </span></h3> |
| <p>Queue persistence can be configured in one of the following ways: |
| </p><p>1. <i>persistent</i> parameter in java.naming.provider.url (configuration/org.eclipse.smila.connectivity.queue.broker.main/jms.properties) |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-text">java.naming.provider.url=vm:(broker:(tcp://localhost:61616?jms.useAsyncSend=true)?persistent=true)?marshal=false</pre></div> |
| <p>This is a global setting which will be applied to all queues. |
| </p><p>2. <i>PersistentDelivery</i> attribute in Send Task (Router/Recycler/Listener configuration) |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">PersistentDelivery</span>=<span class="st0">"false"</span><span class="re2">/></span></span></pre></div> |
| <p>By setting <i>PersistentDelivery=true/false</i> you can define persistence handling on a single message granularity. |
| </p><p><b>Hint on ActiveMQ:</b> If persistence is disabled in the jms.properties, this cannot be overwritten by PersistentDelivery=true in the Send task. On the other hand, an enabled persistence in the jms.properties may be overwritten by setting PersistentDelivery=false. |
| </p> |
| <a name="Redelivery_and_Dead_Letter_Queue"></a><h3> <span class="mw-headline"> Redelivery and Dead Letter Queue </span></h3> |
| <p>You can define how a broker (connection) should handle failures during delivery/processing, e.g. when processing a record causes an exception in a BPEL pipeline. |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerConnectionsConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span><span class="re2">></span></span> |
| ... |
| <span class="sc3"><span class="re1"><URL<span class="re2">></span></span></span>vm://localhost?jms.redeliveryPolicy.maximumRedeliveries=2<span class="sc3"><span class="re1"></URL<span class="re2">></span></span></span> |
| ... |
| <span class="sc3"><span class="re1"></QueueWorkerConnectionsConfig<span class="re2">></span></span></span></pre></div> |
| <p><i>maximumRedeliveries=2</i> means, that in case of a failure the record will be delivered to the pipeline two more times. If that fails again, the record will be sent to the Dead Letter Queue (DLQ). |
| </p><p><b>Note</b> |
| this feature affects the whole message which may contain several records as of <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=281845" class="external text" title="https://bugs.eclipse.org/bugs/show_bug.cgi?id=281845" rel="nofollow">bug 281845</a>. To avoid having all records being put into the DLQ you need to manually add only the offending records into the DLQ in your Piplet or ProcessingService implementation. |
| </p><p><b>Hints on ActiveMQ:</b> |
| </p> |
| <ul><li> In ActiveMQ the DLQ will be created dynamically (if needed) and named "ActiveMQ.DLQ" per default. You can access this queue like any other queue, e.g. to define a special processing for failed records. |
| </li><li> As defined in ActiveMQ DLQ default settings (DeadLetterStrategy), non-persistent messages will not be sent to the DLQ. |
| </li></ul> |
| <a name="Automatic_reconnect"></a><h3> <span class="mw-headline"> Automatic reconnect </span></h3> |
| <p>When using ActiveMQ as JMS provider, you can define that connections should automatically be reconnected after being interrupted, e.g. because of a network failure. This can be done by using the <b>failover</b> protocol in the connection configuration: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerConnectionsConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span><span class="re2">></span></span> |
| ... |
| <span class="sc3"><span class="re1"><URL<span class="re2">></span></span></span>failover:(tcp://10.0.0.1:61617)<span class="sc3"><span class="re1"></URL<span class="re2">></span></span></span> |
| ... |
| <span class="sc3"><span class="re1"></QueueWorkerConnectionsConfig<span class="re2">></span></span></span></pre></div> |
| <p><b>Hint:</b> The failover protocol can also be used to provide alternative connections that may substitute the current connection if it fails. See ActiveMQ documentation for details. |
| </p> |
| <a name="Preprocessing_by_using_JMS_properties"></a><h3> <span class="mw-headline"> Preprocessing by using JMS properties </span></h3> |
| <p>As described before, you can set with the Send-Task your own JMS Properties on a record. This can be used in conjunction with cascaded DFPs, where a preprocessing has to be done, i.e. the result of one DFP is to be fed into another. Such a scenario is the case when diff. types of documents are crawled but processed basically the same after they have been converted to a common format. Here the records need to be marked with a JMS property according to their document type and be directed to the appropriate conversion DFP to do the conversion into the common format. The listener of these conversion DFPs then sends the JMS record back to the queue adding a new JMS property, with the name 'converted' and the value 'true'. Another DFP can select those records via its listener, whose conditions tests for this new JMS Property. |
| </p><p>As described before, you can set with the Send-Task your own JMS Properties on a record. This can be used in conjunction with cascaded DFPs, where the result of one DFP is to be fed into another. Such a scenario is the case when diff. types of documents are crawled but processed basically the same after they have been converted to a common format. Here the records need to be marked with a JMS property according to their document type and be directed to the appropriate conversion DFP to do the conversion into the common format. The listener of these conversion DFPs then sends the JMS record back to the queue adding a new JMS property, with the name 'converted' and the value 'true'. Another DFP can select those records via its listener, whose conditions tests for this new JMS Property. |
| </p><p>A complete use case of own parameters and different listeners in multiple steps: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerListenerConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- detects document mime type and send back to queue --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Detect MIME type Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD' and NOT(detected='true')<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"DetectMimeTypePipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">name</span>=<span class="st0">"detected"</span> <span class="re0">value</span>=<span class="st0">"true"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Send<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- converts documents and send back to queue --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Convert Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD' and detected='true' and NOT(converted='true')<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"ConvertPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">name</span>=<span class="st0">"detected"</span> <span class="re0">value</span>=<span class="st0">"true"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><SetProperty</span> <span class="re0">name</span>=<span class="st0">"converted"</span> <span class="re0">value</span>=<span class="st0">"true"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Send<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- add documents to index --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"ADD Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD' and converted='true'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerListenerConfig<span class="re2">></span></span></span></pre></div> |
| <a name="Preprocessing_by_using_several_queues"></a><h3> <span class="mw-headline"> Preprocessing by using several queues </span></h3> |
| <p>A similar preprocessing scenario like the one described before can be established without JMS parameters by using more than one queue: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><QueueWorkerListenerConfig</span> <span class="re0">xmlns</span>=<span class="st0">"http://www.eclipse.org/smila/queue"</span> |
| <span class="re0">xmlns:xsi</span>=<span class="st0">"http://www.w3.org/2001/XMLSchema-instance"</span> |
| <span class="re0">xsi:noNamespaceSchemaLocation</span>=<span class="st0">"schemas/QueueWorkerConfig.xsd"</span> |
| <span class="re2">></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- detects document mime type and sends to next queue --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Detect MIME type Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"DetectMimeTypePipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity-2"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- converts documents and sends to next queue --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"Convert Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity-2"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"ConvertPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Send</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity-3"</span> <span class="re0">RecordFilter</span>=<span class="st0">"nothing"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="coMULTI"><!-- add documents to index --></span></span> |
| <span class="sc3"><span class="re1"><Rule</span> <span class="re0">Name</span>=<span class="st0">"ADD Rule"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><Source</span> <span class="re0">BrokerId</span>=<span class="st0">"broker1"</span> <span class="re0">Queue</span>=<span class="st0">"SMILA.connectivity-3"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"><Condition<span class="re2">></span></span></span>Operation='ADD'<span class="sc3"><span class="re1"></Condition<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><Process</span> <span class="re0">Workflow</span>=<span class="st0">"AddPipeline"</span><span class="re2">/></span></span> |
| <span class="sc3"><span class="re1"></Task<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"></Rule<span class="re2">></span></span></span> |
| |
| <span class="sc3"><span class="re1"></QueueWorkerListenerConfig<span class="re2">></span></span></span></pre></div> |
| <p><br /> |
| </p> |
| <a name="Useful_Links"></a><h2> <span class="mw-headline"> Useful Links </span></h2> |
| <ol><li> [<a href="http://java.sun.com/products/jms/docs.html" class="external text" title="http://java.sun.com/products/jms/docs.html" rel="nofollow">JMS specification (JSR 914)</a>] |
| </li><li> [<a href="http://java.sun.com/j2ee/sdk_1.3/techdocs/api/javax/jms/Message.html" class="external text" title="http://java.sun.com/j2ee/sdk_1.3/techdocs/api/javax/jms/Message.html" rel="nofollow">JMS API SDK documentation</a>] |
| </li><li> [<a href="http://activemq.apache.org" class="external text" title="http://activemq.apache.org" rel="nofollow">Active MQ</a>] |
| </li><li> [<a href="http://activemq.apache.org/jndi-support.html" class="external text" title="http://activemq.apache.org/jndi-support.html" rel="nofollow">Active MQ – JNDI Support</a>] |
| </li><li> [<a href="http://activemq.apache.org/vm-transport-reference.html" class="external text" title="http://activemq.apache.org/vm-transport-reference.html" rel="nofollow">Active MQ – VM Transport Reference</a>] |
| </li></ol> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 160/1000000 |
| Post-expand include size: 2116/2097152 bytes |
| Template argument size: 449/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:15400-0!1!0!!en!2!edit=0 and timestamp 20110617022202 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="QueueWorker.html">http://wiki.eclipse.org/SMILA/Documentation/QueueWorker</a>"</div> |
| <div id="catlinks"><p class='catlinks'><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Category</a>: <span dir='ltr'><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></span></p></div> <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2011 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 15:04, 20 April 2011 by <a href="http://wiki.eclipse.org/User:Drazen.cindric.attensity.com" title="User:Drazen.cindric.attensity.com">Drazen Cindric</a>. Based on work by <a href="http://wiki.eclipse.org/index.php?title=User:Juergen.schumacher.attensity.com&action=edit" class="new" title="User:Juergen.schumacher.attensity.com">Juergen Schumacher</a>, <a href="http://wiki.eclipse.org/User:Tmenzel.brox.de" title="User:Tmenzel.brox.de">thomas menzel</a> and <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a> and <a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/QueueWorker&action=credits" title="SMILA/Documentation/QueueWorker">others</a>.</p> |
| <p id="footerviews">This page has been accessed 3,442 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.068 secs. --></body></html> |
