| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Specifications/Processing Message Resequencer/Smart Resequencer" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Specifications/Processing_Message_Resequencer/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Specifications/Processing Message Resequencer/Smart Resequencer - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "http://wiki.eclipse.org/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer"; |
| var wgTitle = "SMILA/Specifications/Processing Message Resequencer/Smart Resequencer"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "21924"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "173806"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <link rel="stylesheet" type="text/css" href="Smart_Resequencer.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Specifications_Processing_Message_Resequencer_Smart_Resequencer"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&oldid=173806">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="Smart_Resequencer.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Specifications/Processing Message Resequencer/Smart Resequencer"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Specifications/Processing Message Resequencer/Smart Resequencer</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../../SMILA.html" title="SMILA">SMILA</a> | <a href="../../Specifications.html" title="SMILA/Specifications">Specifications</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="Smart_Resequencer.html#column-one">navigation</a>, <a href="Smart_Resequencer.html#searchInput">search</a></div> <!-- start content --> |
| <p> |
| </p> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#Smart_Resequencer_.28SRS.29"><span class="tocnumber">1</span> <span class="toctext">Smart Resequencer (SRS)</span></a></li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#working_principle"><span class="tocnumber">2</span> <span class="toctext">working principle</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#non-concurrent_REGISTERING_setup"><span class="tocnumber">2.1</span> <span class="toctext">non-concurrent REGISTERING setup</span></a></li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#concurrent_REGISTERING_setup_with_call_in_pipelines"><span class="tocnumber">2.2</span> <span class="toctext">concurrent REGISTERING setup with call in pipelines</span></a></li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#concurrent_REGISTERING_setup_with_mirror_Queue"><span class="tocnumber">2.3</span> <span class="toctext">concurrent REGISTERING setup with mirror Queue</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#Basics_Impl._Ideas"><span class="tocnumber">3</span> <span class="toctext">Basics Impl. Ideas</span></a></li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#Meeting_Requirements"><span class="tocnumber">4</span> <span class="toctext">Meeting Requirements</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#split_records"><span class="tocnumber">4.1</span> <span class="toctext">split records</span></a></li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#.3E1_Processing_Targets"><span class="tocnumber">4.2</span> <span class="toctext">>1 Processing Targets</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="Smart_Resequencer.html#idea_1_-_SRS_ID_rules"><span class="tocnumber">4.2.1</span> <span class="toctext">idea 1 - SRS ID rules</span></a></li> |
| <li class="toclevel-3"><a href="Smart_Resequencer.html#idea_2_-_processing_steps_control_counter"><span class="tocnumber">4.2.2</span> <span class="toctext">idea 2 - processing steps control counter</span></a></li> |
| <li class="toclevel-3"><a href="Smart_Resequencer.html#Opinion"><span class="tocnumber">4.2.3</span> <span class="toctext">Opinion</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#clustering.2C_complex_processing_chain"><span class="tocnumber">4.3</span> <span class="toctext">clustering, complex processing chain</span></a></li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#single_point_of_failure"><span class="tocnumber">4.4</span> <span class="toctext">single point of failure</span></a></li> |
| <li class="toclevel-2"><a href="Smart_Resequencer.html#scalability_and_performance"><span class="tocnumber">4.5</span> <span class="toctext">scalability and performance</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#Cases_introduced_thru_this_solution"><span class="tocnumber">5</span> <span class="toctext">Cases introduced thru this solution</span></a></li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#PRO"><span class="tocnumber">6</span> <span class="toctext">PRO</span></a></li> |
| <li class="toclevel-1"><a href="Smart_Resequencer.html#CON"><span class="tocnumber">7</span> <span class="toctext">CON</span></a></li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Smart_Resequencer_.28SRS.29"></a><h3> <span class="mw-headline"> Smart Resequencer (SRS) </span></h3> |
| <p>Synopsis: The Smart Resequencer will only resequence the operation on a <b>per record</b> basis.<br /> |
| Rationale: in most cases records are independent of each other and so there is no need of ordering all records. |
| </p> |
| <a name="working_principle"></a><h3> <span class="mw-headline"> working principle </span></h3> |
| <ol><li> the SRS follows the registry and resequencer pattern |
| </li><li> the SRS defines boundaries between which, processing of multiple PRs for one given resource can be done concurrently and in any order |
| </li><li> it guarantees that when leaving its boundary the PRs are consolidated or at least in proper order |
| </li><li> the first thing that needs to happen for a PR, is to be REGISTERED with the SRS. by this the PR crosses the boundary entering the realm of the SRS. |
| </li><li> for the SRS to determine the correct order or consolidation of PRs, a PR <b>must</b> declare on REGISTRATION a sequence number (SN) that reflects the order of PRs as created by the agent/crawler |
| </li><li> before a PR is passed to a PT it must be RESEQUENCED by the SRS, which just checks if the PR's SN equals the SN of the latest REGISTED PR. if so, then it is added to the output if not, it is not added the output. a RESQUENCed PR crosses the 2nd boundery and thus leaves the realm of the SRS. |
| </li></ol> |
| <a name="non-concurrent_REGISTERING_setup"></a><h4> <span class="mw-headline"> non-concurrent REGISTERING setup </span></h4> |
| <ol><li> the router will deliver all PRs to Q1 where the SRS has a sole listener that registers all incoming messages |
| </li><li> the output of the SRS can then be processed as normal by multiple pipelines |
| </li><li> each pipeline that calls the PT must inject a call to the SRS with the RESQUENCE command <b>before</b> the call to the PT. |
| </li></ol> |
| <p>a down side to this setup is that REGISTERING adds execution time to the critical path for all records and can only be proced by one thread. |
| question: is there a setup that is better w/o introducing an error? |
| </p> |
| <a name="concurrent_REGISTERING_setup_with_call_in_pipelines"></a><h4> <span class="mw-headline"> concurrent REGISTERING setup with call in pipelines </span></h4> |
| <p>instead of creating a sole pipeline for the SRS, this idea proposes to add the REGISTER call to each pipeline at the very beginning. |
| by this, concurrent processing of all PRs is fostered from the start. |
| </p><p>this will work safely: |
| </p> |
| <ol><li> listener L1 and L2 work concurrently on the same item but with diff. SNs. L1 on PR1.SN1 and L2 on PR2.SN2. |
| </li><li> the critical case is when SRS doesnt know of the more recent PR2 when being asked to RESEQUENCE PR1, like so: |
| <ol><li> If L1 calls SRS with RESEQUENCE before L2 gets to call REGISTER, SRS doesnt know that an SN2 is out there and lets PR with SN1 pass thru. |
| </li><li> this is not an error b/c the order of processing is maintained. it has a drawback however, which is that SRS is unable to supress PR1. |
| </li></ol> |
| </li><li> i cannot think of another case where this setup leads to an aerror , b/c all PRs are first registered before any further processing may take place this setup works in all cases IMO. (please prove me wrong) |
| </li></ol> |
| <p><b>pro:</b> reduced overhead due to one less Q and pipeline |
| </p><p><b>con:</b> it might be easy to forget to call the SRS in each pipeline. |
| </p><p><br /> |
| </p> |
| <a name="concurrent_REGISTERING_setup_with_mirror_Queue"></a><h4> <span class="mw-headline"> concurrent REGISTERING setup with mirror Queue </span></h4> |
| <p>in this setup a copy of the PR with its SN is sent to an additional Q2 in parallel to Q1 (two send tasks in router). SRS is the only listener on Q2 and REGISTERS all the PRs. |
| </p><p>this will introduce an error in case that a PR is RESEQUENCed while for the same item there is a new PR waiting in Q2. |
| </p><p>the problem here is generally that REGISTERing happens asynchronously to processing and hence cannot be safe. |
| </p><p><b>fix:</b> the error introcuced by this setup can be fixed by demanding that the SRS is first to process all PRs on Q2 before RESEQUENCing PRs. however, this may cause PRs only to be added to the index as long as the agent produces PRs. in that case it wont be better than the non-concurent setup. |
| </p> |
| <a name="Basics_Impl._Ideas"></a><h3> <span class="mw-headline"> Basics Impl. Ideas </span></h3> |
| <ul><li> implemented as ProcessingService |
| </li><li> records are sent to it with the command/process mode REGISTER and SEQUENCE |
| </li><li> SN and process mode are given as annotation on the record, called the Config Annotation (CA). this is the same way as with the lucen service. (i first wanted to do this as JMS props but they are not accessible in a ProcessingService) |
| </li><li> map may be in memory or a persisting solution may be implemented/chosen. <br /> IMO the amount of records held im momory should be relativly small, only to the amount of what is in the processing chain. (hm, that can be a lot, since connectivity is not pausing crawlers and agents (yet) if there is much in the MQ) |
| </li></ul> |
| <p><br /> |
| </p> |
| <a name="Meeting_Requirements"></a><h3> <span class="mw-headline"> Meeting Requirements </span></h3> |
| <p>The general ones should be sufficeiently clear from the functional description of the SRS. here come now the further ones: |
| </p> |
| <a name="split_records"></a><h4> <span class="mw-headline"> split records </span></h4> |
| <p>compound and aggregation are handled the same way, like so: |
| </p><p>the processing step splitting the record is responsible for the the following: |
| </p> |
| <ul><li> all descendants inherit the SN from their root |
| </li><li> if internal ordering: <br />order of PRs for descendants is noted in their respective ConfigAnnotation (e.g. a link to the ID of the preceding or succeeding resource or such) |
| </li><li> register the split records with the SRS |
| </li><li> possibly deregister the root and/or intermediate PRs if these are not processed further |
| </li></ul> |
| <p>the SRS will |
| </p> |
| <ul><li> collect all PRs belonging to the tree of split PRs until it is complete |
| <ul><li> missing PRs: |
| <ul><li> timeout |
| </li><li> config on how to continue with non-complete trees: {all or nothing, sequence incomplete} |
| </li></ul> |
| </li></ul> |
| </li></ul> |
| <a name=".3E1_Processing_Targets"></a><h4> <span class="mw-headline"> >1 Processing Targets </span></h4> |
| <p>this can be supported in diff. ways. both have in common: |
| </p> |
| <ol><li> sending the PR to any of the PTs is done thru the SRS by calling it with the RESEQUENCE command (this is just a generalization of the basic concept and repeated here for clarity) |
| </li><li> the SRS needs to know how many (potential) PTs there are for a resource (determine by the ID) and when processing really has finished for a given ID. |
| </li><li> each RESEQUENCE and UNREGISTER command will reduce the count, when it reaches 0 all PRs have reached their PT and the ID can be removed from the map. |
| </li></ol> |
| <a name="idea_1_-_SRS_ID_rules"></a><h5> <span class="mw-headline"> idea 1 - SRS ID rules </span></h5> |
| <ul><li> the config of the SRS contains rules or conditions that determine the count. |
| </li><li> it starts with that count wich is computed on REGISTRATION. |
| </li></ul> |
| <a name="idea_2_-_processing_steps_control_counter"></a><h5> <span class="mw-headline"> idea 2 - processing steps control counter </span></h5> |
| <ul><li> processing steps take care of in- and decerementing the counter in the normal processing chain by using the REGISTER and UNREGISTER commands to reflect additional or obsolte PTs |
| </li></ul> |
| <a name="Opinion"></a><h5> <span class="mw-headline"> Opinion </span></h5> |
| <p>i like idea 2 better b/c it puts the config of the SRS in the same place that also controls the flow of PRs anyhow. it is just a matter of including an SRS call with the respective command.<br /> |
| in contrast, idea 1 would mean that we have to: |
| </p> |
| <ul><li> duplicate the processing chain logic in some other place |
| </li><li> implement a rule/condition engine and config. |
| </li></ul> |
| <a name="clustering.2C_complex_processing_chain"></a><h4> <span class="mw-headline"> clustering, complex processing chain </span></h4> |
| <ul><li> complex processing chains are possible as already described in other places. the SRS just needs to be placed in front of the PT and called in the flow of things |
| </li><li> resequencing in a cluster scenario works OOB just the setup/config changes. |
| <ul><li> SRS is run on several nodes and shares a custer capable map OR |
| </li><li> if the router sends PRs for the same item always to the same processing node, then the SRS can be local to the processing nodes and setup as normal OR |
| </li><li> SRS runs on just one node. then |
| <ul><li> all messages from the router need to be send to the SRS node first for REGISTERING<br />it makes sense to have th SRS and router on the same node to avoid chnaging nodes for the first step. |
| </li><li> all processing nodes dont call he PT directly in their pipeline but send their result to SRS node |
| </li><li> the SRS RESEQUENCE pipeline will call the PT. |
| </li></ul> |
| </li></ul> |
| </li></ul> |
| <p><b>Note</b><br /> |
| i think the SRS will even work if the assumption that each PT has only one instance and node that solely accesses the PT holds not true. |
| a setup like this will then segment all PRs by some scheme that depends on the ID and then the SRS has only to resequence the one segement and thus: all is well. |
| </p> |
| <a name="single_point_of_failure"></a><h4> <span class="mw-headline"> single point of failure </span></h4> |
| <p>hm. this is a tough one as the single'nes is inherent. i have no clue yet, how to solve this, other than to use a fail-over solution. |
| i guess, just as the PT itself, it needs to be monitored closely to detect malfunctions. |
| </p> |
| <a name="scalability_and_performance"></a><h4> <span class="mw-headline"> scalability and performance </span></h4> |
| <p>there is some performance degradation to be expected because |
| </p> |
| <ol><li> SRS increases the number of threads |
| </li><li> SRS is inserted at least 2 times into the processing flow, namely at the beginning and end. |
| <ol><li> when it registers an item |
| </li><li> when it resquences it |
| </li></ol> |
| </li></ol> |
| <p>the internal workings of these steps are fairly simple and should not take much time compared to the rest of the processing, albeit in a highly concurrent scenario the synchronization will take its toll. |
| </p> |
| <a name="Cases_introduced_thru_this_solution"></a><h3> <span class="mw-headline"> Cases introduced thru this solution </span></h3> |
| <p>this section lists cases and problems that need to be covered that are introduced thru the solution itself. |
| some of the items listed here will also apply to the FSR!! |
| </p> |
| <ul><li> handling of unregistered records<br />what happens when SEQUENCING a PR that is already @ count 0 /not existing. |
| </li><li> what to do with recods that miss needed config data? <br /> handling depends on the process mode: |
| <ul><li> SEQUENCE: error as default , but outcome could be config'able such as : DLQ, any other Q |
| </li><li> REGISTER: error |
| </li></ul> |
| </li><li> overflow of the SN<br />a reset signal must be sent to SRS |
| </li></ul> |
| <p><br /> |
| </p> |
| <a name="PRO"></a><h3> <span class="mw-headline"> PRO </span></h3> |
| <ul><li> smarter ;) than FRS |
| </li><li> no change to APIs are needed, implementation of agents/crawlers (controller) needs to add the SN as an annotation to each record. which can be turned on or off via config. |
| </li><li> unobstrutive, SRS can be used or not. |
| </li></ul> |
| <a name="CON"></a><h3> <span class="mw-headline"> CON </span></h3> |
| <ul><li> see also almost all CONS @ FRS |
| </li><li> oscillating resources (that constantly change) will never make it into the index. |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 28/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:21924-0!1!0!!en!2!edit=0 and timestamp 20130416061101 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="Smart_Resequencer.html">http://wiki.eclipse.org/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer</a>"</div> |
| <div id="catlinks"><p class='catlinks'><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Category</a>: <span dir='ltr'><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></span></p></div> <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2013 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 20:43, 7 October 2009 by <a href="http://wiki.eclipse.org/User:Tmenzel.brox.de" title="User:Tmenzel.brox.de">thomas menzel</a>. </p> |
| <p id="footerviews">This page has been accessed 1,743 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.056 secs. --></body></html> |
