Google Git
Sign in
eclipse / www.eclipse.org / smila / aae8130a6ca7e93ae19733f9f9d3a2b06e59f539 / . / documentation / 1.3 / wiki.eclipse.org / SMILA / Specifications / Processing_Message_Resequencer / Smart_Resequencer.html
blob: f8aef1d19b2c69b35ef5c74684c5edfd2f862b99 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir="ltr" class="client-nojs">
<head>
<meta charset="UTF-8" />
<title>SMILA/Specifications/Processing Message Resequencer/Smart Resequencer - Eclipsepedia</title>
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />
<meta name="generator" content="MediaWiki 1.23.2" />
<link rel="shortcut icon" href="http://wiki.eclipse.org/eclipse.org-common/themes/solstice/public/images/favicon.ico" />
<link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (en)" />
<link rel="EditURI" type="application/rsd+xml" href="http://wiki.eclipse.org/api.php?action=rsd" />
<link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom feed" href="http://wiki.eclipse.org/index.php?title=Special:RecentChanges&amp;feed=atom" />
<link rel="stylesheet" href="http://wiki.eclipse.org/load.php?debug=false&amp;lang=en&amp;modules=mediawiki.legacy.commonPrint%2Cshared%7Cmediawiki.ui.button&amp;only=styles&amp;skin=solstice&amp;*" />
<link rel="stylesheet" href="http://wiki.eclipse.org/skins/solstice/public/stylesheets/styles.min.css?303" media="screen, print" /><meta name="ResourceLoaderDynamicStyles" content="" />
<style>a:lang(ar),a:lang(kk-arab),a:lang(mzn),a:lang(ps),a:lang(ur){text-decoration:none}
/* cache key: my_wiki:resourceloader:filter:minify-css:7:14ece53a42aa314864e5fd8c57f0d98f */</style>
<script src="http://wiki.eclipse.org/load.php?debug=false&amp;lang=en&amp;modules=startup&amp;only=scripts&amp;skin=solstice&amp;*"></script>
<script>if(window.mw){
mw.config.set({"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer","wgTitle":"SMILA/Specifications/Processing Message Resequencer/Smart Resequencer","wgCurRevisionId":173806,"wgRevisionId":173806,"wgArticleId":21924,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["SMILA"],"wgBreakFrames":false,"wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgMonthNamesShort":["","Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],"wgRelevantPageName":"SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer","wgIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgWikiEditorEnabledModules":{"toolbar":false,"dialogs":false,"hidesig":true,"preview":false,"previewDialog":false,"publish":false},"wgCategoryTreePageCategoryOptions":"{\"mode\":0,\"hideprefix\":20,\"showcount\":true,\"namespaces\":false}"});
}</script><script>if(window.mw){
mw.loader.implement("user.options",function($,jQuery){mw.user.options.set({"ccmeonemails":0,"cols":80,"date":"default","diffonly":0,"disablemail":0,"editfont":"default","editondblclick":0,"editsectiononrightclick":0,"enotifminoredits":0,"enotifrevealaddr":0,"enotifusertalkpages":1,"enotifwatchlistpages":1,"extendwatchlist":0,"fancysig":0,"forceeditsummary":0,"gender":"unknown","hideminor":0,"hidepatrolled":0,"imagesize":2,"math":1,"minordefault":0,"newpageshidepatrolled":0,"nickname":"","norollbackdiff":0,"numberheadings":0,"previewonfirst":0,"previewontop":1,"rcdays":7,"rclimit":50,"rows":25,"showhiddencats":0,"shownumberswatching":1,"showtoolbar":1,"skin":"solstice","stubthreshold":0,"thumbsize":2,"underline":2,"uselivepreview":0,"usenewrc":0,"watchcreations":1,"watchdefault":1,"watchdeletion":0,"watchlistdays":3,"watchlisthideanons":0,"watchlisthidebots":0,"watchlisthideliu":0,"watchlisthideminor":0,"watchlisthideown":0,"watchlisthidepatrolled":0,"watchmoves":0,"wllimit":250,
"useeditwarning":1,"prefershttps":1,"language":"en","variant-gan":"gan","variant-iu":"iu","variant-kk":"kk","variant-ku":"ku","variant-shi":"shi","variant-sr":"sr","variant-tg":"tg","variant-uz":"uz","variant-zh":"zh","searchNs0":true,"searchNs1":false,"searchNs2":false,"searchNs3":false,"searchNs4":false,"searchNs5":false,"searchNs6":false,"searchNs7":false,"searchNs8":false,"searchNs9":false,"searchNs10":false,"searchNs11":false,"searchNs12":false,"searchNs13":false,"searchNs14":false,"searchNs15":false,"variant":"en"});},{},{});mw.loader.implement("user.tokens",function($,jQuery){mw.user.tokens.set({"editToken":"+\\","patrolToken":false,"watchToken":false});},{},{});
/* cache key: my_wiki:resourceloader:filter:minify-js:7:70d74423d3fc1e1c18fa9a1ff645a84a */
}</script>
<script>if(window.mw){
mw.loader.load(["mediawiki.page.startup","mediawiki.legacy.wikibits","mediawiki.legacy.ajax"]);
}</script>
<meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body class="mediawiki ltr sitedir-ltr ns-0 ns-subject page-SMILA_Specifications_Processing_Message_Resequencer_Smart_Resequencer skin-solstice action-view" id="solstice">
<a class="sr-only" href="Smart_Resequencer.html#content">Skip to main content</a>
<div class="thin-header">
<header role="banner" class="hidden-print noprint">
<div class="container-fluid">
<div id="row-logo-search">
<div id="header-left">
<div class="row">
<div class="hidden-xs col-sm-6 logo-container">
<a href="https://www.eclipse.org/" ><img class="logo-eclipse-default" src="http://wiki.eclipse.org/skins/solstice/public/images/logo/eclipse-800x188.png" alt="Eclipsepedia"></a>
</div>
<div class="navbar col-sm-18 yamm" id="main-menu">
<div class="navbar-collapse collapse" id="navbar-collapse-1">
<ul class="nav navbar-nav">
<li><a target="_self" href="https://eclipse.org/downloads/">Download</a></li>
<li><a target="_self" href="https://eclipse.org/users/">Getting Started </a></li>
<li><a target="_self" href="https://eclipse.org/membership/">Members</a></li>
<li><a target="_self" href="https://eclipse.org/projects/">Projects</a></li>
<li class="dropdown visible-xs"><a class="dropdown-toggle" data-toggle="dropdown" href="Smart_Resequencer.html#">Community <b class="caret"></b></a><ul class="dropdown-menu"><li><a href="http://marketplace.eclipse.org">Marketplace</a></li><li><a href="http://events.eclipse.org">Events</a></li><li><a href="http://www.planeteclipse.org/">Planet Eclipse</a></li><li><a href="https://eclipse.org/community/eclipse_newsletter/">Newsletter</a></li><li><a href="https://www.youtube.com/user/EclipseFdn">Videos</a></li></ul></li><li class="dropdown visible-xs"><a class="dropdown-toggle" data-toggle="dropdown" href="Smart_Resequencer.html#">Participate <b class="caret"></b></a><ul class="dropdown-menu"><li><a href="https://bugs.eclipse.org/bugs/">Report a Bug</a></li><li><a href="https://eclipse.org/forums/">Forums</a></li><li><a href="https://eclipse.org/mail/">Mailing Lists</a></li><li><a href="https://wiki.eclipse.org/">Wiki</a></li><li><a href="https://wiki.eclipse.org/IRC">IRC</a></li><li><a href="https://eclipse.org/contribute/">How to Contribute</a></li></ul></li><li class="dropdown visible-xs"><a class="dropdown-toggle" data-toggle="dropdown" href="Smart_Resequencer.html#">Working Groups <b class="caret"></b></a><ul class="dropdown-menu"><li><a href="http://wiki.eclipse.org/Auto_IWG">Automotive</a></li><li><a href="http://iot.eclipse.org">Internet of Things</a></li><li><a href="http://locationtech.org">LocationTech</a></li><li><a href="http://lts.eclipse.org">Long-Term Support</a></li><li><a href="http://polarsys.org">PolarSys</a></li><li><a href="http://science.eclipse.org">Science</a></li><li><a href="http://openmdm.org">OpenMDM</a></li></ul></li><!-- More -->
<li class="dropdown hidden-xs"><a class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
<ul class="dropdown-menu">
<li>
<!-- Content container to add padding -->
<div class="yamm-content">
<div class="row">
<ul class="col-sm-8 list-unstyled"><li><p><strong>Community</strong></p></li><li><a href="http://marketplace.eclipse.org">Marketplace</a></li><li><a href="http://events.eclipse.org">Events</a></li><li><a href="http://www.planeteclipse.org/">Planet Eclipse</a></li><li><a href="https://eclipse.org/community/eclipse_newsletter/">Newsletter</a></li><li><a href="https://www.youtube.com/user/EclipseFdn">Videos</a></li></ul><ul class="col-sm-8 list-unstyled"><li><p><strong>Participate</strong></p></li><li><a href="https://bugs.eclipse.org/bugs/">Report a Bug</a></li><li><a href="https://eclipse.org/forums/">Forums</a></li><li><a href="https://eclipse.org/mail/">Mailing Lists</a></li><li><a href="https://wiki.eclipse.org/">Wiki</a></li><li><a href="https://wiki.eclipse.org/IRC">IRC</a></li><li><a href="https://eclipse.org/contribute/">How to Contribute</a></li></ul><ul class="col-sm-8 list-unstyled"><li><p><strong>Working Groups</strong></p></li><li><a href="http://wiki.eclipse.org/Auto_IWG">Automotive</a></li><li><a href="http://iot.eclipse.org">Internet of Things</a></li><li><a href="http://locationtech.org">LocationTech</a></li><li><a href="http://lts.eclipse.org">Long-Term Support</a></li><li><a href="http://polarsys.org">PolarSys</a></li><li><a href="http://science.eclipse.org">Science</a></li><li><a href="http://openmdm.org">OpenMDM</a></li></ul> </div>
</div>
</li>
</ul>
</li>
</ul>
</div>
<div class="navbar-header">
<button data-target="#navbar-collapse-1" data-toggle="collapse" class="navbar-toggle" type="button">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a href="https://www.eclipse.org/" class="visible-xs navbar-brand"><img class="logo-eclipse-default" src="http://wiki.eclipse.org/skins/solstice/public/images/logo/eclipse-800x188.png" alt="Eclipsepedia" width="174"></a>
</div>
</div>
</div>
</div>
</div>
</div>
</header>
<section class="defaut-breadcrumbs hidden-print noprint hidden-print clearfix" id="breadcrumb">
<div>
<ol class="breadcrumb">
<li><a href="https://www.eclipse.org/">Home</a></li>
<li><a href="http://wiki.eclipse.org/Main_Page">Eclipse Wiki</a></li>
<li class="active">SMILA/Specifications/Processing Message Resequencer/Smart Resequencer</li></ol>
</div>
</section>
</div>
<div class="toolbar-menu breadcrumbs-offset noprint hidden-print margin-bottom-0 clearfix">
<div class="col-md-24">
<ol class="breadcrumb" role="navigation">
<li id="pt-login">
<a href="http://wiki.eclipse.org/index.php?title=Special:UserLogin&amp;returnto=SMILA%2FSpecifications%2FProcessing+Message+Resequencer%2FSmart+Resequencer">
<i class="fa fa-sign-in fa-fw orange"></i> Log in </a>
</li>
</ul>
</div>
</div>
<main role="main" class="background-grey">
<div class="container-full padding-top-25">
<!-- content -->
<section id="content" class="mw-body container-full clearfix 0">
<div id="mw-js-message" style="display:none;"></div>
<!-- bodyContent -->
<div id="bodyContent">
<!-- jumpto -->
<div id="jump-to-nav" class="mw-jump">
Jump to: <a href="Smart_Resequencer.html#mw-head">navigation</a>,
<a href="Smart_Resequencer.html#p-search">search</a>
</div>
<!-- /jumpto -->
<!-- leftcol -->
<aside class="col-md-4 noprint hidden-print" id="leftcol">
<form class="input-group" role="form" id="form-eclipse-search" action="http://wiki.eclipse.org/index.php" id="searchform">
<input id="searchInput" class="search-query form-control" type="search" accesskey="f" title="Special:Search" placeholder="Search" name="search" value="">
<span class="input-group-btn">
<button value="search" id="mw-searchButton" type="submit" class="btn btn-default" title="Search the pages for this text" name="fulltext">
<i class="fa fa-search"></i>
</button>
</span>
</form>
<select class="form-control margin-top-10 margin-bottom-10 visible-xs visible-sm" onchange="this.options[this.selectedIndex].value && (window.location = this.options[this.selectedIndex].value);"><option class="fw-700 "><span class="fw-700">---Navigation---</span></option><option value="/Main_Page">Main Page</option><option value="/Eclipsepedia:Community_portal">Community portal</option><option value="/Eclipsepedia:Current_events">Current events</option><option value="/Special:RecentChanges">Recent changes</option><option value="/Special:Random">Random page</option><option value="https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Contents">Help</option></select><ul class="ul-left-nav fa-ul hidden-print leftnav hidden-xs hidden-sm"><li class="separator"><span class="separator">Navigation</span></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Main_Page" id="n-mainpage" title="Visit the main page [z]" accesskey="z">Main Page</a></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Eclipsepedia:Community_portal" id="n-portal" title="About the project, what you can do, where to find things">Community portal</a></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Eclipsepedia:Current_events" id="n-currentevents" title="Find background information on current events">Current events</a></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Special:RecentChanges" id="n-recentchanges" title="A list of recent changes in the wiki [r]" accesskey="r">Recent changes</a></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Special:Random" id="n-randompage" title="Load a random page [x]" accesskey="x">Random page</a></li> <li class=""><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Contents" id="n-help" title="The place to find out">Help</a></li></ul> <select class="form-control margin-top-10 margin-bottom-10 visible-xs visible-sm" onchange="this.options[this.selectedIndex].value && (window.location = this.options[this.selectedIndex].value);"><option class="fw-700 "><span class="fw-700">---Toolbox---</span></option><option value="/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;action=info">Page information</option><option value="/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;oldid=173806">Permanent link</option><option value="/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;printable=yes">Printable version</option><option value="/Special:SpecialPages">Special pages</option><option value="/Special:RecentChangesLinked/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer">Related changes</option><option value="/Special:WhatLinksHere/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer">What links here</option></select><ul class="ul-left-nav fa-ul hidden-print leftnav hidden-xs hidden-sm"><li class="separator"><span class="separator">Toolbox</span></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;action=info" id="t-info">Page information</a></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;oldid=173806" id="t-permalink" title="Permanent link to this revision of the page">Permanent link</a></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;printable=yes" id="t-print" rel="alternate" title="Printable version of this page [p]" accesskey="p">Printable version</a></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Special:SpecialPages" id="t-specialpages" title="A list of all special pages [q]" accesskey="q">Special pages</a></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Special:RecentChangesLinked/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer" id="t-recentchangeslinked" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li><i class="fa fa-angle-double-right orange fa-fw"></i> <a href="http://wiki.eclipse.org/Special:WhatLinksHere/SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer" id="t-whatlinkshere" title="A list of all wiki pages that link here [j]" accesskey="j">What links here</a></li></ul> </aside>
<!-- /leftcol -->
<!-- mainContent -->
<div id="mainContent" class="col-md-20">
<ul class="nav nav-tabs noprint hidden-print" role="tablist">
<li id="ca-nstab-main" class="active"><a href="Smart_Resequencer.html" title="View the content page [c]" accesskey="c" tabindex="-1">Page</a></li>
<li id="ca-talk" class="new"><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;action=edit&amp;redlink=1" title="Discussion about the content page [t]" accesskey="t" tabindex="-1">Discussion</a></li>
<li id="ca-viewsource"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;action=edit" title="This page is protected.&#10;You can view its source [e]" accesskey="e" tabindex="-1">View source</a></li>
<li id="ca-history" class="collapsible"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;action=history" title="Past revisions of this page [h]" accesskey="h" tabindex="-1">History</a></li>
</ul> <div class="tab-content background-white">
<div id="tab-pane-main-page-content" class="tab-pane active">
<h1 id="firstHeading" class="firstHeading page-header">
<span dir="auto">SMILA/Specifications/Processing Message Resequencer/Smart Resequencer</span>
</h1>
<div id="main-page-content">
<!-- subtitle -->
<div id="contentSub" class="alert alert-small alert-warning"><span class="subpages">&lt; <a href="../../../SMILA.html" title="SMILA">SMILA</a>&lrm; | <a href="../../Specifications.html" title="SMILA/Specifications">Specifications</a></span></div>
<!-- /subtitle -->
<div id="mw-content-text" lang="en" dir="ltr" class="mw-content-ltr"><p><br />
</p>
<div id="toc" class="toc"><div id="toctitle"><h2>Contents</h2></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="Smart_Resequencer.html#Smart_Resequencer_.28SRS.29"><span class="tocnumber">1</span> <span class="toctext">Smart Resequencer (SRS)</span></a></li>
<li class="toclevel-1 tocsection-2"><a href="Smart_Resequencer.html#working_principle"><span class="tocnumber">2</span> <span class="toctext">working principle</span></a>
<ul>
<li class="toclevel-2 tocsection-3"><a href="Smart_Resequencer.html#non-concurrent_REGISTERING_setup"><span class="tocnumber">2.1</span> <span class="toctext">non-concurrent REGISTERING setup</span></a></li>
<li class="toclevel-2 tocsection-4"><a href="Smart_Resequencer.html#concurrent_REGISTERING_setup_with_call_in_pipelines"><span class="tocnumber">2.2</span> <span class="toctext">concurrent REGISTERING setup with call in pipelines</span></a></li>
<li class="toclevel-2 tocsection-5"><a href="Smart_Resequencer.html#concurrent_REGISTERING_setup_with_mirror_Queue"><span class="tocnumber">2.3</span> <span class="toctext">concurrent REGISTERING setup with mirror Queue</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-6"><a href="Smart_Resequencer.html#Basics_Impl._Ideas"><span class="tocnumber">3</span> <span class="toctext">Basics Impl. Ideas</span></a></li>
<li class="toclevel-1 tocsection-7"><a href="Smart_Resequencer.html#Meeting_Requirements"><span class="tocnumber">4</span> <span class="toctext">Meeting Requirements</span></a>
<ul>
<li class="toclevel-2 tocsection-8"><a href="Smart_Resequencer.html#split_records"><span class="tocnumber">4.1</span> <span class="toctext">split records</span></a></li>
<li class="toclevel-2 tocsection-9"><a href="Smart_Resequencer.html#.3E1_Processing_Targets"><span class="tocnumber">4.2</span> <span class="toctext">&gt;1 Processing Targets</span></a>
<ul>
<li class="toclevel-3 tocsection-10"><a href="Smart_Resequencer.html#idea_1_-_SRS_ID_rules"><span class="tocnumber">4.2.1</span> <span class="toctext">idea 1 - SRS ID rules</span></a></li>
<li class="toclevel-3 tocsection-11"><a href="Smart_Resequencer.html#idea_2_-_processing_steps_control_counter"><span class="tocnumber">4.2.2</span> <span class="toctext">idea 2 - processing steps control counter</span></a></li>
<li class="toclevel-3 tocsection-12"><a href="Smart_Resequencer.html#Opinion"><span class="tocnumber">4.2.3</span> <span class="toctext">Opinion</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-13"><a href="Smart_Resequencer.html#clustering.2C_complex_processing_chain"><span class="tocnumber">4.3</span> <span class="toctext">clustering, complex processing chain</span></a></li>
<li class="toclevel-2 tocsection-14"><a href="Smart_Resequencer.html#single_point_of_failure"><span class="tocnumber">4.4</span> <span class="toctext">single point of failure</span></a></li>
<li class="toclevel-2 tocsection-15"><a href="Smart_Resequencer.html#scalability_and_performance"><span class="tocnumber">4.5</span> <span class="toctext">scalability and performance</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-16"><a href="Smart_Resequencer.html#Cases_introduced_thru_this_solution"><span class="tocnumber">5</span> <span class="toctext">Cases introduced thru this solution</span></a></li>
<li class="toclevel-1 tocsection-17"><a href="Smart_Resequencer.html#PRO"><span class="tocnumber">6</span> <span class="toctext">PRO</span></a></li>
<li class="toclevel-1 tocsection-18"><a href="Smart_Resequencer.html#CON"><span class="tocnumber">7</span> <span class="toctext">CON</span></a></li>
</ul>
</div>
<h3><span class="mw-headline" id="Smart_Resequencer_.28SRS.29">Smart Resequencer (SRS)</span></h3>
<p>Synopsis: The Smart Resequencer will only resequence the operation on a <b>per record</b> basis.<br />
Rationale: in most cases records are independent of each other and so there is no need of ordering all records.
</p>
<h3><span class="mw-headline" id="working_principle">working principle</span></h3>
<ol>
<li> the SRS follows the registry and resequencer pattern
</li>
<li> the SRS defines boundaries between which, processing of multiple PRs for one given resource can be done concurrently and in any order
</li>
<li> it guarantees that when leaving its boundary the PRs are consolidated or at least in proper order
</li>
<li> the first thing that needs to happen for a PR, is to be REGISTERED with the SRS. by this the PR crosses the boundary entering the realm of the SRS.
</li>
<li> for the SRS to determine the correct order or consolidation of PRs, a PR <b>must</b> declare on REGISTRATION a sequence number (SN) that reflects the order of PRs as created by the agent/crawler
</li>
<li> before a PR is passed to a PT it must be RESEQUENCED by the SRS, which just checks if the PR's SN equals the SN of the latest REGISTED PR. if so, then it is added to the output if not, it is not added the output. a RESQUENCed PR crosses the 2nd boundery and thus leaves the realm of the SRS.
</li>
</ol>
<h4><span class="mw-headline" id="non-concurrent_REGISTERING_setup">non-concurrent REGISTERING setup</span></h4>
<ol>
<li> the router will deliver all PRs to Q1 where the SRS has a sole listener that registers all incoming messages
</li>
<li> the output of the SRS can then be processed as normal by multiple pipelines
</li>
<li> each pipeline that calls the PT must inject a call to the SRS with the RESQUENCE command <b>before</b> the call to the PT.
</li>
</ol>
<p>a down side to this setup is that REGISTERING adds execution time to the critical path for all records and can only be proced by one thread.
question: is there a setup that is better w/o introducing an error?
</p>
<h4><span class="mw-headline" id="concurrent_REGISTERING_setup_with_call_in_pipelines">concurrent REGISTERING setup with call in pipelines</span></h4>
<p>instead of creating a sole pipeline for the SRS, this idea proposes to add the REGISTER call to each pipeline at the very beginning.
by this, concurrent processing of all PRs is fostered from the start.
</p><p>this will work safely:
</p>
<ol>
<li> listener L1 and L2 work concurrently on the same item but with diff. SNs. L1 on PR1.SN1 and L2 on PR2.SN2.
</li>
<li> the critical case is when SRS doesnt know of the more recent PR2 when being asked to RESEQUENCE PR1, like so:
<ol>
<li> If L1 calls SRS with RESEQUENCE before L2 gets to call REGISTER, SRS doesnt know that an SN2 is out there and lets PR with SN1 pass thru.
</li>
<li> this is not an error b/c the order of processing is maintained. it has a drawback however, which is that SRS is unable to supress PR1.
</li>
</ol>
</li>
<li> i cannot think of another case where this setup leads to an aerror , b/c all PRs are first registered before any further processing may take place this setup works in all cases IMO. (please prove me wrong)
</li>
</ol>
<p><b>pro:</b> reduced overhead due to one less Q and pipeline
</p><p><b>con:</b> it might be easy to forget to call the SRS in each pipeline.
</p><p><br />
</p>
<h4><span class="mw-headline" id="concurrent_REGISTERING_setup_with_mirror_Queue">concurrent REGISTERING setup with mirror Queue</span></h4>
<p>in this setup a copy of the PR with its SN is sent to an additional Q2 in parallel to Q1 (two send tasks in router). SRS is the only listener on Q2 and REGISTERS all the PRs.
</p><p>this will introduce an error in case that a PR is RESEQUENCed while for the same item there is a new PR waiting in Q2.
</p><p>the problem here is generally that REGISTERing happens asynchronously to processing and hence cannot be safe.
</p><p><b>fix:</b> the error introcuced by this setup can be fixed by demanding that the SRS is first to process all PRs on Q2 before RESEQUENCing PRs. however, this may cause PRs only to be added to the index as long as the agent produces PRs. in that case it wont be better than the non-concurent setup.
</p>
<h3><span class="mw-headline" id="Basics_Impl._Ideas">Basics Impl. Ideas</span></h3>
<ul>
<li> implemented as ProcessingService
</li>
<li> records are sent to it with the command/process mode REGISTER and SEQUENCE
</li>
<li> SN and process mode are given as annotation on the record, called the Config Annotation (CA). this is the same way as with the lucen service. (i first wanted to do this as JMS props but they are not accessible in a ProcessingService)
</li>
<li> map may be in memory or a persisting solution may be implemented/chosen. <br /> IMO the amount of records held im momory should be relativly small, only to the amount of what is in the processing chain. (hm, that can be a lot, since connectivity is not pausing crawlers and agents (yet) if there is much in the MQ)
</li>
</ul>
<p><br />
</p>
<h3><span class="mw-headline" id="Meeting_Requirements">Meeting Requirements</span></h3>
<p>The general ones should be sufficeiently clear from the functional description of the SRS. here come now the further ones:
</p>
<h4><span class="mw-headline" id="split_records">split records</span></h4>
<p>compound and aggregation are handled the same way, like so:
</p><p>the processing step splitting the record is responsible for the the following:
</p>
<ul>
<li> all descendants inherit the SN from their root
</li>
<li> if internal ordering: <br />order of PRs for descendants is noted in their respective ConfigAnnotation (e.g. a link to the ID of the preceding or succeeding resource or such)
</li>
<li> register the split records with the SRS
</li>
<li> possibly deregister the root and/or intermediate PRs if these are not processed further
</li>
</ul>
<p>the SRS will
</p>
<ul>
<li> collect all PRs belonging to the tree of split PRs until it is complete
<ul>
<li> missing PRs:
<ul>
<li> timeout
</li>
<li> config on how to continue with non-complete trees: {all or nothing, sequence incomplete}
</li>
</ul>
</li>
</ul>
</li>
</ul>
<h4><span class="mw-headline" id=".3E1_Processing_Targets">&gt;1 Processing Targets</span></h4>
<p>this can be supported in diff. ways. both have in common:
</p>
<ol>
<li> sending the PR to any of the PTs is done thru the SRS by calling it with the RESEQUENCE command (this is just a generalization of the basic concept and repeated here for clarity)
</li>
<li> the SRS needs to know how many (potential) PTs there are for a resource (determine by the ID) and when processing really has finished for a given ID.
</li>
<li> each RESEQUENCE and UNREGISTER command will reduce the count, when it reaches 0 all PRs have reached their PT and the ID can be removed from the map.
</li>
</ol>
<h5><span class="mw-headline" id="idea_1_-_SRS_ID_rules">idea 1 - SRS ID rules</span></h5>
<ul>
<li> the config of the SRS contains rules or conditions that determine the count.
</li>
<li> it starts with that count wich is computed on REGISTRATION.
</li>
</ul>
<h5><span class="mw-headline" id="idea_2_-_processing_steps_control_counter">idea 2 - processing steps control counter</span></h5>
<ul>
<li> processing steps take care of in- and decerementing the counter in the normal processing chain by using the REGISTER and UNREGISTER commands to reflect additional or obsolte PTs
</li>
</ul>
<h5><span class="mw-headline" id="Opinion">Opinion</span></h5>
<p>i like idea 2 better b/c it puts the config of the SRS in the same place that also controls the flow of PRs anyhow. it is just a matter of including an SRS call with the respective command.<br />
in contrast, idea 1 would mean that we have to:
</p>
<ul>
<li> duplicate the processing chain logic in some other place
</li>
<li> implement a rule/condition engine and config.
</li>
</ul>
<h4><span class="mw-headline" id="clustering.2C_complex_processing_chain">clustering, complex processing chain</span></h4>
<ul>
<li> complex processing chains are possible as already described in other places. the SRS just needs to be placed in front of the PT and called in the flow of things
</li>
<li> resequencing in a cluster scenario works OOB just the setup/config changes.
<ul>
<li> SRS is run on several nodes and shares a custer capable map OR
</li>
<li> if the router sends PRs for the same item always to the same processing node, then the SRS can be local to the processing nodes and setup as normal OR
</li>
<li> SRS runs on just one node. then
<ul>
<li> all messages from the router need to be send to the SRS node first for REGISTERING<br />it makes sense to have th SRS and router on the same node to avoid chnaging nodes for the first step.
</li>
<li> all processing nodes dont call he PT directly in their pipeline but send their result to SRS node
</li>
<li> the SRS RESEQUENCE pipeline will call the PT.
</li>
</ul>
</li>
</ul>
</li>
</ul>
<p><b>Note</b><br />
i think the SRS will even work if the assumption that each PT has only one instance and node that solely accesses the PT holds not true.
a setup like this will then segment all PRs by some scheme that depends on the ID and then the SRS has only to resequence the one segement and thus: all is well.
</p>
<h4><span class="mw-headline" id="single_point_of_failure">single point of failure</span></h4>
<p>hm. this is a tough one as the single'nes is inherent. i have no clue yet, how to solve this, other than to use a fail-over solution.
i guess, just as the PT itself, it needs to be monitored closely to detect malfunctions.
</p>
<h4><span class="mw-headline" id="scalability_and_performance">scalability and performance</span></h4>
<p>there is some performance degradation to be expected because
</p>
<ol>
<li> SRS increases the number of threads
</li>
<li> SRS is inserted at least 2 times into the processing flow, namely at the beginning and end.
<ol>
<li> when it registers an item
</li>
<li> when it resquences it
</li>
</ol>
</li>
</ol>
<p>the internal workings of these steps are fairly simple and should not take much time compared to the rest of the processing, albeit in a highly concurrent scenario the synchronization will take its toll.
</p>
<h3><span class="mw-headline" id="Cases_introduced_thru_this_solution">Cases introduced thru this solution</span></h3>
<p>this section lists cases and problems that need to be covered that are introduced thru the solution itself.
some of the items listed here will also apply to the FSR!!
</p>
<ul>
<li> handling of unregistered records<br />what happens when SEQUENCING a PR that is already @ count 0 /not existing.
</li>
<li> what to do with recods that miss needed config data? <br /> handling depends on the process mode:
<ul>
<li> SEQUENCE: error as default , but outcome could be config'able such as&#160;: DLQ, any other Q
</li>
<li> REGISTER: error
</li>
</ul>
</li>
<li> overflow of the SN<br />a reset signal must be sent to SRS
</li>
</ul>
<p><br />
</p>
<h3><span class="mw-headline" id="PRO">PRO</span></h3>
<ul>
<li> smarter&#160;;) than FRS
</li>
<li> no change to APIs are needed, implementation of agents/crawlers (controller) needs to add the SN as an annotation to each record. which can be turned on or off via config.
</li>
<li> unobstrutive, SRS can be used or not.
</li>
</ul>
<h3><span class="mw-headline" id="CON">CON</span></h3>
<ul>
<li> see also almost all CONS @ FRS
</li>
<li> oscillating resources (that constantly change) will never make it into the index.
</li>
</ul>
<!--
NewPP limit report
CPU time usage: 0.032 seconds
Real time usage: 0.033 seconds
Preprocessor visited node count: 71/1000000
Preprocessor generated node count: 76/1000000
Post‐expand include size: 0/2097152 bytes
Template argument size: 0/2097152 bytes
Highest expansion depth: 2/40
Expensive parser function count: 0/100
-->
<!-- Saved in parser cache with key my_wiki:pcache:idhash:21924-0!*!*!!en!*!* and timestamp 20150414084733 and revision id 173806
-->
</div>
<!-- catlinks -->
<div id='catlinks' class='catlinks'><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Category</a>: <ul><li><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></li></ul></div></div> <!-- /catlinks -->
</div>
</div>
</div>
</div>
<!-- /maincontent -->
<!-- printfooter -->
<div class="printfooter">
Retrieved from "<a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;oldid=173806">http://wiki.eclipse.org/index.php?title=SMILA/Specifications/Processing_Message_Resequencer/Smart_Resequencer&amp;oldid=173806</a>" </div>
<!-- /printfooter -->
<!-- debughtml -->
<!-- /debughtml -->
</div>
<!-- /bodyContent -->
</section>
<!-- /content -->
<!-- footer -->
</div> <section id="footer-contribution-info" style="border-top:1px solid #ccc;" class="footer-offset background-white margin-top-25"><div class="container text-center padding-top-10 padding-bottom-10"><p id="footercredit">This page was last modified 16:43, 7 October 2009 by <a href="http://wiki.eclipse.org/User:Tmenzel.brox.de" title="User:Tmenzel.brox.de">thomas menzel</a>.</p><p id="footerviews">This page has been accessed 2,326 times.</p></div></section> </main> <!-- /#main-content-container-row -->
<p id="back-to-top" class="noprint hidden-print">
<a class="visible-xs" href="Smart_Resequencer.html#top">Back to the top</a>
</p>
<footer role="contentinfo" class="noprint hidden-print">
<div class="container">
<div class="row">
<section id="footer-eclipse-foundation" class="col-xs-offset-1 col-xs-11 col-sm-7 col-md-6 col-md-offset-0">
<h2 class="section-title">Eclipse Foundation</h2>
<ul class="nav">
<li><a href="https://eclipse.org/org/">About us</a></li>
<li><a href="https://eclipse.org/org/foundation/contact.php">Contact Us</a></li>
<li><a href="https://eclipse.org/donate">Donate</a></li>
<li><a href="https://eclipse.org/org/documents/">Governance</a></li>
<li><a href="https://eclipse.org/artwork/">Logo and Artwork</a></li>
<li><a href="https://eclipse.org/org/foundation/directors.php">Board of Directors</a></li>
</ul>
</section>
<section id="footer-legal" class="col-xs-offset-1 col-xs-11 col-sm-7 col-md-6 col-md-offset-0">
<h2 class="section-title">Legal</h2>
<ul class="nav">
<li><a href="https://eclipse.org/legal/privacy.php">Privacy Policy</a></li>
<li><a href="https://eclipse.org/legal/termsofuse.php">Terms of Use</a></li>
<li><a href="https://eclipse.org/legal/copyright.php">Copyright Agent</a></li>
<li><a href="https://eclipse.org/org/documents/epl-v10.php">Eclipse Public License </a></li>
<li><a href="https://eclipse.org/legal/">Legal Resources </a></li>
</ul>
</section>
<section id="footer-useful-links" class="col-xs-offset-1 col-xs-11 col-sm-7 col-md-6 col-md-offset-0">
<h2 class="section-title">Useful Links</h2>
<ul class="nav">
<li><a href="https://bugs.eclipse.org/bugs/">Report a Bug</a></li>
<li><a href="http://help.eclipse.org/">Documentation</a></li>
<li><a href="https://eclipse.org/contribute/">How to Contribute</a></li>
<li><a href="https://eclipse.org/mail/">Mailing Lists</a></li>
<li><a href="https://eclipse.org/forums/">Forums</a></li>
<li><a href="http://marketplace.eclipse.org/">Marketplace</a></li>
</ul>
</section>
<section id="footer-other" class="col-xs-offset-1 col-xs-11 col-sm-7 col-md-6 col-md-offset-0">
<h2 class="section-title">Other</h2>
<ul class="nav">
<li><a href="https://eclipse.org/ide/">IDE and Tools</a></li>
<li><a href="https://eclipse.org/projects">Community of Projects</a></li>
<li><a href="https://eclipse.org/org/workinggroups/">Working Groups</a></li>
</ul>
<ul class="list-inline social-media">
<li><a href="https://twitter.com/EclipseFdn"><i class="fa fa-twitter-square"></i></a></li>
<li><a href="https://plus.google.com/+Eclipse"><i class="fa fa-google-plus-square"></i></a></li>
<li><a href="https://www.facebook.com/eclipse.org"><i class="fa fa-facebook-square"></i> </a></li>
<li><a href="https://www.youtube.com/user/EclipseFdn"><i class="fa fa-youtube-square"></i></a></li>
</ul>
</section>
<div id="copyright" class="col-xs-offset-1 col-sm-14 col-md-24 col-md-offset-0">
<div>
<span><img src="http://eclipse.org/eclipse.org-common/themes/solstice/public/images/logo/eclipse-logo-bw-800x188.png" alt="Eclipse.org black and white logo" width="166" height="39" id="logo-eclipse-white"/></span>
<p id="copyright-text">Copyright &copy; 2014 The Eclipse Foundation. All Rights Reserved.</p>
</div>
</div>
<a href="Smart_Resequencer.html#" class="scrollup">Back to the top</a>
</div>
</div>
</footer>
<script src="http://wiki.eclipse.org/skins/solstice/public/javascript/main.min.js"></script>
<!-- Placed at the end of the document so the pages load faster -->
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-910670-2']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script> <!-- /footer -->
<script>/*<![CDATA[*/window.jQuery && jQuery.ready();/*]]>*/</script><script>if(window.mw){
mw.loader.state({"skins.solstice":"loading","site":"ready","user":"ready","user.groups":"ready"});
}</script>
<script src="http://wiki.eclipse.org/load.php?debug=false&amp;lang=en&amp;modules=skins.solstice&amp;only=scripts&amp;skin=solstice&amp;*"></script>
<script>if(window.mw){
mw.loader.load(["mediawiki.action.view.postEdit","mediawiki.user","mediawiki.hidpi","mediawiki.page.ready","mediawiki.searchSuggest"],null,true);
}</script>
<script>if(window.mw){
mw.config.set({"wgBackendResponseTime":348});
}</script> </body>
</html>