| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Specifications/ProcessingPerformanceDiscussion,Daniel.stucky.empolis.com" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Specifications/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Specifications/ProcessingPerformanceDiscussion - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Specifications/ProcessingPerformanceDiscussion"; |
| var wgTitle = "SMILA/Specifications/ProcessingPerformanceDiscussion"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "20463"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "161195"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <link rel="stylesheet" type="text/css" href="ProcessingPerformanceDiscussion.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Specifications_ProcessingPerformanceDiscussion"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Specifications/ProcessingPerformanceDiscussion">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Specifications/ProcessingPerformanceDiscussion">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Specifications/ProcessingPerformanceDiscussion">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/ProcessingPerformanceDiscussion&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/ProcessingPerformanceDiscussion&oldid=161195">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="ProcessingPerformanceDiscussion.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/Talk:SMILA/Specifications/ProcessingPerformanceDiscussion"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/ProcessingPerformanceDiscussion&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Specifications/ProcessingPerformanceDiscussion&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Specifications/ProcessingPerformanceDiscussion"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Specifications/ProcessingPerformanceDiscussion</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Specifications.1.html" title="SMILA/Specifications">Specifications</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="ProcessingPerformanceDiscussion.html#column-one">navigation</a>, <a href="ProcessingPerformanceDiscussion.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="ProcessingPerformanceDiscussion.html#Discussion_about_Processing_Performance"><span class="tocnumber">1</span> <span class="toctext">Discussion about Processing Performance</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="ProcessingPerformanceDiscussion.html#Send_and_Receive_Queue_messages_in_blocks"><span class="tocnumber">1.1</span> <span class="toctext">Send and Receive Queue messages in blocks</span></a></li> |
| <li class="toclevel-2"><a href="ProcessingPerformanceDiscussion.html#CrawlerController_using_blocks_of_Records"><span class="tocnumber">1.2</span> <span class="toctext">CrawlerController using blocks of Records</span></a></li> |
| <li class="toclevel-2"><a href="ProcessingPerformanceDiscussion.html#asynchronus_ConnectivityManager"><span class="tocnumber">1.3</span> <span class="toctext">asynchronus ConnectivityManager</span></a></li> |
| <li class="toclevel-2"><a href="ProcessingPerformanceDiscussion.html#Tuning_Block_Sizes_and_Threads"><span class="tocnumber">1.4</span> <span class="toctext">Tuning Block Sizes and Threads</span></a></li> |
| <li class="toclevel-2"><a href="ProcessingPerformanceDiscussion.html#Conclusion"><span class="tocnumber">1.5</span> <span class="toctext">Conclusion</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Discussion_about_Processing_Performance"></a><h2> <span class="mw-headline"> Discussion about Processing Performance </span></h2> |
| <p>This page contains some thoughts about issues of the current implementation and ideas to improve overall processing performance. |
| </p><p>The Queue is the core component in regards to processing. It is filled by Routers and it's messages are consumed by Listeners and everything depends on a fast enqueuing and dequeuing of messages. The current implementation sends and receives single messages only. For each received message a BPEL pipeline is initialized and executed with a single record. |
| </p> |
| <a name="Send_and_Receive_Queue_messages_in_blocks"></a><h3> <span class="mw-headline"> Send and Receive Queue messages in blocks </span></h3> |
| <p>The components that provide the records to be sent (ConnectivityManager) and processed the received records (WorkflowProcessor, Pipelets/ProcessingServices) already offer the possibility to be executed not only with single records but with arrays of records (or record Ids). Therefore we should send/receive more than one single message. This can be achieved by sending/receiving multiple messages before committing the JMS session in the the Router and Listener implementations. Both should not work with fixed block sizes and wait for it to be filled but use dynamic block sizes (with a maximum size). |
| One drawback of this optimization is that we do not get feedback (exceptions) on single records anymore. If an error occurs then processing of all records of the current block is aborted. |
| </p><p><a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>: I did a prototype implementation for the Listener and also tried to do it for the Router. Some interfaces (e.g. TaskExecutionService, AbstractTask) had to be improved to support arrays of Record, Id and Operation objects. The ConnectivityManagerImpl had to be modified so that the incoming array of records are not processed record by record but passed in whole to the Router. |
| </p><p>The Router implementation gets a littlee more complicated, as records from different data sources may be contained in the passed Record array (this does currently not happend when used by the CrawlerController). So each record of the array may be routed by a different RouterRule (e.g. routed to different pipelines or even to different Queues). So we need to rearrange the records of the incoming array into arrays per RouterRule and send those new smaller arrays to the queue in one session. How many messages are sent in one session depends solely in the size of the array. |
| </p><p>A more general approach to increase messages per session (one session per rule) would be to buffer messages per rule in the Send Task. We would have to watch these buffers by separate threads in order to send the content of the buffer if it has reached its maximum size or if a timeout has elapsed. This logic is not trivial. I did a quick hack implementation (for just one rule) and did some tests. Sending blocks of messages in one session does improve import performance but the gain is not in proportion to the effort needed to implement this behavior. There is already a component specified that could do the trick: the Buffer of the ConnectivityManager. It could be used to optimize the records passed top the Router (e.g. by combining all records of the same data source). <b>So buffering in the Send Task should NOT be implemented!!!</b> |
| </p><p>In contrast the Listener was easier to implement, as you can receive messages from the Queue directly or using a timeout to wait for messages to arrive. And all received messages are processed by the same pipeeline, so no extra checking is required. Here are the result of the tests I made importing 5078 html files concerning the Listener changes: |
| </p> |
| <table border="1"> |
| <tr> |
| <th>#records</th><th>Listener Block Size</th><th>BPEL invokes</th><th>runtime</th><th>improvement |
| </th></tr> |
| <tr> |
| <td>5078</td><td>1</td><td>5078</td><td>17:58</td><td> |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>501</td><td>14:40</td><td>18% faster |
| </td></tr></table> |
| <p><br /> |
| As we can see the number of BPEL invokes is drastically reduced. With a block size of 20 the minimal number of invokes would be 254. The actual number is still twice as much. This leads to the conclusion that the Listener wants to receive more messages from the Queue than are available. |
| </p> |
| <a name="CrawlerController_using_blocks_of_Records"></a><h3> <span class="mw-headline"> CrawlerController using blocks of Records </span></h3> |
| <p>So the next option is to call <tt>ConnectivityManager.add(Record[])</tt> with more than one record as it is currently implemented. Therefore the CrawlerController (to be more precise the class CrawlThread) has to buffer incoming records from the Crawler and send them in a bigger array to ConnectivityManager. |
| The drawback is that if one record produces an error processing of the whole array is aborted. In addition the usage of memory is slightly increased. |
| </p><p><br /> |
| <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>: I also implemented another prototype doing the same tests as before with a array size > 1. |
| </p> |
| <table border="1"> |
| <tr> |
| <th>#records</th><th>Add Block Size</th><th>Listener Block Size</th><th>BPEL invokes</th><th>runtime</th><th>improvement |
| </th></tr> |
| <tr> |
| <td>5078</td><td>1</td><td>1</td><td>5078</td><td>17:58</td><td> |
| </td></tr> |
| <tr> |
| <td>5078</td><td>10</td><td>20</td><td>266</td><td>13:08</td><td>27% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>20</td><td>261</td><td>12:21</td><td>31% faster |
| </td></tr></table> |
| <p>Of course this improvement is only beneficial if the Listener works with a block size > 1 ! |
| </p><p><br /> |
| </p> |
| <a name="asynchronus_ConnectivityManager"></a><h3> <span class="mw-headline"> asynchronus ConnectivityManager </span></h3> |
| <p>In the Router the records metadata and attachments are persisted in the stores and a JMS message is created and sent to the Queue. The calling CrawlerController blocks execution until the call to <tt>ConnectivityManager.add(Record[])</tt> returns. It seems a good practice to do the persisting and messaging in a separate Thread so that ConnectivityManager does not block the CrawlerController and it can continue to send the next block of records. |
| </p><p>A major drawback is that the CrawlerController does not get any feedback whether the records have been successfully added to the Queue or not and so can't update the state in DeltaIndexingManager (set the visited flag). The ConnectivityManager thread in execution would have to use a callback on CrawlerController or directly communicate with DeltaIndexing (using the CrawlerControllers session) to update it. |
| </p><p><a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>: I did another prototype implementation to find out that either the number of threads increases drastically (about half the number of records sent) or (if limited to a maximum count) you have to synchronize <tt>add()</tt> calls. Compared to the already optimized version using blocks it got even slower, though the number of BPEL invokes was significantly lower. |
| </p> |
| <table border="1"> |
| <tr> |
| <th>#records</th><th>Listener Block Size</th><th>BPEL invokes</th><th>runtime</th><th>method call</th><th>improvement |
| </th></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>501</td><td>14:40</td><td>blocking</td><td> |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>258</td><td>16:10</td><td>oneway</td><td> 10% slower |
| </td></tr></table> |
| <p><b>So this should NOT be implemented !!!</b> |
| </p><p><br /> |
| </p> |
| <a name="Tuning_Block_Sizes_and_Threads"></a><h3> <span class="mw-headline"> Tuning Block Sizes and Threads </span></h3> |
| <p>Depending on the system resources different configurations may lead to better results. Therefore one has to adjust the block sizes used and the number of Listener worker threads. |
| </p><p>Test machine: Intel Core 2 CPU 2.66 GHz, 2 GB Ram |
| </p> |
| <table border="1"> |
| <tr> |
| <th>#records</th><th>Add Block Size</th><th>Listener Block Size</th><th>Worker-Threads</th><th>BPEL invokes</th><th>runtime</th><th>improvement |
| </th></tr> |
| <tr> |
| <td>5078</td><td>1</td><td>1</td><td>2</td><td>5078</td><td>17:58</td><td> |
| </td></tr> |
| <tr> |
| <td>5078</td><td>10</td><td>20</td><td>2</td><td>266</td><td>13:08</td><td>~27% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>20</td><td>2</td><td>261</td><td>12:21</td><td>~31% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>20</td><td>4</td><td>294</td><td>10:23</td><td>~42% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>40</td><td>20</td><td>4</td><td>261</td><td>08:54</td><td>~50% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>40</td><td>40</td><td>4</td><td>147</td><td>08:58</td><td>~50% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>20</td><td>20</td><td>8</td><td>355</td><td>09:07</td><td>~49% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>40</td><td>20</td><td>8</td><td>299</td><td>09:03</td><td>~50% faster |
| </td></tr> |
| <tr> |
| <td>5078</td><td>40</td><td>40</td><td>8</td><td>186</td><td>09:37</td><td>~46% faster |
| </td></tr></table> |
| <p><br /> |
| Remember the following rules of thumb: |
| </p> |
| <ul><li> bigger Add block size -> higher memory usage, more enqueued messages |
| </li><li> bigger Listener block size -> higher memory usage, less BPEL invokes |
| </li><li> bigger number of worker threads -> more CPU usage, more BPEL invokes, more dequeued messages |
| </li></ul> |
| <p><br /> |
| </p> |
| <a name="Conclusion"></a><h3> <span class="mw-headline"> Conclusion </span></h3> |
| <p>We should implement the following enhancements to gain a processing performance boost: |
| </p> |
| <ul><li> make use of Record and Id arrays (prerequisite for all other enhancements) |
| <ul><li>change the ConnectivityManager implementation to pass the array to the router |
| </li><li>change interfaces and implementations of TaskExecutionService, AbstractTask, etc. |
| </li></ul> |
| </li><li> build arrays per Router-Rule in the Router and sent those arrays in one session |
| </li><li> use a dynamic block size in the Listener when receiving messages (receive multiple messages per session) |
| </li><li> use a dynamic block size in the Crawlhread |
| </li></ul> |
| <p>Of course some values should be configurable (maximum array/block sizes, timeouts). |
| </p><p>If there are no objections I will create a bugzilla Id and implement these enhancements. |
| </p> |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 12/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:20463-0!1!0!!en!2!edit=0 and timestamp 20120203101556 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="ProcessingPerformanceDiscussion.html">http://wiki.eclipse.org/SMILA/Specifications/ProcessingPerformanceDiscussion</a>"</div> |
| <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2012 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 10:24, 29 June 2009 by <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>. Based on work by <a href="http://wiki.eclipse.org/User:Igor.novakovic.empolis.com" title="User:Igor.novakovic.empolis.com">Igor Novakovic</a>.</p> |
| <p id="footerviews">This page has been accessed 1,556 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.123 secs. --></body></html> |
