| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Default configuration workflow overview" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Default configuration workflow overview - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Default_configuration_workflow_overview"; |
| var wgTitle = "SMILA/Default configuration workflow overview"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "16371"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "247208"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <link rel="stylesheet" type="text/css" href="Workflow_Overview.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Default_configuration_workflow_overview"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Default_configuration_workflow_overview">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Default_configuration_workflow_overview">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Default_configuration_workflow_overview">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Default_configuration_workflow_overview&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Default_configuration_workflow_overview&oldid=247208">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="Default_configuration_workflow_overview.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Default_configuration_workflow_overview&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Default_configuration_workflow_overview&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Default_configuration_workflow_overview&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Default%20configuration%20workflow%20overview"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Default configuration workflow overview</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../SMILA.html" title="SMILA">SMILA</a></span>(Redirected from <a href="http://wiki.eclipse.org/index.php?title=SMILA/Workflow_Overview&redirect=no" title="SMILA/Workflow Overview">SMILA/Workflow Overview</a>)</div> |
| <div id="jump-to-nav">Jump to: <a href="Workflow_Overview.html#column-one">navigation</a>, <a href="Workflow_Overview.html#searchInput">search</a></div> <!-- start content --> |
| <p><a href="http://wiki.eclipse.org/Image:Schema_.jpg" class="image" title="Image:Schema_.jpg"><img alt="Image:Schema_.jpg" src="http://wiki.eclipse.org/images/6/61/Schema_.jpg" width="1249" height="816" border="0" /></a> |
| </p><p><br /> |
| </p> |
| <a name="The_diagramme_description"></a><h2> <span class="mw-headline"> The diagramme description</span></h2> |
| <ul><li>1. Creation or update of an <b>index</b> is initiated by a <b>user</b> request. The user sends a name of a configuration file (so called <b>IndexOrderConfiguration</b>) to the <b>Crawler Controller</b>. |
| </li></ul> |
| <p>The configuration file describes which <b>Crawler</b> should access a specific <b>Data Source</b>. Furthermore it contains all necessary parameters for the crawling process. |
| </p> |
| <ul><li>2. The <b>Crawler Controller</b> initializes this <b>Crawlers</b>: |
| <ul><li> a. The <b>Crawler Controller</b> reads the Index order configuration file and assigns the <b>Data Source</b> from the configuration file to the <b>Crawler</b>. |
| </li><li> b. The <b>Crawler Controller</b> starts the Crawler’s thread. |
| </li></ul> |
| </li></ul> |
| <ul><li>3. The <b>Crawler</b> retrieves data records from the <b>Data Source</b> and returns each record’s attributes to the <b>Crawler Controller</b> which are required for the generation of the <b>Record’s ID</b> and its <b>Hash</b>. |
| </li></ul> |
| <ul><li>4. The <b>Crawler Controller</b> generates <b>ID</b> and <b>Hash</b> and then determines whether this particular record is new or was already indexed by querying the <b>Delta Indexing Service</b>. |
| </li></ul> |
| <ul><li>5. If the <b>record</b> was <b>not</b> previously indexed, the <b>Crawler Controller</b> instructs the <b>Crawler</b> to retrieve the full content of the <b>record</b> (metadata and content). |
| </li></ul> |
| <ul><li>6. The <b>Crawler</b> fetches the complete <b>record</b> from the <b>Data Source</b>. Each <b>record</b> has an ID and can contain attributes, attachments (contain binary content) and annotations. |
| </li></ul> |
| <ul><li>7. The <b>Crawler Controller</b> sends the complete retrieved <b>Records</b> to the <b>Connectivity</b> module. |
| </li></ul> |
| <ul><li>8. The <b>Connectivity module</b> in turn persists the record to the <b>storage tier</b> via the <b>Blackboard Service</b>. |
| </li></ul> |
| <p>The <b>Blackboard</b> provides central access to the record storage layer for all SMILA components thus effectively constituting an abstraction layer: clients do not need to know about the underlying persistence / storage technologies. |
| </p><p><b>Record's</b> data without attachments (binaries content) is stored in <b>XML-Storage</b>, attachments are stored in <b>Bin-Storage</b>. |
| </p> |
| <ul><li>9. In the next step the <b>Connectivity module</b> transmits the <b>record’s</b> to the <b>Router</b> which is part of the <b>Connectivity module</b>. |
| </li></ul> |
| <p>The <b>Router</b> filters the <b>record’s</b> attributes according to its configuration. <i>Note</i>: usually only the ID is passed (it is defined in a filter configurations file). |
| </p><p>After processing and filtering the record, the <b>Router</b> pushes the record in a JMS message queue (<b>Active MQ</b>) for further processing. The Router can filter the <b>Record</b> , then only necessary data (like <b>ID</b>) is fetched in the <b> Queue</b>. |
| </p><p>Should any subsequent processes require the <b>record’s</b> full content, they can access it via the <b>Blackboard</b>. |
| </p> |
| <ul><li>10. A <b>Listener</b> subscribing to the <b>Queue’s</b> topic receives the message and invokes further processing logic according to its configuration. |
| </li></ul> |
| <ul><li>11. The <b>Listener</b> passes the <b>record</b> to the respective <b>pipeline</b> – the <b>Add pipeline</b>, if it is a new record, the <b>Delete pipeline</b> if the <b>record</b> needs to be removed from the <b>index</b> (because it has been deleted from the original <b>Data Source</b>). |
| </li></ul> |
| <p>A <b>pipeline</b> is a <b>BPEL</b> process using a set of <b>pipelets</b> to process a <b>Record’s</b> data (e.g. extracting text from various document or image file types). |
| </p><p><b>BPEL</b> is a XML-based language to define business processes by means of orchestrating loosely coupled (web) services. |
| <b>BPEL</b> processes require a <b>BPEL</b> runtime environment (e.g. Apache ODE). |
| </p> |
| <ul><li>12. After processing the <b>record</b> the <b>pipelets / service</b> stores the gathered additional data via the <b>Blackboard</b> service. The <b>Listener</b> will send the <b>Record</b> (can be filter too) to the <b>Queue</b> back and other <b>Data Flow Process</b> can access it. |
| </li></ul> |
| <ul><li>13. Finally the <b>pipeline</b> can invoke the <b>Lucene Index Service</b> via a small pipelet, the LuceneIndexPipelet. (matter of the configuration). |
| </li></ul> |
| <ul><li>14. The <b>Lucene Index Service</b> updates the <b>Lucene Index</b>. |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 2/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:16371-0!1!0!!en!2!edit=0 and timestamp 20110617074934 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="Default_configuration_workflow_overview.html">http://wiki.eclipse.org/SMILA/Default_configuration_workflow_overview</a>"</div> |
| <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2011 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 14:12, 20 April 2011 by <a href="http://wiki.eclipse.org/index.php?title=User:Juergen.schumacher.attensity.com&action=edit" class="new" title="User:Juergen.schumacher.attensity.com">Juergen Schumacher</a>. Based on work by <a href="http://wiki.eclipse.org/User:Igor.novakovic.empolis.com" title="User:Igor.novakovic.empolis.com">Igor Novakovic</a>, <a href="http://wiki.eclipse.org/index.php?title=User:Rrausch.brox.de&action=edit" class="new" title="User:Rrausch.brox.de">Ralf Rausch</a> and <a href="http://wiki.eclipse.org/index.php?title=User:Szhbankova.brox.de&action=edit" class="new" title="User:Szhbankova.brox.de">Sofya Zhbankova</a>.</p> |
| <p id="footerviews">This page has been accessed 4,143 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.076 secs. --></body></html> |
