| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Project Concepts/Core Indexing Process (global view),SMILA/Project Concepts/Components and Modules" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Project_Concepts/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Project Concepts/Core Indexing Process (global view) - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Project_Concepts/Core_Indexing_Process_(global_view)"; |
| var wgTitle = "SMILA/Project Concepts/Core Indexing Process (global view)"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "15295"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "113714"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <link rel="stylesheet" type="text/css" href="Core_Indexing_Process_(global_view).html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Project_Concepts_Core_Indexing_Process_global_view"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Project_Concepts/Core_Indexing_Process_(global_view)">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29&oldid=113714">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="Core_Indexing_Process_(global_view).html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Project%20Concepts/Core%20Indexing%20Process%20(global%20view)"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Project Concepts/Core Indexing Process (global view)</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Project_Concepts.1.html" title="SMILA/Project Concepts">Project Concepts</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="Core_Indexing_Process_(global_view).html#column-one">navigation</a>, <a href="Core_Indexing_Process_(global_view).html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="Core_Indexing_Process_(global_view).html#Description"><span class="tocnumber">1</span> <span class="toctext">Description</span></a></li> |
| <li class="toclevel-1"><a href="Core_Indexing_Process_(global_view).html#Discussion"><span class="tocnumber">2</span> <span class="toctext">Discussion</span></a></li> |
| <li class="toclevel-1"><a href="Core_Indexing_Process_(global_view).html#Technical_proposal"><span class="tocnumber">3</span> <span class="toctext">Technical proposal</span></a></li> |
| <li class="toclevel-1"><a href="Core_Indexing_Process_(global_view).html#SMILA_Core_Indexing_Process_.28Global_View.29"><span class="tocnumber">4</span> <span class="toctext">SMILA Core Indexing Process (Global View)</span></a></li> |
| <li class="toclevel-1"><a href="Core_Indexing_Process_(global_view).html#Process_overview"><span class="tocnumber">5</span> <span class="toctext">Process overview</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="Core_Indexing_Process_(global_view).html#Create.2FDelete_Record"><span class="tocnumber">5.1</span> <span class="toctext">Create/Delete Record</span></a></li> |
| <li class="toclevel-2"><a href="Core_Indexing_Process_(global_view).html#Delta-Indexing"><span class="tocnumber">5.2</span> <span class="toctext">Delta-Indexing</span></a></li> |
| <li class="toclevel-2"><a href="Core_Indexing_Process_(global_view).html#Index_creation"><span class="tocnumber">5.3</span> <span class="toctext">Index creation</span></a></li> |
| <li class="toclevel-2"><a href="Core_Indexing_Process_(global_view).html#Compound_Management"><span class="tocnumber">5.4</span> <span class="toctext">Compound Management</span></a></li> |
| <li class="toclevel-2"><a href="Core_Indexing_Process_(global_view).html#Maintenance_Operations"><span class="tocnumber">5.5</span> <span class="toctext">Maintenance Operations</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Description"></a><h2> <span class="mw-headline"> Description </span></h2> |
| <p>This concept describes the general system architecture. Additionally core or general concepts are referenced. Good and bad practices to keep a ceirtain maintainability are handled also. |
| </p><p>Key parts of the architecture could be found in the following concepts: |
| </p> |
| <ul><li> <a href="Components_and_Modules.html" title="SMILA/Project Concepts/Components and Modules">SMILA/Project_Concepts/Components and Modules</a> |
| </li></ul> |
| <a name="Discussion"></a><h2> <span class="mw-headline"> Discussion </span></h2> |
| <a name="Technical_proposal"></a><h2> <span class="mw-headline"> Technical proposal </span></h2> |
| <p>{info} |
| Note: This section may only be edited by assigned developer(s). His responsibility is also to reflect any agreed changes/details in discussion section. |
| {info} |
| </p><p><br /> |
| </p> |
| <a name="SMILA_Core_Indexing_Process_.28Global_View.29"></a><h2> <span class="mw-headline"> SMILA Core Indexing Process (Global View) </span></h2> |
| <a name="Process_overview"></a><h2> <span class="mw-headline"> Process overview </span></h2> |
| <a name="Create.2FDelete_Record"></a><h3> <span class="mw-headline"> Create/Delete Record </span></h3> |
| <ul><li> Compounds |
| </li><li> Optimized queue access -> could obsolete requests be avoided. An example for a obsolete request is the indexing of a document while a delete operation is already present in the queue. It must be guaranteed that its not required to read all messages in the queue (in client process); The question is are queues available that could cover such issues? |
| </li><li> Parallel processing could lead to difficulties when concurrently performing create/delete operations |
| </li><li> The update of fields of a document in the storage must be possible (e.g. the user rights field) |
| </li><li> Delete by Query (several objects, XQuery, Source, ...) |
| </li></ul> |
| <a name="Delta-Indexing"></a><h3> <span class="mw-headline"> Delta-Indexing </span></h3> |
| <ul><li> Source or subset (a part of information in storage) based |
| </li><li> Compounds |
| </li><li> Status storage using an interface (status for delta discovery at IRM, probably Lucene or an Indexer as storage (e.g. hashes, data, URLs, modifications in user rights) |
| </li></ul> |
| <a name="Index_creation"></a><h3> <span class="mw-headline"> Index creation </span></h3> |
| <ul><li> Pre/post actions of an index process (e.g. starting of services, invoke a functionality of another external system) |
| </li></ul> |
| <p>Due to queue usage we did not have a real end "of a indexing process"; how do we solve this? |
| </p> |
| <ul><li> Initial index creation |
| </li><li> Delta indexing |
| </li><li> Continue indexing (start at point XY) |
| </li><li> Stupid append of information (from any origin/source) |
| </li></ul> |
| <a name="Compound_Management"></a><h3> <span class="mw-headline"> Compound Management </span></h3> |
| <ul><li> Processing via BPEL or via a sole pipelet (which approach is better?) |
| </li><li> How do we cover filters (is it possible to design a relationship between IRM and filter configuration \[P2 for this remark\]) |
| </li><li> Warning: Large streams |
| </li><li> Recursion |
| </li><li> Delta indexing |
| </li><li> Extensibility of compound management (e.g. using extensions points) |
| </li><li> Ability for debugging |
| </li><li> Project templates for covering best practices |
| </li><li> Inheritance of data to child records (e.g. user rights) |
| </li><li> MIME Type detection |
| </li><li> |
| </li><li> In den unterschiedlichen Ausprägungen der Installationen |
| </li></ul> |
| <a name="Maintenance_Operations"></a><h3> <span class="mw-headline"> Maintenance Operations </span></h3> |
| <ul><li> CRUD (e.g. collections, indexes) |
| </li><li> Backup/Restore/Reset (remove all process related data; |
| </li></ul> |
| <p>Empty temp storage for delta indexing; empty collection XY) |
| </p> |
| <ul><li> Backup/Restore/Reset (? Maintenance concept; Probably hosted ad eccenca) |
| </li><li> Migration of software versions (including the data) |
| </li><li> Reorganization, save/security points, training (e.g. for search, what's related) |
| </li><li> Adding of nodes (indices, SMILA, ...) |
| </li><li> Creation of reports or statistics |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 11/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:15295-0!1!0!!en!2!edit=0 and timestamp 20120203101455 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="Core_Indexing_Process_(global_view).html">http://wiki.eclipse.org/SMILA/Project_Concepts/Core_Indexing_Process_%28global_view%29</a>"</div> |
| <div id="catlinks"><p class='catlinks'><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Category</a>: <span dir='ltr'><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></span></p></div> <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2012 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 08:19, 15 August 2008 by <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>. </p> |
| <p id="footerviews">This page has been accessed 2,765 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.124 secs. --></body></html> |
