| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Project Concepts/IRM,IRMDiscussion,SMILA/Project Concepts/CompoundManagement,SMILA/Project Concepts/Connectivity,SMILA/Project Concepts/Deployment&Implementation Concept for the IRM Framework,SMILA/Project Concepts/IRM Framework UML Description,SMILA/Project Concepts/Index Order Configuration Schema,Daniel.stucky.empolis.com,Juergen.schumacher.empolis.com" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Project_Concepts/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Project Concepts/IRM - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Project_Concepts/IRM"; |
| var wgTitle = "SMILA/Project Concepts/IRM"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "15209"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "285056"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <style type="text/css">/*<![CDATA[*/ |
| .source-java {line-height: normal; font-size: medium;} |
| .source-java li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for java |
| * CSS class: source-java, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-java .de1, .source-java .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-java {} |
| .source-java .head {} |
| .source-java .foot {} |
| .source-java .imp {font-weight: bold; color: red;} |
| .source-java .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-java li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-java li.li2 {font-weight: bold;} |
| .source-java .kw1 {color: #7F0055; font-weight: bold;} |
| .source-java .kw2 {color: #7F0055; font-weight: bold;} |
| .source-java .kw3 {color: #000000; font-weight: normal} |
| .source-java .kw4 {color: #7F0055; font-weight: bold;} |
| .source-java .co1 {color: #3F7F5F; font-style: italic;} |
| .source-java .co2 {color: #3F7F5F;} |
| .source-java .co3 {color: #3F7F5F; font-style: italic; font-weight: bold;} |
| .source-java .coMULTI {color: #3F5FBF; font-style: italic;} |
| .source-java .es0 {color: #000000;} |
| .source-java .br0 {color: #000000;} |
| .source-java .st0 {color: #2A00ff;} |
| .source-java .nu0 {color: #000000;} |
| .source-java .me1 {color: #000000;} |
| .source-java .me2 {color: #000000;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><style type="text/css">/*<![CDATA[*/ |
| .source-xml {line-height: normal; font-size: medium;} |
| .source-xml li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for xml |
| * CSS class: source-xml, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-xml .de1, .source-xml .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-xml {} |
| .source-xml .head {} |
| .source-xml .foot {} |
| .source-xml .imp {font-weight: bold; color: red;} |
| .source-xml .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-xml li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-xml li.li2 {font-weight: bold;} |
| .source-xml .coMULTI {color: #808080; font-style: italic;} |
| .source-xml .es0 {color: #000099; font-weight: bold;} |
| .source-xml .br0 {color: #66cc66;} |
| .source-xml .st0 {color: #ff0000;} |
| .source-xml .nu0 {color: #cc66cc;} |
| .source-xml .sc0 {color: #00bbdd;} |
| .source-xml .sc1 {color: #ddbb00;} |
| .source-xml .sc2 {color: #339933;} |
| .source-xml .sc3 {color: #009900;} |
| .source-xml .re0 {color: #000066;} |
| .source-xml .re1 {font-weight: bold; color: black;} |
| .source-xml .re2 {font-weight: bold; color: black;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><link rel="stylesheet" type="text/css" href="IRM.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Project_Concepts_IRM"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Project_Concepts/IRM">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Project_Concepts/IRM">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Project_Concepts/IRM">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/IRM&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/IRM&oldid=285056">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="IRM.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Project_Concepts/IRM&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/IRM&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Project_Concepts/IRM&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Project%20Concepts/IRM"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Project Concepts/IRM</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Project_Concepts.1.html" title="SMILA/Project Concepts">Project Concepts</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="IRM.html#column-one">navigation</a>, <a href="IRM.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="IRM.html#Description"><span class="tocnumber">1</span> <span class="toctext">Description</span></a></li> |
| <li class="toclevel-1"><a href="IRM.html#Discussion"><span class="tocnumber">2</span> <span class="toctext">Discussion</span></a></li> |
| <li class="toclevel-1"><a href="IRM.html#Technical_proposal"><span class="tocnumber">3</span> <span class="toctext">Technical proposal</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="IRM.html#Technologies"><span class="tocnumber">3.1</span> <span class="toctext">Technologies</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#Architecture"><span class="tocnumber">3.2</span> <span class="toctext">Architecture</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="IRM.html#Agent_Controller"><span class="tocnumber">3.2.1</span> <span class="toctext">Agent Controller</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Agent"><span class="tocnumber">3.2.2</span> <span class="toctext">Agent</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Crawler_Controller"><span class="tocnumber">3.2.3</span> <span class="toctext">Crawler Controller</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Crawler"><span class="tocnumber">3.2.4</span> <span class="toctext">Crawler</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Compound_Management"><span class="tocnumber">3.2.5</span> <span class="toctext">Compound Management</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Connectivity_Module"><span class="tocnumber">3.2.6</span> <span class="toctext">Connectivity Module</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Delta_Indexing_Manager"><span class="tocnumber">3.2.7</span> <span class="toctext">Delta Indexing Manager</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Configuration_Management"><span class="tocnumber">3.2.8</span> <span class="toctext">Configuration Management</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Notes"><span class="tocnumber">3.2.9</span> <span class="toctext">Notes</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-2"><a href="IRM.html#Business_Interfaces"><span class="tocnumber">3.3</span> <span class="toctext">Business Interfaces</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="IRM.html#Agent_Controller_2"><span class="tocnumber">3.3.1</span> <span class="toctext">Agent Controller</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#CrawlerFactory_.28obsolete.29"><span class="tocnumber">3.3.2</span> <span class="toctext">CrawlerFactory (obsolete)</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Crawler_2"><span class="tocnumber">3.3.3</span> <span class="toctext">Crawler</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#CompoundManagement"><span class="tocnumber">3.3.4</span> <span class="toctext">CompoundManagement</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Connectivity"><span class="tocnumber">3.3.5</span> <span class="toctext">Connectivity</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Delta_Indexing_Manager_2"><span class="tocnumber">3.3.6</span> <span class="toctext">Delta Indexing Manager</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Notes_2"><span class="tocnumber">3.3.7</span> <span class="toctext">Notes</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#How_to_implement_the_interfaces"><span class="tocnumber">3.3.8</span> <span class="toctext">How to implement the interfaces</span></a></li> |
| <li class="toclevel-3"><a href="IRM.html#Performance_Evaluation"><span class="tocnumber">3.3.9</span> <span class="toctext">Performance Evaluation</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-2"><a href="IRM.html#CrawlerController_Processing_Logic"><span class="tocnumber">3.4</span> <span class="toctext">CrawlerController Processing Logic</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#AgentController_Processing_Logic"><span class="tocnumber">3.5</span> <span class="toctext">AgentController Processing Logic</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#Process_Compound_Logic"><span class="tocnumber">3.6</span> <span class="toctext">Process Compound Logic</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#Data_Unification_.28unify_data.29"><span class="tocnumber">3.7</span> <span class="toctext">Data Unification (unify data)</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#Configuration"><span class="tocnumber">3.8</span> <span class="toctext">Configuration</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#Combining_Agents_and_Crawlers"><span class="tocnumber">3.9</span> <span class="toctext">Combining Agents and Crawlers</span></a></li> |
| <li class="toclevel-2"><a href="IRM.html#miscellaneous"><span class="tocnumber">3.10</span> <span class="toctext">miscellaneous</span></a> |
| <ul> |
| <li class="toclevel-3"><a href="IRM.html#Deployment_.26_Implementation_concept_.28OSGI-Bundles.2C_.28Remote.29_Communcation.2C_Data_Flow.29"><span class="tocnumber">3.10.1</span> <span class="toctext">Deployment & Implementation concept (OSGI-Bundles, (Remote) Communcation, Data Flow)</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Description"></a><h2> <span class="mw-headline"> Description </span></h2> |
| <p>Work out the concept for IRM interface. Cover the aspects of Agents/Crawlers and partly Connectivity in it. |
| </p> |
| <a name="Discussion"></a><h2> <span class="mw-headline"> Discussion </span></h2> |
| <p>see <a href="http://wiki.eclipse.org/IRMDiscussion" class="mw-redirect" title="IRMDiscussion">IRMDiscussion</a> |
| </p><p><br /> |
| </p> |
| <a name="Technical_proposal"></a><h2> <span class="mw-headline"> Technical proposal </span></h2> |
| <p>The basic idea is to provide a framework to easily integrate data from external systems via Agents and Crawlers. The processing logic of the data is implemented once in so called Controllers, which use additional functionality provided by other components. To integrate a new external data source only a new Agent or Crawler has to be implemented. Implementations for Agents/Crawlers are NOT restricted to Java only\! |
| </p> |
| <a name="Technologies"></a><h3> <span class="mw-headline"> Technologies </span></h3> |
| <p>Here is an overview of the suggested technologies. Other technologies might be suitable as well and a switch should have only minor impacts on the concepts described below. |
| </p> |
| <a name="Architecture"></a><h3> <span class="mw-headline"> Architecture </span></h3> |
| <p>The chart shows the architecture of the IRM framework with it's plugable components (Agents/Crawlers) and relationship to the SMILA entry point Connectivity Module. |
| <a href="http://wiki.eclipse.org/Image:Architecture.png" class="image" title="Image:architecture.png"><img alt="Image:architecture.png" src="http://wiki.eclipse.org/images/3/37/Architecture.png" width="960" height="720" border="0" /></a> |
| </p><p>The IRM Framework is provided and implemented by SMILA. Agents/Crawlers can be integrated easily by implementing the defined interfaces. An advanced implementation might support even both interfaces. |
| </p> |
| <a name="Agent_Controller"></a><h4> <span class="mw-headline"> Agent Controller </span></h4> |
| <p>The Agent Controller implements the general processing logic common for all Agents. It's service interface is used by Agents to execute a add/update/delete action. It has references to |
| </p> |
| <ul><li> ConfigurationManagement: to get Configurations for itself and Agents |
| </li><li> Connectivity: as an entry point for the data for later processing by for example BPEL |
| </li><li> CompoundManagement: to delegate processing of compound documents to |
| </li><li> Delta Indexing Manager: this is only needed in conjunction with CompoundManagement, regular documents need not be checked. For regular documents an Agent knows what action to perform (add/update/delete) with the data. But elements of compound objects should be updated incrementally (e.g. a zip file is changed in the file system. With the delta indexing logic only those elements that actually changed are added/updated/deleted) |
| <ul><li> <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a> : we agreed that DeltaIndexingManager should be used for all add/update requests by Agents |
| </li></ul> |
| </li></ul> |
| <a name="Agent"></a><h4> <span class="mw-headline"> Agent </span></h4> |
| <p>Agents monitor a data source for changes (add/update/delete) or are triggered by events (e.g. trigger in databases). |
| </p> |
| <a name="Crawler_Controller"></a><h4> <span class="mw-headline"> Crawler Controller </span></h4> |
| <p>The Crawler Controller implements the general processing logic common for all Crawlers. It has no service interface. All needed functionality should be addressed by a configuration/monitoring interface. It has references to |
| </p> |
| <ul><li> ConfigurationManagement: to get Configurations for itself and Crawlers |
| </li><li> Connectivity: as an entry point for the data for later processing by for example BPEL |
| </li><li> CompoundManagement: to delegate processing of compound documents to |
| </li><li> Delta Indexing Manager: to determine if a crawled object needs to be processed (because it is new or was modified) or can be skipped. |
| </li></ul> |
| <a name="Crawler"></a><h4> <span class="mw-headline"> Crawler </span></h4> |
| <p>A Crawler actively crawls a DataSource and provides access to collected data. |
| {info:title=Useful Information} |
| Premium Crawlers (provided by brox or empolis) should optimize their performance by using Producer/Consumer pattern internally. Where one thread gathers the information and stores it in an internal queue and the Crawler gets the data from the queue. Of course it depends on the data source if this pattern is applicable. At least for Filesystem and Web it works. |
| {info} |
| \\ |
| </p> |
| <a name="Compound_Management"></a><h4> <span class="mw-headline"> Compound Management </span></h4> |
| <p>Handles processing of compound objects (e.g. zip, chm, ...). It does NOT implement the "Process Compound Logic" described below\! This has to be done by the Controllers. |
| See <a href="CompoundManagement.html" title="SMILA/Project Concepts/CompoundManagement">SMILA/Project_Concepts/CompoundManagement</a> for details. |
| </p> |
| <a name="Connectivity_Module"></a><h4> <span class="mw-headline"> Connectivity Module </span></h4> |
| <p>Stores content and metadata for later processing and generates Queue entry (best case: Queue only contains an ID) |
| See <a href="Connectivity.html" title="SMILA/Project Concepts/Connectivity">SMILA/Project_Concepts/Connectivity</a> for details. |
| </p> |
| <a name="Delta_Indexing_Manager"></a><h4> <span class="mw-headline"> Delta Indexing Manager </span></h4> |
| <p>This is a Sub-Component of the Connectivity Module. Stores information about last modification of each document (even compound elements) and can determine if document has changed. The information about last modification could be some kind of HashToken. Each Crawler and the CompoundManagement should have it's own configurable way of generating such a token. For Filesystem it may be computed from last modification date and security information. For a database it may be computed over some columns. Some of it's functionality is exposed trough the Connectivity Module's API. See <a href="Connectivity.html#Delta_Indexing_Manager" title="SMILA/Project Concepts/Connectivity">SMILA/Project_Concepts/Connectivity#Delta Indexing Manager</a> for details. |
| </p> |
| <a name="Configuration_Management"></a><h4> <span class="mw-headline"> Configuration Management </span></h4> |
| <p>This not part of this specification. It is assumed that this component manages configurations for all kinds of services, e.g. DataSources for crawlers. Instead of the Controllers, each Agent/Crawler could have a reference to the Configuration Management. We separate business configuration (what a Agent/Crawler does) from the setup/deployment configuration (e.g. what Agents/Crawlers are connected to an according controller). |
| A concept for the business configuration is here: <a href="Index_Order_Configuration_Schema.html" title="SMILA/Project Concepts/Index Order Configuration Schema">SMILA/Project_Concepts/Index Order Configuration Schema</a>. |
| </p> |
| <a name="Notes"></a><h4> <span class="mw-headline"> Notes </span></h4> |
| <ul><li> {color:red}Most components must provide a configuration and monitoring interface (at least Agent\- and CrawlerController), which should be based on snmp (which in turn could be based on JMX for java components).{color} |
| </li><li> We need some kind of MimeType detection to decide when to do compound processing. This could but does not have to be a separate component |
| </li><li> we have to provide a mechanism to restart aborted crawls on certain entry points. Such an entry point may be the last known successful queued element of a data source (ignoring elements of compound objects). What this entry points means to the Crawler is up to the Crawler. This is in addition to the logic provided by the Delta Indexing Manager, as it reduces runtime (especially interesting for mostly static content). Of course not all Crawlers / Data Sources may allow selection of an entry point. Therefore this logic should be implemented directly in the Crawler if possible and configured. No Crawler is foreced to implement this logic\! Changes on already indexed elemnts of a previous run are NOT considered\!. ThisThis feature could be realized by simply segmenting the data into small parts (e.g. not one root directory but multiple sub-sub directories) trough configuration alone, here are some basic implementation ideas: |
| <ul><li> FileSystem: In this case the entry point would a file (absolute path). I assume that the elemnts of a directory are always ordered. So the crawling starts on the parent directory of this entry point file. It iterates trough it (ignoring all elements it finds) until it finds the entry point file. From now on all elemnts are returned by the crawler. So far all elemnts below the entry point can be found. However, there may be elements in the entry points parent folder sister folders. So this procedure has to be executed up the tree structure until the base directory is reached. |
| </li><li> Database: The only prerequisite is that the database returns the results of a query always in the same order, either a natural order provided by the database or an ORDER BY in the SQL statement. Then the entry point can be any unique id or key. Accessing the entry point can be done in the SQL statement adding a statement restricting the result to all ids >= "entry point" |
| </li><li> Web: this is much more complicated, as the links in HTML files are not bi-directional. |
| </li></ul> |
| </li></ul> |
| <a name="Business_Interfaces"></a><h3> <span class="mw-headline"> Business Interfaces </span></h3> |
| <ul><li> ConfigID: the ID of a configuration for an Agent or Crawler Job |
| </li><li> Config: a configuration for a Agent/Crawler |
| </li><li> DIRecord: a data structure containing only DeltaIndexing information of a single object provided by an agent/crawler |
| </li><li> Record: a data structure containing all data of a single object provided by an agent/crawler |
| </li></ul> |
| <a name="Agent_Controller_2"></a><h4> <span class="mw-headline"> Agent Controller </span></h4> |
| <div dir="ltr" style="text-align: left;"><pre class="source-java"><span class="kw1">interface</span> AgentController |
| <span class="br0">{</span> |
| <span class="kw4">void</span> add<span class="br0">(</span>Record<span class="br0">)</span> <span class="co1">// triggers the add process</span> |
| <span class="kw4">void</span> update<span class="br0">(</span>Record<span class="br0">)</span> <span class="co1">// triggers the update process</span> |
| <span class="kw4">void</span> delete<span class="br0">(</span>Record<span class="br0">)</span> <span class="co1">// triggers the delete process, the Record most likely will only contain the ID and no data</span> |
| <span class="br0">}</span></pre></div> |
| <p><br /> |
| </p> |
| <a name="CrawlerFactory_.28obsolete.29"></a><h4> <span class="mw-headline"> CrawlerFactory (obsolete) </span></h4> |
| <div dir="ltr" style="text-align: left;"><pre class="source-java"><span class="kw1">interface</span> CrawlerFactory |
| <span class="br0">{</span> |
| Crawler createCrawler<span class="br0">(</span>Config<span class="br0">)</span> |
| <span class="br0">}</span></pre></div> |
| <p>Creates and new Crawler object with the given Configuration and returns it. |
| </p> |
| <a name="Crawler_2"></a><h4> <span class="mw-headline"> Crawler </span></h4> |
| <div dir="ltr" style="text-align: left;"><pre class="source-java"><span class="kw1">interface</span> CrawlerController |
| <span class="br0">{</span> |
| <span class="coMULTI">/** |
| * Returns an array of DIRecord objects (the maximum size of the array is determined by configuration or Crawler implementation) or null, if no more DIRecords exist |
| */</span> |
| DIRecord<span class="br0">[</span><span class="br0">]</span> getNextDIRecords<span class="br0">(</span><span class="br0">)</span>; |
| |
| <span class="coMULTI">/** |
| * Returns a Record object. The parameter pos refers to the position of the DIRecord from the DIRecord[] returned by getNextDIRecords(). |
| */</span> |
| Record getRecord<span class="br0">(</span><span class="kw4">int</span> pos<span class="br0">)</span>; |
| |
| <span class="coMULTI">/** |
| * Management method used to initialize a Crawler |
| */</span> |
| <span class="kw4">void</span> initialize<span class="br0">(</span>Config config<span class="br0">)</span>; |
| |
| <span class="coMULTI">/** |
| * Management method used to close Conversations |
| */</span> |
| <span class="kw4">void</span> close<span class="br0">(</span><span class="br0">)</span>; |
| <span class="br0">}</span></pre></div> |
| <p>The Crawler's method getNextDIRecords() returns arrays of DIRecord objects. If no more DIRecord objects exist it returns null to signal end of "iteration". The maximum size of the returned DIRecord[] should be configurable. Some Crawlers may only allow for size=1, because of implementation limitations. |
| For each DIRecord returned, the CrawlerController applies DeltaIndexingLogic. Only those DIRecord objects that are new or have changed are requested by method getRecord() as full Record objects, using the position in the DIRecord[] to identify the Record. This is used like a frame on current DIRecords. If getNextDIRecords() is called again, the frame moves on to the next elements. |
| </p> |
| <a name="CompoundManagement"></a><h4> <span class="mw-headline"> CompoundManagement </span></h4> |
| <p>see <a href="CompoundManagement.html#Interfaces" title="SMILA/Project Concepts/CompoundManagement">SMILA/Project_Concepts/CompoundManagement#Interfaces</a> for details. |
| </p> |
| <a name="Connectivity"></a><h4> <span class="mw-headline"> Connectivity </span></h4> |
| <p>see <a href="Connectivity.html#Interfaces" title="SMILA/Project Concepts/Connectivity">SMILA/Project_Concepts/Connectivity#Interfaces</a> for details. |
| </p> |
| <a name="Delta_Indexing_Manager_2"></a><h4> <span class="mw-headline"> Delta Indexing Manager </span></h4> |
| <p>see <a href="Connectivity.html#Interfaces" title="SMILA/Project Concepts/Connectivity">SMILA/Project_Concepts/Connectivity#Interfaces</a> for details. |
| </p> |
| <a name="Notes_2"></a><h4> <span class="mw-headline"> Notes </span></h4> |
| <ul><li> Monitoring functionality (e.g. what crawl jobs are running on a crawler and what is their state) is needed for each Component, but is not part of the SCA interface |
| </li><li> Only one crawl job per ConfigID must run at the same time \! If the crawl job finishes without fatal errors in inc update mode, then the list of obsolete IDs is deleted via according Data objects. |
| </li><li> More meaningful terms for Classes, Methods and Parameters are welcome. |
| </li></ul> |
| <a name="How_to_implement_the_interfaces"></a><h4> <span class="mw-headline"> How to implement the interfaces </span></h4> |
| <p>The idea is that integration developers only have to implement Agents and Crawlers. The Agent\- and Crawler Controllers should be provided by the framework, prefferably developed in Java. Initial implementations for Components like Compound Processing and Delta Indexing Manager should also be provided by the framework, but it should be possible to replace them by other implementations. |
| So most of the time developers will create new Agents and/or Crawlers. These can be implemented in any programming language that is supported by an SCA implementation. An implementation could support both Agent and Crawler interfaces, but should never be forced to implement both\! This makes development of a Agent/Crawler for a specific purpose much more easily. |
| </p><p>Different implementations of the Delta Indexing Manager may be interesting in regards to performance and storage of this information, e.g.it could be stored directly in the search index. |
| </p><p><br /> |
| </p> |
| <a name="Performance_Evaluation"></a><h4> <span class="mw-headline"> Performance Evaluation </span></h4> |
| <p>Regarding the discussions about Crawler (formerly Iterator) interface design, I setup some performance tests to compare a classic Iterator pattern (1 object at a time) against the proposal to use a list of objects. |
| I tested it with a FilesystemCrawler that iterates on Delta-Indexing information only (an ID and a hash). The implementation uses a blocking Queue with a maximum number of elements (capacity). The Crawler itself returns a list of objects with a configurable maximum number of elements (step). |
| </p><p>The test iterates over a directory containing 10001 html files (it does not open the file content, it just reads the path and last modification date). Using plain java objects (no SCA) it takes about 600 ms, no real difference is measurable. But we want to be able to distribute the software components. Therefore I also used SCA in the tests. SCA incures some overhead, as proxy objects are used with any binding mechanism. |
| </p> |
| <ul><li> single VM |
| </li></ul> |
| <table class="wikitable" border="1"> |
| |
| <tr> |
| <th> capacity/step |
| </th><th> 1/1 |
| </th><th> 10/1 |
| </th><th> 100/1 |
| </th><th> 10/10 |
| </th><th> 100/10 |
| </th><th> 1000/100 |
| </th></tr> |
| <tr> |
| <th> binding.sca |
| </th><td> 22.7 sec |
| </td><td> 22.7 sec |
| </td><td> 22.7 sec |
| </td><td> 2.8 sec |
| </td><td> 2.6 sec |
| </td><td> 765 ms |
| </td></tr> |
| <tr> |
| <th> binding.rmi |
| </th><td> 26.9 sec |
| </td><td> 26.7 sec |
| </td><td> 26.9 sec |
| </td><td> 3.5 sec |
| </td><td> 3.3 sec |
| </td><td> 1.0 sec |
| </td></tr> |
| <tr> |
| <th> binding.ws |
| </th><td> 3,8 min |
| </td><td> 3,8 min |
| </td><td> 3,8 min |
| </td><td> 2,2 min |
| </td><td> 2,2 min |
| </td><td> 1,9 min |
| </td></tr></table> |
| <ul><li> separate VMs |
| </li></ul> |
| <table class="wikitable" border="1"> |
| |
| <tr> |
| <th> capacity/step |
| </th><th> 1/1 |
| </th><th> 10/1 |
| </th><th> 100/1 |
| </th><th> 10/10 |
| </th><th> 100/10 |
| </th><th> 1000/100 |
| </th></tr> |
| <tr> |
| <th> binding.rmi |
| </th><td> 29.8 sec |
| </td><td> 29.0 sec |
| </td><td> 30.2 sec |
| </td><td> 4.3 sec |
| </td><td> 3.6 sec |
| </td><td> 1.3 sec |
| </td></tr> |
| <tr> |
| <th> binding.ws |
| </th><td> 4,5 min |
| </td><td> 4,5 min |
| </td><td> 4,5 min |
| </td><td> 2,4 min |
| </td><td> 2,4 min |
| </td><td> 2,0 min |
| </td></tr></table> |
| <ul><li> remote machines (KL-GT) |
| </li></ul> |
| <table class="wikitable" border="1"> |
| |
| <tr> |
| <th> capacity/step |
| </th><th> 1/1 |
| </th><th> 10/1 |
| </th><th> 100/1 |
| </th><th> 10/10 |
| </th><th> 100/10 |
| </th><th> 1000/100 |
| </th></tr> |
| <tr> |
| <th> binding.rmi |
| </th><td> 8,9 min |
| </td><td> 9,1 min |
| </td><td> 8,6 min |
| </td><td> 54.5 sec |
| </td><td> 59.0 sec |
| </td><td> 6.8 sec |
| </td></tr> |
| <tr> |
| <th> binding.ws |
| </th><td> 75,4min |
| </td><td> 72,6 min |
| </td><td> 74,7 min |
| </td><td> 7,4 min |
| </td><td> 6,8 min |
| </td><td> 2,4 min |
| </td></tr></table> |
| <p>So, it's quite obvious that with SCA and especially with remote communication there is a big difference between single object iteration and lists. Therefore the Crawler interface should support lists. If a special implementation is not capable of supporting list it is still possible to set step to 1. |
| </p> |
| <a name="CrawlerController_Processing_Logic"></a><h3> <span class="mw-headline"> CrawlerController Processing Logic </span></h3> |
| <p>This chart shows the CrawlerController Processing Logic: |
| <a href="http://wiki.eclipse.org/Image:Crawler_controller_processing_logic.png" class="image" title="Image:crawler_controller_processing_logic.png"><img alt="Image:crawler_controller_processing_logic.png" src="http://wiki.eclipse.org/images/7/7f/Crawler_controller_processing_logic.png" width="960" height="720" border="0" /></a> |
| </p><p>The Process starts when a Carwler was created from a CrawlerFactory and received by the CrawlerController. The logic checks if there is a next data element. If so it checks if this data is new or has changed using the Delta Indexing Manager. If so, then the data is retrieved and unified in a generic representation. Then it is checked if the data is some kind of compound object (e.g. zip) and the Process Compound Logic is triggered. Otherwise the data is sent to the Connectivity Component. For Compound objects it may be configured to (not) send them to the Connectivity Component. |
| </p> |
| <a name="AgentController_Processing_Logic"></a><h3> <span class="mw-headline"> AgentController Processing Logic </span></h3> |
| <p>This chart shows the AgentController Processing Logic: |
| <a href="http://wiki.eclipse.org/Image:Agent_controller-Processing_logic.png" class="image" title="Image:agent_controller-Processing_logic.png"><img alt="Image:agent_controller-Processing_logic.png" src="http://wiki.eclipse.org/images/4/40/Agent_controller-Processing_logic.png" width="960" height="720" border="0" /></a> |
| </p><p>The Agent Controller provides the 3 processes Add, Update and Delete. |
| </p> |
| <ul><li> Add: At first the data provided by the Agent is unified. Then it is checked if the data is a compound object and if it's the case it is processed just as in the Crawler Processing logic. Otherwise\- Connectivity:add() is executed with the data. |
| </li><li> Update: is very similar to Add, the only difference is that at the end Connectivity:update() is called |
| </li><li> Delete: Here just a call to Connectivity:delete() is made. |
| </li></ul> |
| <p>{info:title=Note}There is a special case in Agent Controller Processing Logic during "Update" that is not shown in the chart: After the Process Compound Logic is done, we need to check if there are elements in the index that need to be removed (that were formerly included in this container). So we need functionality to perform deleteDelta() on a compound object and it's subelements in addition to performing it on a whole dataSource. |
| {info} |
| </p> |
| <a name="Process_Compound_Logic"></a><h3> <span class="mw-headline"> Process Compound Logic </span></h3> |
| <p>This chart shows the Process Compound Logic in detail: |
| <a href="http://wiki.eclipse.org/Image:Process_compound.png" class="image" title="Image:process_compound.png"><img alt="Image:process_compound.png" src="http://wiki.eclipse.org/images/8/85/Process_compound.png" width="960" height="720" border="0" /></a> |
| </p><p>This processing logic is NOT implemented in the Component "Compound Processing", this component is only used to extract the data and provide functionality to iterate over it. This logic is implemented in the Agent\- and Crawler Controllers. |
| The Process Compound Logic is very similar to the Crawler Controller Processing Logic. The main difference is that before iteration, the Compound object's elements have to be extracted (using the Compound Prosessing Component). Note that Process Compound calls itself recursively if an element is a Compound object. This is needed, because you may have zip in zip in ... |
| {info:title=Note} |
| The Process Compound Logic uses Delta Indexing Manager to determine if an element has to be added/updated. There is a special case in Agent Controller Processing Logic during "Add": Here the agent knows that the Compound object and all it's elements are new. No check against the Delta Indexing Manager is needed here. Also no check for objects to delete must be performed. This is not shown in the charts, as this is only a minor performance issue that complicates the process. Nevertheless, this logic should be implemented. |
| {info} |
| \\ |
| </p> |
| <a name="Data_Unification_.28unify_data.29"></a><h3> <span class="mw-headline"> Data Unification (unify data) </span></h3> |
| <p>All data from various data sources has to be represented in a unified format. This includes functionality to normalize the data (e.g. one representation for date formats) and a renaming mechanism for data identifiers (mapping of attribute names). Both should be configurable. |
| </p><p>The resulting format could be an XML like structure: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml"><span class="sc3"><span class="re1"><attribute</span> <span class="re0">name</span>=<span class="st0">"myName"</span> <span class="re0">type</span>=<span class="st0">"myDataType"</span><span class="re2">></span></span> |
| <span class="sc3"><span class="re1"><value<span class="re2">></span></span></span>ABC<span class="sc3"><span class="re1"></value<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><value<span class="re2">></span></span></span>XYZ<span class="sc3"><span class="re1"></value<span class="re2">></span></span></span> |
| ... |
| <span class="sc3"><span class="re1"></attribute<span class="re2">></span></span></span></pre></div> |
| <p>Each attribute has a name and a data type (string, int, double, date, boolean, byte\[\]). If an attribute has multiple values this can be represented by multiple <value> tags. In case of data type byte\[\], the <value> tag should NOT contain the bytes but a link to where the bytes are stored. |
| {note:title=Dependencies} |
| This whole process has to be specified in more detail. |
| But the process depends on what data structures the following processes expect and how and where the data is stored. (e.g. XML database) |
| </p><p><br /> |
| <a href="http://wiki.eclipse.org/User:Juergen.schumacher.empolis.com" title="User:Juergen.schumacher.empolis.com">Juergen Schumacher</a> : |
| We need an an XML format for SMILA objects anyway. Probably we should reuse it here instead of inventing a second one. It will have to be a bit more complex, though. |
| {note} |
| \\ |
| </p> |
| <a name="Configuration"></a><h3> <span class="mw-headline"> Configuration </span></h3> |
| <p>Configurations for Agents/Crawlers should share as much as possible and should be extendable for new DataSources. This has to be specified in detail. |
| Here are some ideas what configurations should contain: |
| </p> |
| <ul><li> unique ID |
| </li><li> Filter |
| <ul><li> filename |
| </li><li> path |
| </li><li> mimetype |
| </li><li> filesize |
| </li><li> date |
| </li><li> ... |
| </li></ul> |
| </li><li> entry points to continue aborted jobs |
| </li><li> sleep parameter (to avoid system overload/DOS) |
| </li></ul> |
| <a name="Combining_Agents_and_Crawlers"></a><h3> <span class="mw-headline"> Combining Agents and Crawlers </span></h3> |
| <p>For certain use cases it may be desirable to easily combine Agent and Crawler functionality. |
| E.g, an Agent monitors a folder structure in the filesystem. Because of some network problems the Agent doe not monitor all changes made in the meantime. A user may want the Agent to automatically synchronize the current state of the folder structure with the search index. Thus - the folder structure needs to be crawled using delta indexing. |
| </p><p>Of course the IRM supports this use case, but the logic "what to execute when" has to be provided externally (by a user). |
| Can we provide functionality that allows an Agent to execute a crawl on it's monitored data ? What are the requirements ? |
| </p> |
| <ul><li> Agent and Crawler configurations should be interchangeable or harmonize |
| </li><li> Agents (or AgentController) needs a reference to CrawlerController |
| </li></ul> |
| <a name="miscellaneous"></a><h3> <span class="mw-headline"> miscellaneous </span></h3> |
| <p>The following issues must be addressed and specified in detail: |
| </p> |
| <ul><li> error handling |
| </li><li> retry logic (connection failures, timeouts) |
| </li><li> synchronization: synchronization of accessing external data is desirable. I think it is possible to ensure that a certain Configuration (identified by an ID) is used only once at a time. This should be managed by the controllers. However, I think it's not possible to detect overlapping configurations,e.g. Config1 crawls C:\data, Config2 crawls c:\data\somwhere\else and Config3 watches C:\data\triggerfolder. This may even be more complicated using UNC paths and folder shares, or imagine the same data may be accessible via filesystem and http. |
| </li><li> what about access to single documents by ID (e.g. URI)? Should this be supported by Agents/Crawlers or should this be implemented in a separate service used in BPEL? Use case could be a scenario, where URLs are stored in a database. So a DB-Crawler would provide the URLs but how is the content of the URLs retrieved ? |
| </li></ul> |
| <a name="Deployment_.26_Implementation_concept_.28OSGI-Bundles.2C_.28Remote.29_Communcation.2C_Data_Flow.29"></a><h4> <span class="mw-headline"> Deployment & Implementation concept (OSGI-Bundles, (Remote) Communcation, Data Flow) </span></h4> |
| <p>see <a href="Deployment&Implementation_Concept_for_the_IRM_Framework.html" title="SMILA/Project Concepts/Deployment&Implementation Concept for the IRM Framework">SMILA/Project_Concepts/Deployment&Implementation Concept for the IRM Framework</a> |
| </p><p>see <a href="IRM_Framework_UML_Description.html" title="SMILA/Project Concepts/IRM Framework UML Description">SMILA/Project_Concepts/IRM Framework UML Description</a> |
| </p> |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 45/1000000 |
| Post-expand include size: 0/2097152 bytes |
| Template argument size: 0/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:15209-0!1!0!!en!2!edit=0 and timestamp 20120203054223 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="IRM.html">http://wiki.eclipse.org/SMILA/Project_Concepts/IRM</a>"</div> |
| <div id="catlinks"><p class='catlinks'><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Category</a>: <span dir='ltr'><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></span></p></div> <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2012 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 09:54, 19 January 2012 by <a href="http://wiki.eclipse.org/index.php?title=User:Nadine.auslaender.attensity.com&action=edit" class="new" title="User:Nadine.auslaender.attensity.com"> </a>. Based on work by <a href="http://wiki.eclipse.org/index.php?title=User:Rrausch.brox.de&action=edit" class="new" title="User:Rrausch.brox.de">Ralf Rausch</a> and <a href="http://wiki.eclipse.org/User:Daniel.stucky.empolis.com" title="User:Daniel.stucky.empolis.com">Daniel Stucky</a>.</p> |
| <p id="footerviews">This page has been accessed 2,663 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.068 secs. --></body></html> |
