| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Documentation/5 more minutes to change the workflow,SMILA/Documentation/BPEL Workflow Processor,SMILA/Documentation/JobManager,SMILA/Documentation/Processing/JSON REST API for BPEL pipelines,SMILA/Documentation/Solr,SMILA/Documentation/Worker/PipelineProcessorWorker,SMILA/Documentation for 5 Minutes to Success" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Documentation/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Documentation/5 more minutes to change the workflow - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Documentation/5_more_minutes_to_change_the_workflow"; |
| var wgTitle = "SMILA/Documentation/5 more minutes to change the workflow"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "35503"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "286060"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <style type="text/css">/*<![CDATA[*/ |
| .source-xml {line-height: normal; font-size: medium;} |
| .source-xml li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for xml |
| * CSS class: source-xml, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-xml .de1, .source-xml .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-xml {} |
| .source-xml .head {} |
| .source-xml .foot {} |
| .source-xml .imp {font-weight: bold; color: red;} |
| .source-xml .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-xml li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-xml li.li2 {font-weight: bold;} |
| .source-xml .coMULTI {color: #808080; font-style: italic;} |
| .source-xml .es0 {color: #000099; font-weight: bold;} |
| .source-xml .br0 {color: #66cc66;} |
| .source-xml .st0 {color: #ff0000;} |
| .source-xml .nu0 {color: #cc66cc;} |
| .source-xml .sc0 {color: #00bbdd;} |
| .source-xml .sc1 {color: #ddbb00;} |
| .source-xml .sc2 {color: #339933;} |
| .source-xml .sc3 {color: #009900;} |
| .source-xml .re0 {color: #000066;} |
| .source-xml .re1 {font-weight: bold; color: black;} |
| .source-xml .re2 {font-weight: bold; color: black;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><link rel="stylesheet" type="text/css" href="5_more_minutes_to_change_the_workflow.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Documentation_5_more_minutes_to_change_the_workflow"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/5_more_minutes_to_change_the_workflow">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Documentation/5_more_minutes_to_change_the_workflow">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Documentation/5_more_minutes_to_change_the_workflow">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/5_more_minutes_to_change_the_workflow&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/5_more_minutes_to_change_the_workflow&oldid=286060">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="5_more_minutes_to_change_the_workflow.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Documentation/5_more_minutes_to_change_the_workflow&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/5_more_minutes_to_change_the_workflow&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/5_more_minutes_to_change_the_workflow&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/5%20more%20minutes%20to%20change%20the%20workflow"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Documentation/5 more minutes to change the workflow</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../SMILA.html" title="SMILA">SMILA</a> | <a href="../Documentation.1.html" title="SMILA/Documentation">Documentation</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="5_more_minutes_to_change_the_workflow.html#column-one">navigation</a>, <a href="5_more_minutes_to_change_the_workflow.html#searchInput">search</a></div> <!-- start content --> |
| <p> |
| </p> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="5_more_minutes_to_change_the_workflow.html#Just_another_5_minutes_to_change_the_workflow"><span class="tocnumber">1</span> <span class="toctext">Just another 5 minutes to change the workflow</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="5_more_minutes_to_change_the_workflow.html#Configure_new_solr_index"><span class="tocnumber">1.1</span> <span class="toctext">Configure new solr index</span></a></li> |
| <li class="toclevel-2"><a href="5_more_minutes_to_change_the_workflow.html#Create_a_new_BPEL_pipeline"><span class="tocnumber">1.2</span> <span class="toctext">Create a new BPEL pipeline</span></a></li> |
| <li class="toclevel-2"><a href="5_more_minutes_to_change_the_workflow.html#Create_and_start_a_new_indexing_job"><span class="tocnumber">1.3</span> <span class="toctext">Create and start a new indexing job</span></a></li> |
| <li class="toclevel-2"><a href="5_more_minutes_to_change_the_workflow.html#Update_the_web_crawl_job"><span class="tocnumber">1.4</span> <span class="toctext">Update the web crawl job</span></a></li> |
| <li class="toclevel-2"><a href="5_more_minutes_to_change_the_workflow.html#Put_it__all_together"><span class="tocnumber">1.5</span> <span class="toctext">Put it all together</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="5_more_minutes_to_change_the_workflow.html#Configuration_overview"><span class="tocnumber">2</span> <span class="toctext">Configuration overview</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Just_another_5_minutes_to_change_the_workflow"></a><h1> <span class="mw-headline"> Just another 5 minutes to change the workflow </span></h1> |
| <p>In the <a href="../Documentation_for_5_Minutes_to_Success.html" title="SMILA/Documentation for 5 Minutes to Success">5 minutes to success</a> all data collected by crawlers was processed with the same asynchronous "indexUpdate" workflow using the BPEL pipeline "AddPipeline". All data was indexed into the same solr/lucene index "DefaultCore". |
| It is possible, however, to configure SMILA so that data from different data sources will go through different workflows and pipelines and will be indexed into different indices. This will require more advanced configuration features than before but still quite simple ones. |
| </p><p>In the following sections we are going to use the generic asynchronous "importToPipeline" workflow which let you specify the BPEL pipeline to process the data. We create an additional BPEL pipeline for webcrawler records so that webcrawler data will be indexed into a separate index named "WebCore". |
| </p> |
| <a name="Configure_new_solr_index"></a><h2> <span class="mw-headline"> Configure new solr index </span></h2> |
| <table width="100%" style="background-color:#d8e4f1; padding-left:30px;"> |
| <tr> |
| <td> |
| <p>It's very important to shutdown and restart the SMILA engine after the following configuration changes are done because modified configurations are loaded during startup only. |
| </p> |
| </td></tr></table> |
| <p>To configure your own index "WebCore" follow the description in the SMILA documentation for <a href="Solr.html#Setup_another_core" title="SMILA/Documentation/Solr">creating your own solr index</a>. |
| </p> |
| <table width="100%" style="background-color:#d8e4f1; padding-left:30px;"> |
| <tr> |
| <td>If you already started SMILA before (as we suppose you did), please copy your new Core configuration and the modified <tt>solr.xml</tt> file to the folder <tt>workspace\.metadata\.plugins\org.eclipse.smila.solr</tt> because the configuration will not be copied again, after the first start of the Solr bundle. |
| </td></tr></table> |
| <p>For more information about the solr indexing, please see the <a href="Solr.html" title="SMILA/Documentation/Solr">SMILA solr documentation</a>. |
| </p> |
| <a name="Create_a_new_BPEL_pipeline"></a><h2> <span class="mw-headline"> Create a new BPEL pipeline </span></h2> |
| <p>We need to add the <i>AddWebPipeline</i> pipeline to the BPEL WorkflowProcessor. For more information about BPEL WorkflowProcessor please check the <a href="BPEL_Workflow_Processor.html" title="SMILA/Documentation/BPEL Workflow Processor">BPEL WorkflowProcessor</a> documentation. |
| Predefined BPEL WorkflowProcessor configuration files are contained in the <tt>configuration/org.eclipse.smila.processing.bpel/pipelines</tt> directory. However, we can add new BPEL pipelines with the SMILA REST API. |
| </p><p>Start SMILA if it's not yet running, and use your favourite REST client to add the "AddWebPipeline" BPEL pipeline: (the BPEL XML is a little bit unreadable cause we have to escape it for being valid JSON content; after posting the new pipeline you can get a readable version via monitoring REST API - see below) |
| </p> |
| <pre> |
| POST http://localhost:8080/smila/pipeline |
| { |
| "name":"AddWebPipeline", |
| "definition":"<?xml version=\"1.0\" encoding=\"utf-8\" ?>\r\n<process name=\"AddWebPipeline\" targetNamespace=\"http://www.eclipse.org/smila/processor\" xmlns=\"http://docs.oasis-open.org/wsbpel/2.0/process/executable\"\r\n xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:proc=\"http://www.eclipse.org/smila/processor\" xmlns:rec=\"http://www.eclipse.org/smila/record\"\r\n xmlns:bpel=\"http://docs.oasis-open.org/wsbpel/2.0/process/executable\">\r\n\r\n <import location=\"processor.wsdl\" namespace=\"http://www.eclipse.org/smila/processor\" importType=\"http://schemas.xmlsoap.org/wsdl/\" />\r\n\r\n <partnerLinks>\r\n <partnerLink name=\"Pipeline\" partnerLinkType=\"proc:ProcessorPartnerLinkType\" myRole=\"service\" />\r\n <partnerLink name=\"AdaptWebCrawlerWorkerOutput\" partnerLinkType=\"proc:ProcessorPartnerLinkType\" partnerRole=\"service\" />\r\n </partnerLinks>\r\n\r\n <extensions>\r\n <extension namespace=\"http://www.eclipse.org/smila/processor\" mustUnderstand=\"no\" />\r\n </extensions>\r\n\r\n <variables>\r\n <variable name=\"request\" messageType=\"proc:ProcessorMessage\" />\r\n </variables>\r\n\r\n <sequence name=\"AddWebPipeline\">\r\n <receive name=\"start\" partnerLink=\"Pipeline\" portType=\"proc:ProcessorPortType\" operation=\"process\" variable=\"request\"\r\n createInstance=\"yes\" />\r\n\r\n <invoke name=\"adaptWebCrawlerWorkerOutput\" inputVariable=\"request\" partnerLink=\"AdaptWebCrawlerWorkerOutput\"\r\n outputVariable=\"request\" operation=\"process\" portType=\"proc:ProcessorPortType\" />\r\n\r\n <forEach counterName=\"index\" parallel=\"yes\" name=\"iterateRecords\">\r\n <startCounterValue>1</startCounterValue>\r\n <finalCounterValue>count($request.records/rec:Record)</finalCounterValue>\r\n <scope>\r\n <variables>\r\n <variable name=\"identifiedRecord\" messageType=\"proc:ProcessorMessage\" />\r\n </variables>\r\n <sequence>\r\n <if name=\"MimeTypeNotSet\">\r\n <condition>not($request.records/rec:Record[position()=$index]/rec:Val[@key=\"MimeType\"])</condition>\r\n <sequence>\r\n <extensionActivity>\r\n <proc:invokePipelet name=\"detectMimeType\">\r\n <proc:pipelet class=\"org.eclipse.smila.processing.pipelets.MimeTypeIdentifyPipelet\" />\r\n <proc:variables input=\"request\" index=\"index\" output=\"identifiedRecord\" />\r\n <proc:configuration>\r\n <rec:Val key=\"FileExtensionAttribute\">Extension</rec:Val>\r\n <rec:Val key=\"MetaDataAttribute\">MetaData</rec:Val>\r\n <rec:Val key=\"MimeTypeAttribute\">MimeType</rec:Val>\r\n </proc:configuration>\r\n </proc:invokePipelet>\r\n </extensionActivity>\r\n <assign name=\"copy result into original variable for next tests\">\r\n <copy>\r\n <from>$identifiedRecord.records/rec:Record[1]</from>\r\n <to>$request.records/rec:Record[position()=$index]</to>\r\n </copy>\r\n </assign>\r\n </sequence>\r\n </if>\r\n\r\n <!-- only process text based content, skip everything else -->\r\n <if name=\"IsText\">\r\n <condition>starts-with($request.records/rec:Record[position()=$index]/rec:Val[@key=\"MimeType\"],\"text/\")</condition>\r\n <if name=\"IsHTML\">\r\n <condition>$request.records/rec:Record[position()=$index]/rec:Val[@key=\"MimeType\"] = \"text/html\" or\r\n $request.records/rec:Record[position()=$index]/rec:Val[@key=\"MimeType\"] = \"text/xml\"\r\n </condition>\r\n <!-- extract txt from html and xml files -->\r\n <extensionActivity>\r\n <proc:invokePipelet name=\"invokeHtml2Txt\">\r\n <proc:pipelet class=\"org.eclipse.smila.processing.pipelets.HtmlToTextPipelet\" />\r\n <proc:variables input=\"request\" index=\"index\" />\r\n <proc:configuration>\r\n <rec:Val key=\"inputType\">ATTACHMENT</rec:Val>\r\n <rec:Val key=\"outputType\">ATTRIBUTE</rec:Val>\r\n <rec:Val key=\"inputName\">Content</rec:Val>\r\n <rec:Val key=\"outputName\">Content</rec:Val>\r\n <rec:Val key=\"meta:title\">Title</rec:Val>\r\n </proc:configuration>\r\n </proc:invokePipelet>\r\n </extensionActivity>\r\n <else>\r\n <!-- copy txt from attachment to attribute -->\r\n <extensionActivity>\r\n <proc:invokePipelet name=\"invokeCopyContent\">\r\n <proc:pipelet class=\"org.eclipse.smila.processing.pipelets.CopyPipelet\" />\r\n <proc:variables input=\"request\" index=\"index\" />\r\n <proc:configuration>\r\n <rec:Val key=\"inputType\">ATTACHMENT</rec:Val>\r\n <rec:Val key=\"outputType\">ATTRIBUTE</rec:Val>\r\n <rec:Val key=\"inputName\">Content</rec:Val>\r\n <rec:Val key=\"outputName\">Content</rec:Val>\r\n <rec:Val key=\"mode\">COPY</rec:Val>\r\n </proc:configuration>\r\n </proc:invokePipelet>\r\n </extensionActivity>\r\n </else>\r\n </if>\r\n </if>\r\n </sequence>\r\n </scope>\r\n </forEach>\r\n\r\n <extensionActivity>\r\n <proc:invokePipelet name=\"SolrIndexPipelet\">\r\n <proc:pipelet class=\"org.eclipse.smila.solr.index.SolrIndexPipelet\" />\r\n <proc:variables input=\"request\" output=\"request\" />\r\n <proc:configuration>\r\n <rec:Val key=\"ExecutionMode\">ADD</rec:Val>\r\n <rec:Val key=\"CoreName\">WebCore</rec:Val>\r\n <rec:Seq key=\"CoreFields\">\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">_source</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Path</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Url</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Filename</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">MimeType</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Size</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">LastModifiedDate</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Content</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Extension</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Title</rec:Val>\r\n </rec:Map>\r\n <rec:Map>\r\n <rec:Val key=\"FieldName\">Author</rec:Val>\r\n </rec:Map>\r\n </rec:Seq>\r\n </proc:configuration>\r\n </proc:invokePipelet>\r\n </extensionActivity>\r\n\r\n <reply name=\"end\" partnerLink=\"Pipeline\" portType=\"proc:ProcessorPortType\" operation=\"process\" variable=\"request\" />\r\n </sequence>\r\n</process>\r\n" |
| } |
| </pre> |
| <p>You can monitor the defined BPEL pipelines via browser, so you should find your new pipeline there: |
| </p> |
| <pre> |
| http://localhost:8080/smila/pipeline |
| </pre> |
| <p>Note that we used "WebCore" index name for the Solr index in the BPEL above: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-xml">... |
| <span class="sc3"><span class="re1"><proc:configuration<span class="re2">></span></span></span> |
| <span class="sc3"><span class="re1"><rec:Val</span> <span class="re0">key</span>=<span class="st0">"CoreName"</span><span class="re2">></span></span>WebCore<span class="sc3"><span class="re1"></rec:Val<span class="re2">></span></span></span> |
| ... |
| <span class="sc3"><span class="re1"></proc:configuration<span class="re2">></span></span></span> |
| ...</pre></div> |
| <a name="Create_and_start_a_new_indexing_job"></a><h2> <span class="mw-headline"> Create and start a new indexing job </span></h2> |
| <p>We define an indexing job based on the predefined asynchronous workflow "importToPipeline" (see <tt>SMILA/configuration/org.eclipse.smila.jobmanager/workflows.json</tt>). This indexing job will process the imported data by using our new BPEL pipeline "AddWebPipeline". |
| </p><p>The "importToPipeline" workflow contains a <a href="Worker/PipelineProcessorWorker.html" title="SMILA/Documentation/Worker/PipelineProcessorWorker">PipelineProcessorWorker worker</a> which is not configured for dedicated BPEL pipelines, so the BPEL pipelines handling adds and deletes have to be set via job parameter. |
| </p><p>Use your favourite REST Client to create an appropriate job definition: |
| </p> |
| <pre> |
| POST http://localhost:8080/smila/jobmanager/jobs/ |
| { |
| "name":"indexWebJob", |
| "parameters":{ |
| "tempStore": "temp", |
| "addPipeline": "AddWebPipeline", |
| "deletePipeline": "DeletePipeline" |
| }, |
| "workflow":"importToPipeline" |
| } |
| </pre> |
| <p>Note that the "DeletePipeline" is not needed for our test szenario here, but we must fulfill all undefined workflow parameters. |
| </p><p>Afterwards, start a job run for the defined job: |
| </p> |
| <pre> |
| POST http://localhost:8080/smila/jobmanager/jobs/indexWebJob |
| </pre> |
| <a name="Update_the_web_crawl_job"></a><h2> <span class="mw-headline"> Update the web crawl job </span></h2> |
| <p>Since the web crawl job already is predefined to push the crawled records to the <tt>indexUpdate</tt> job, we now either must define a new job or update the crawl job's definition in the <span style="font-family:monospace;">job.json</span> file. Here we choose the <i>new job</i> option. |
| </p><p>POST the following update json using your favorite REST client: |
| </p> |
| <pre> |
| POST http://localhost:8080/smila/jobmanager/jobs/ |
| { |
| "name":"crawlWikiToWebCore", |
| "workflow":"webCrawling", |
| "parameters":{ |
| "tempStore":"temp", |
| "dataSource":"web", |
| "startUrl":"http://wiki.eclipse.org/SMILA", |
| "filter":{ |
| "urlPrefix":"http://wiki.eclipse.org/SMILA" |
| }, |
| "jobToPushTo":"indexWebJob" |
| } |
| } |
| </pre> |
| <p>Please note that we used the following line to let the crawl job push the records to our new job: |
| </p> |
| <pre> |
| "jobToPushTo":"indexWebJob" |
| </pre> |
| <p>Now start the crawl job (don't forget runOnce!): |
| </p> |
| <pre> |
| POST http://localhost:8080/smila/jobmanager/jobs/crawlWikiToWebCore |
| { |
| "mode": "runOnce" |
| } |
| </pre> |
| <p>After a sufficiently long time to crawl, process and commit the data, you can have another look at the <a href="http://localhost:8080/SMILA/search" class="external text" title="http://localhost:8080/SMILA/search" rel="nofollow">SMILA search page</a> to find your new core listed among the available cores, and if you choose it, you can search for e.g. <i>SMILA</i> in the new WebCore. |
| </p> |
| <a name="Put_it__all_together"></a><h2> <span class="mw-headline"> Put it all together </span></h2> |
| <p>Ok, now it seems that we have finally finished configuring SMILA for using separate BPEL pipelines for file system and web crawling and index data from these crawlers into different indices. |
| Here is what we have done so far: |
| </p> |
| <ol><li> We added the <tt>WebCore</tt> index to the Solr configuration and copied it to the workspace. |
| </li><li> We created a new BPEL pipeline for Web crawler data referencing the new Lucene index. |
| </li><li> We used a separate job for web indexing that references the new BPEL pipeline. |
| </li><li> We updated the web crawl job to push the records to a different indexing job which references the new BPEL pipeline. |
| </li></ol> |
| <a name="Configuration_overview"></a><h1> <span class="mw-headline"> Configuration overview </span></h1> |
| <p>SMILA configuration files are located in the <tt>configuration</tt> directory of the SMILA application. |
| The following lists the configuration files and documentation links relevant to this tutorial, regarding SMILA components: |
| </p><p><b>Jobmanager</b> |
| </p> |
| <ul><li> configuration folder: <tt>org.eclipse.smila.jobmanager</tt> |
| <ul><li> <tt>workflows.json</tt> (Predefined asynchronous workflows) |
| </li></ul> |
| </li><li> Documentation |
| <ul><li> <a href="JobManager.html" title="SMILA/Documentation/JobManager">JobManager</a> |
| </li><li> <a href="Worker/PipelineProcessorWorker.html" title="SMILA/Documentation/Worker/PipelineProcessorWorker">PipelineProcessorWorker</a> |
| </li></ul> |
| </li><li> REST API: <a href="http://localhost:8080/smila/jobmanager" class="external free" title="http://localhost:8080/smila/jobmanager" rel="nofollow">http://localhost:8080/smila/jobmanager</a> |
| </li></ul> |
| <p><b>BPEL Pipelines</b> |
| </p> |
| <ul><li> configuration folder: <tt>org.eclipse.smila.processing.bpel</tt> |
| <ul><li> <tt>pipelines/*.bpel</tt> (Predefined BPEL pipelines) |
| </li></ul> |
| </li><li> Documentation |
| <ul><li> <a href="BPEL_Workflow_Processor.html" title="SMILA/Documentation/BPEL Workflow Processor">BPELWorkflowProcessor</a> |
| </li><li> <a href="Processing/JSON_REST_API_for_BPEL_pipelines.html" title="SMILA/Documentation/Processing/JSON REST API for BPEL pipelines">JSON REST API for BPEL pipelines</a> |
| </li></ul> |
| </li><li> REST API: <a href="http://localhost:8080/smila/pipeline" class="external free" title="http://localhost:8080/smila/pipeline" rel="nofollow">http://localhost:8080/smila/pipeline</a> |
| </li></ul> |
| <p><b>Solr</b> |
| </p> |
| <ul><li> DataDictionary |
| <ul><li> configuration folder: <tt>org.eclipse.smila.solr</tt> |
| </li></ul> |
| </li><li> Documentation |
| <ul><li> <a href="Solr.html" title="SMILA/Documentation/Solr">SMILA/Documentation/Solr</a> |
| </li></ul> |
| </li></ul> |
| |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 49/1000000 |
| Post-expand include size: 52/2097152 bytes |
| Template argument size: 8/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:35503-0!1!0!!en!2!edit=0 and timestamp 20120203101128 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="5_more_minutes_to_change_the_workflow.html">http://wiki.eclipse.org/SMILA/Documentation/5_more_minutes_to_change_the_workflow</a>"</div> |
| <div id="catlinks"><p class='catlinks'><a href="http://wiki.eclipse.org/Special:Categories" title="Special:Categories">Categories</a>: <span dir='ltr'><a href="http://wiki.eclipse.org/Category:SMILA" title="Category:SMILA">SMILA</a></span> | <span dir='ltr'><a href="http://wiki.eclipse.org/Category:HowTo" title="Category:HowTo">HowTo</a></span></p></div> <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2012 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 12:23, 24 January 2012 by <a href="http://wiki.eclipse.org/User:Andreas.schank.attensity.com" title="User:Andreas.schank.attensity.com">A. Schank</a>. </p> |
| <p id="footerviews">This page has been accessed 97 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.155 secs. --></body></html> |
