| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
| <meta name="keywords" content="SMILA/Documentation/HowTo/How to setup SMILA in a cluster,SMILA/Documentation/Enable Remote Access,SMILA/Documentation/REST API Reference,SMILA/Documentation/TaskManager" /> |
| <link rel="shortcut icon" href="http://wiki.eclipse.org/SMILA/Documentation/HowTo/favicon.ico" /> |
| <link rel="search" type="application/opensearchdescription+xml" href="http://wiki.eclipse.org/opensearch_desc.php" title="Eclipsepedia (English)" /> |
| <link rel="alternate" type="application/rss+xml" title="Eclipsepedia RSS Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=rss" /> |
| <link rel="alternate" type="application/atom+xml" title="Eclipsepedia Atom Feed" href="http://wiki.eclipse.org/index.php?title=Special:Recentchanges&feed=atom" /> |
| |
| |
| <title>SMILA/Documentation/HowTo/How to setup SMILA in a cluster - Eclipsepedia</title> |
| |
| <style type="text/css" media="screen,projection">/*<![CDATA[*/ @import "http://wiki.eclipse.org/skins/eclipsenova/novaWide.css?116"; /*]]>*/</style> |
| <link rel="stylesheet" type="text/css" media="print" href="http://wiki.eclipse.org/skins/eclipsenova/eclipsenovaPrint.css?116" /> |
| <link rel="stylesheet" type="text/css" media="handheld" href="http://wiki.eclipse.org/skins/eclipsenova/handheld.css?116" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/header.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/tabs.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/visual.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/layout.css" media="screen" /> |
| <link rel="stylesheet" type="text/css" href="http://wiki.eclipse.org/skins/eclipsenova/Nova/css/footer.css" media="screen" /> |
| <!--[if IE]><link rel="stylesheet" type="text/css" href="/skins/eclipsenova/IEpngfix.css" media="screen" /><![endif]--> |
| <!--[if lt IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE50Fixes.css?116";</style> <![endif]--> |
| <!--[if IE 5.5000]><style type="text/css">@import "/skins/eclipsenova/IE55Fixes.css?116";</style><![endif]--> |
| <!--[if IE 6]><style type="text/css">@import "/skins/eclipsenova/IE60Fixes.css?116";</style><![endif]--> |
| <!--[if IE 7]><style type="text/css">@import "/skins/eclipsenova/IE70Fixes.css?116";</style><![endif]--> |
| <!--[if lt IE 7]><script type="text/javascript" src="/skins/common/IEFixes.js?116"></script> |
| <meta http-equiv="imagetoolbar" content="no" /><![endif]--> |
| <script type= "text/javascript">/*<![CDATA[*/ |
| var skin = "eclipsenova"; |
| var stylepath = "/skins"; |
| var wgArticlePath = "/$1"; |
| var wgScriptPath = ""; |
| var wgScript = "/index.php"; |
| var wgServer = "http://wiki.eclipse.org"; |
| var wgCanonicalNamespace = ""; |
| var wgCanonicalSpecialPageName = false; |
| var wgNamespaceNumber = 0; |
| var wgPageName = "SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster"; |
| var wgTitle = "SMILA/Documentation/HowTo/How to setup SMILA in a cluster"; |
| var wgAction = "view"; |
| var wgRestrictionEdit = []; |
| var wgRestrictionMove = []; |
| var wgArticleId = "39273"; |
| var wgIsArticle = true; |
| var wgUserName = null; |
| var wgUserGroups = null; |
| var wgUserLanguage = "en"; |
| var wgContentLanguage = "en"; |
| var wgBreakFrames = false; |
| var wgCurRevisionId = "319466"; |
| var wgVersion = "1.12.0"; |
| var wgEnableAPI = true; |
| var wgEnableWriteAPI = false; |
| /*]]>*/</script> |
| |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/wikibits.js?116"><!-- wikibits js --></script> |
| |
| <!-- Performance mods similar to those for bug 166401 --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/index.php?title=-&action=raw&gen=js&useskin=eclipsenova"><!-- site js --></script> |
| |
| <!-- Head Scripts --> |
| <script type="text/javascript" src="http://wiki.eclipse.org/skins/common/ajax.js?116"></script> |
| <style type="text/css">/*<![CDATA[*/ |
| .source-javascript {line-height: normal; font-size: medium;} |
| .source-javascript li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for javascript |
| * CSS class: source-javascript, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-javascript .de1, .source-javascript .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-javascript {} |
| .source-javascript .head {} |
| .source-javascript .foot {} |
| .source-javascript .imp {font-weight: bold; color: red;} |
| .source-javascript .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-javascript li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-javascript li.li2 {font-weight: bold;} |
| .source-javascript .kw1 {color: #000066; font-weight: bold;} |
| .source-javascript .kw2 {color: #003366; font-weight: bold;} |
| .source-javascript .kw3 {color: #000066;} |
| .source-javascript .co1 {color: #009900; font-style: italic;} |
| .source-javascript .coMULTI {color: #009900; font-style: italic;} |
| .source-javascript .es0 {color: #000099; font-weight: bold;} |
| .source-javascript .br0 {color: #66cc66;} |
| .source-javascript .st0 {color: #3366CC;} |
| .source-javascript .nu0 {color: #CC0000;} |
| .source-javascript .me1 {color: #006600;} |
| .source-javascript .sc0 {} |
| .source-javascript .sc1 {} |
| .source-javascript .sc2 {} |
| .source-javascript .sc3 {} |
| .source-javascript .re0 {color: #0066FF;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "http://wiki.eclipse.org/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><style type="text/css">/*<![CDATA[*/ |
| .source-text {line-height: normal; font-size: medium;} |
| .source-text li {line-height: normal;} |
| /** |
| * GeSHi Dynamically Generated Stylesheet |
| * -------------------------------------- |
| * Dynamically generated stylesheet for text |
| * CSS class: source-text, CSS id: |
| * GeSHi (C) 2004 - 2007 Nigel McNie (http://qbnz.com/highlighter) |
| */ |
| .source-text .de1, .source-text .de2 {font-family: 'Courier New', Courier, monospace; font-weight: normal;} |
| .source-text {} |
| .source-text .head {} |
| .source-text .foot {} |
| .source-text .imp {font-weight: bold; color: red;} |
| .source-text .ln-xtra {color: #cc0; background-color: #ffc;} |
| .source-text li {font-family: 'Courier New', Courier, monospace; color: black; font-weight: normal; font-style: normal;} |
| .source-text li.li2 {font-weight: bold;} |
| |
| /*]]>*/ |
| </style> |
| <style type="text/css">/*<![CDATA[*/ |
| @import "http://wiki.eclipse.org/index.php?title=MediaWiki:Geshi.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"; |
| /*]]>*/ |
| </style><link rel="stylesheet" type="text/css" href="How_to_setup_SMILA_in_a_cluster.html" /> </head> |
| <body class="mediawiki ns-0 ltr page-SMILA_Documentation_HowTo_How_to_setup_SMILA_in_a_cluster"> |
| <div id="globalWrapper"> |
| |
| |
| <div id="column-one"> |
| <!-- Eclipse Additions for the Top Nav start here M. Ward--> |
| |
| <div id="header"> |
| <div id="header-graphic"> |
| <img src="http://wiki.eclipse.org/skins/eclipsenova/eclipse.png" alt="Eclipse Wiki"> |
| </div> |
| <!-- Pulled 101409 Mward --> |
| |
| <div class="portlet" id="p-personal"> |
| <div class="pBody"> |
| <ul> |
| <li id="pt-login"><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster">Log in</a></li> |
| </ul> |
| </div> |
| </div> |
| |
| <div id="header-icons"> |
| <div id="sites"> |
| <ul id="sitesUL"> |
| <li><a href="http://www.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/eclipseIcon.png" width="28" height="28" alt="Eclipse Foundation" title="Eclipse Foundation" /><div>Eclipse Foundation</div></a></li> |
| <li><a href="http://marketplace.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/marketplace.png" width="28" height="28" alt="Eclipse Marketplace" title="Eclipse Marketplace" /><div>Eclipse Marketplace</div></a></li> |
| <li><a href="https://bugs.eclipse.org/bugs"><img src="http://dev.eclipse.org/custom_icons/system-search-bw.png" width="28" height="28" alt="Bugzilla" title="Bugzilla" /><div>Bugzilla</div></a></li> |
| <li><a href="http://live.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/audio-input-microphone-bw.png" width="28" height="28" alt="Live" title="Live" /><div>Eclipse Live</div></a></li> |
| <li><a href="http://planeteclipse.org"><img src="http://dev.eclipse.org/large_icons/devices/audio-card.png" width="28" height="28" alt="PlanetEclipse" title="Planet" /><div>Planet Eclipse</div></a></li> |
| <li><a href="http://portal.eclipse.org"><img src="http://dev.eclipse.org/custom_icons/preferences-system-network-proxy-bw.png" width="28" height="28" alt="Portal" title="Portal" /><div>My Foundation Portal</div></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <!-- NEW HEADER STUFF HERE --> |
| <div id="header-menu"> |
| <div id="header-nav"> |
| <ul> <li><a class="first_one" href="http://wiki.eclipse.org/" target="_self">Home</a></li> <li><a href="http://www.eclipse.org/downloads/" target="_self">Downloads</a></li> |
| <li><a href="http://www.eclipse.org/users/" target="_self">Users</a></li> |
| <li><a href="http://www.eclipse.org/membership/" target="_self">Members</a></li> |
| <li><a href="http://wiki.eclipse.org/index.php/Development_Resources" target="_self">Committers</a></li> |
| <li><a href="http://www.eclipse.org/resources/" target="_self">Resources</a></li> |
| <li><a href="http://www.eclipse.org/projects/" target="_self">Projects</a></li> |
| <li><a href="http://www.eclipse.org/org/" target="_self">About Us</a></li> |
| </ul> |
| </div> |
| <div id="header-utils"> |
| <!-- moved the search window here --> |
| <form action="http://wiki.eclipse.org/Special:Search" > |
| <input class="input" name="search" type="text" accesskey="f" value="" /> |
| <input type='submit' onclick="this.submit();" name="go" id="searchGoButton" class="button" title="Go to a page with this exact name if one exists" value="Go" /> |
| <input type='submit' onclick="this.submit();" name="fulltext" class="button" id="mw-searchButton" title="Search Eclipsepedia for this text" value="Search" /> |
| </form> |
| </div> |
| </div> |
| |
| |
| <!-- Eclipse Additions for the Header stop here --> |
| <!-- Additions and mods for leftside nav Start here --> |
| |
| <!--Started nav rip here--> |
| <!-- these are the nav controls main page, changes etc --> |
| <div id="novaContent" class="faux"> |
| <div id="leftcol"> |
| <ul id="leftnav"> |
| <!-- these are the page controls, edit history etc --> |
| <li class="separator"><a class="separator">Navigation   </li> |
| <li id="n-mainpage"><a href="http://wiki.eclipse.org/Main_Page">Main Page</a></li> |
| <li id="n-portal"><a href="http://wiki.eclipse.org/Eclipsepedia:Community_Portal">Community portal</a></li> |
| <li id="n-currentevents"><a href="http://wiki.eclipse.org/Eclipsepedia:Current_events">Current events</a></li> |
| <li id="n-recentchanges"><a href="http://wiki.eclipse.org/Special:Recentchanges">Recent changes</a></li> |
| <li id="n-randompage"><a href="http://wiki.eclipse.org/Special:Random">Random page</a></li> |
| <li id="n-help"><a href="http://wiki.eclipse.org/Help:Contents">Help</a></li> |
| <li class="separator"><a class="separator">Toolbox   </a></li> |
| |
| <li id="t-whatlinkshere"><a href="http://wiki.eclipse.org/Special:Whatlinkshere/SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster">What links here</a></li> |
| <li id="t-recentchangeslinked"><a href="http://wiki.eclipse.org/Special:Recentchangeslinked/SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster">Related changes</a></li> |
| <!-- This is the toolbox section --> |
| <li id="t-upload"><a href="http://wiki.eclipse.org/Special:Upload">Upload file</a></li> |
| <li id="t-specialpages"><a href="http://wiki.eclipse.org/Special:Specialpages">Special pages</a></li> |
| <li id="t-print"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster&printable=yes">Printable version</a></li> <li id="t-permalink"><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster&oldid=319466">Permanent link</a></li> </ul> |
| </div> |
| |
| |
| <!-- Additions and mods for leftside nav End here --> |
| |
| |
| <div id="column-content"> |
| <div id="content"> |
| <a name="top" id="top"></a> |
| |
| <div id="tabs"> |
| <ul class="primary"> |
| <li class="active"><a href="How_to_setup_SMILA_in_a_cluster.html"><span class="tab">Page</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Talk:SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster&action=edit"><span class="tab">Discussion</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster&action=edit"><span class="tab">View source</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster&action=history"><span class="tab">History</span></a></li> |
| <li><a href="http://wiki.eclipse.org/index.php?title=Special:Userlogin&returnto=SMILA/Documentation/HowTo/How to setup SMILA in a cluster"><span class="tab">Edit</span></a></li> |
| </ul> |
| </div> |
| |
| |
| <script type="text/javascript"> if (window.isMSIE55) fixalpha(); </script> |
| <h1 class="firstHeading">SMILA/Documentation/HowTo/How to setup SMILA in a cluster</h1> |
| <div id="bodyContent"> |
| <h3 id="siteSub">From Eclipsepedia</h3> |
| <div id="contentSub"><span class="subpages">< <a href="../../../SMILA.html" title="SMILA">SMILA</a> | <a href="../../Documentation.1.html" title="SMILA/Documentation">Documentation</a> | <a href="../HowTo.html" title="SMILA/Documentation/HowTo">HowTo</a></span></div> |
| <div id="jump-to-nav">Jump to: <a href="How_to_setup_SMILA_in_a_cluster.html#column-one">navigation</a>, <a href="How_to_setup_SMILA_in_a_cluster.html#searchInput">search</a></div> <!-- start content --> |
| <table id="toc" class="toc" summary="Contents"><tr><td><div id="toctitle"><h2>Contents</h2></div> |
| <ul> |
| <li class="toclevel-1"><a href="How_to_setup_SMILA_in_a_cluster.html#Introduction"><span class="tocnumber">1</span> <span class="toctext">Introduction</span></a></li> |
| <li class="toclevel-1"><a href="How_to_setup_SMILA_in_a_cluster.html#Install_external_Solr_server"><span class="tocnumber">2</span> <span class="toctext">Install external Solr server</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Single_node_server"><span class="tocnumber">2.1</span> <span class="toctext">Single node server</span></a></li> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Distributed_server"><span class="tocnumber">2.2</span> <span class="toctext">Distributed server</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="How_to_setup_SMILA_in_a_cluster.html#Configuring_SMILA_on_cluster_nodes"><span class="tocnumber">3</span> <span class="toctext">Configuring SMILA on cluster nodes</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Cluster_configuration"><span class="tocnumber">3.1</span> <span class="toctext">Cluster configuration</span></a></li> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Objectstore_configuration"><span class="tocnumber">3.2</span> <span class="toctext">Objectstore configuration</span></a></li> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Solr_configuration"><span class="tocnumber">3.3</span> <span class="toctext">Solr configuration</span></a></li> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Jetty_configuration"><span class="tocnumber">3.4</span> <span class="toctext">Jetty configuration</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="How_to_setup_SMILA_in_a_cluster.html#Monitoring"><span class="tocnumber">4</span> <span class="toctext">Monitoring</span></a> |
| <ul> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Startup"><span class="tocnumber">4.1</span> <span class="toctext">Startup</span></a></li> |
| <li class="toclevel-2"><a href="How_to_setup_SMILA_in_a_cluster.html#Running_jobs"><span class="tocnumber">4.2</span> <span class="toctext">Running jobs</span></a></li> |
| </ul> |
| </li> |
| <li class="toclevel-1"><a href="How_to_setup_SMILA_in_a_cluster.html#Some_useful_commands"><span class="tocnumber">5</span> <span class="toctext">Some useful commands</span></a></li> |
| </ul> |
| </li> |
| </ul> |
| </td></tr></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script> |
| <a name="Introduction"></a><h3> <span class="mw-headline"> Introduction </span></h3> |
| <p>SMILA is primarily thought of as a framework where you can plug in your own or third-party high-performant/high-scalable components (e.g. for data storage). Nevertheless, it is also possible to set up SMILA out-of-the-box on a cluster by using its default implementations. This permits <i>horizontal scaling</i> having the effect that importing and processing jobs/tasks will be shared across the cluster nodes. (Remark: We also have a <i>vertical scaling</i> on each cluster machine, but this is not new, because you also have this with a single-node SMILA.) |
| </p><p>The following steps describe how to set up SMILA on multiple cluster nodes. |
| </p> |
| <a name="Install_external_Solr_server"></a><h3> <span class="mw-headline"> Install external Solr server </span></h3> |
| <p>If you want to use Solr for indexing, you need to set up a separate Solr server, because the Solr instances embedded in SMILA cannot be shared with the other SMILA instances. |
| </p> |
| <a name="Single_node_server"></a><h4> <span class="mw-headline"> Single node server </span></h4> |
| <ul><li>Download a Solr 3.x archive from <a href="http://lucene.apache.org/solr/" class="external free" title="http://lucene.apache.org/solr/" rel="nofollow">http://lucene.apache.org/solr/</a>. This HowTo was tested with Solr v. 3.6.1. |
| </li><li>Unpack the archive to a local directory; you will get a directory like <tt>/home/smila/solr/apache-solr-3.6.1</tt>. |
| </li><li>Copy the files from <tt>SMILA/configuration/org.eclipse.smila.solr</tt> to the Solr machine (<tt>solr.properties</tt> isn't needed here), e.g. to <tt>/home/smila/solr/smila-cores</tt>. |
| </li><li>Go to <tt>/home/smila/solr/apache-solr-3.6.1/example</tt> and run: |
| </li></ul> |
| <pre>java -Dsolr.solr.home=/home/smila/solr/smila-cores -jar start.jar |
| </pre> |
| <ul><li>Check if Solr is running at <a href="http://localhost:8983/solr/DefaultCore/admin/" class="external free" title="http://localhost:8983/solr/DefaultCore/admin/" rel="nofollow">http://localhost:8983/solr/DefaultCore/admin/</a> (replace <tt>localhost</tt> with the name of your Solr server, if necessary). |
| </li></ul> |
| <a name="Distributed_server"></a><h4> <span class="mw-headline"> Distributed server </span></h4> |
| <p>For larger data volumes you will need to set up Solr in a distributed way, too. However, using a distributed Solr setup is not yet fully supported by the SMILA integration (especially during indexing). |
| </p> |
| <a name="Configuring_SMILA_on_cluster_nodes"></a><h3> <span class="mw-headline"> Configuring SMILA on cluster nodes </span></h3> |
| <p>On each cluster node, you have to do the following SMILA configuration changes. |
| </p> |
| <a name="Cluster_configuration"></a><h4> <span class="mw-headline"> Cluster configuration </span></h4> |
| <p>You have to define which nodes belong to the cluster. |
| </p> |
| Configuration file: <pre>SMILA/configuration/org.eclipse.smila.clusterconfig.simple/clusterconfig.json</pre> |
| <p>Enter new section "clusterNodes" stating the host names of the individual cluster nodes: |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-javascript"><span class="br0">{</span> |
| <span class="st0">"clusterNodes"</span>: <span class="br0">[</span><span class="st0">"PC-1"</span>, <span class="st0">"PC-2"</span>, <span class="st0">"PC-3"</span><span class="br0">]</span>, |
| <span class="st0">"taskmanager"</span>:<span class="br0">{</span> |
| ... |
| <span class="br0">}</span></pre></div> |
| <a name="Objectstore_configuration"></a><h4> <span class="mw-headline"> Objectstore configuration </span></h4> |
| <p>You have to define a <b>shared</b> data directory for all nodes ("shared" means that the selected directory must be accessbile from every machine in your cluster under the same path). |
| </p> |
| Configuration file: <pre>SMILA/configuration/org.eclipse.smila.objectstore.filesystem/objectstoreservice.properties</pre> |
| <p>(Directory/File will not exist in older SMILA versions - just create it). |
| </p><p>Set a root path to the shared directory: |
| </p><p><code> |
| </p> |
| <pre> root.path=/data/smila/shared |
| ... |
| </pre> |
| <p></code> |
| </p> |
| <div class="messagebox" style="background-color: #def3fe; border: 1px solid #c5d7e0; color: black; padding: 5px; margin: 1ex 0; min-height: 35px; padding-left: 45px;"> |
| <div style="float: left; margin-left: -40px;"><a href="http://wiki.eclipse.org/Image:Idea.png" class="image" title="Idea.png"><img alt="" src="http://wiki.eclipse.org/images/a/a4/Idea.png" width="35" height="35" border="0" /></a></div> |
| <div><b>NFS or SMB/CIFS?</b> |
| <p>When running on Linux, you can use either an NFS or an SMB/CIFS directory (mounted via Samba) for the objectstore. First tests seem to indicate that using a SMB/CIFS directory is much faster, especially if lots of small files are written (as is the case during crawling processes by the Delta or Visited Links service). Also, we had stability issues with an NFS mount, where a lot of "state NFS file handle" errors occurred. |
| </p> |
| Of course, the results may largely depend on your environment and could be completely different in your network.<b><br /></div></b> |
| </div> |
| <p><br /> |
| </p> |
| <a name="Solr_configuration"></a><h4> <span class="mw-headline"> Solr configuration </span></h4> |
| <p>You have to point to the Solr server that we installed above. |
| </p> |
| Configuration file: <pre>SMILA/configuration/org.eclipse.smila.solr/solr.properties</pre> |
| <p><code> |
| </p> |
| <pre> solr.embedded=false |
| ... |
| solr.serverUrl=http://<SOLR-HOST>:8983/solr |
| </pre> |
| <p></code> |
| </p> |
| <a name="Jetty_configuration"></a><h4> <span class="mw-headline"> Jetty configuration </span></h4> |
| <p>To monitor the cluster node, you have to make the SMILA HTTP server accessible from external. |
| </p> |
| File: <pre>SMILA/SMILA.ini</pre> |
| <p><code> |
| </p> |
| <pre> ... |
| -Djetty.host=0.0.0.0 |
| ... |
| </pre> |
| <p></code> |
| </p><p>See also <a href="../Enable_Remote_Access.html" title="SMILA/Documentation/Enable Remote Access">Enabling Remote Access to SMILA</a> |
| </p> |
| <a name="Monitoring"></a><h3> <span class="mw-headline"> Monitoring </span></h3> |
| <p>You can use the <a href="../REST_API_Reference.html" title="SMILA/Documentation/REST API Reference">REST API</a> to monitor SMILA cluster activities. |
| </p> |
| <a name="Startup"></a><h4> <span class="mw-headline"> Startup </span></h4> |
| <p>After having started SMILA, accessing <tt>http://<CLUSTER-NODE>:8080/smila</tt> should return the configured cluster nodes in the response (SMILA 1.2): |
| <code> |
| </p> |
| <pre> ... |
| cluster: { |
| nodes: [ |
| "PC-1", |
| "PC-2", |
| "PC-3" |
| ] |
| } |
| ... |
| </pre> |
| <p></code> |
| </p> |
| <a name="Running_jobs"></a><h4> <span class="mw-headline"> Running jobs </span></h4> |
| <p>After having started a job run, you can check the number of tasks that are currently being processed on each node in ZooKeeper's state at <tt>http://<CLUSTER-NODE>:8080/zookeeper/smila/taskmanager/hosts/</tt>. |
| </p><p>There, you should see a list of cluster nodes and the following output for each of them: (The given sample output means that 6 tasks are currently being processed on the given cluster node.) |
| <code> |
| </p> |
| <pre> stat: ... |
| data: "6" |
| </pre> |
| <p></code> |
| </p><p>You can also count the <code>inprogress</code> tasks under <tt>http://<CLUSTER-NODE>:8080/smila/tasks</tt>, which is the number of tasks currently processed in the whole cluster. This number can be compared with the <code>maxScaleUp</code> setting for a worker in the <code>clusterconfig.json</code> which is the max. number of tasks allowed to be processed on one node. (see also <a href="../TaskManager.html#External_REST_API" title="SMILA/Documentation/TaskManager"> Taskmanager REST API</a>). |
| </p> |
| <a name="Some_useful_commands"></a><h3> <span class="mw-headline"> Some useful commands </span></h3> |
| <p>Removing all documents from a Solr core (Unix-shell command): |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-text">curl http://localhost:8983/solr/DefaultCore/update?commit=true -H "Content-Type: text/xml" --data-binary '<delete><query>*:*</query></delete>'</pre></div> |
| <p>Optimizing the Solr index (Unix-shell command): |
| </p> |
| <div dir="ltr" style="text-align: left;"><pre class="source-text">curl http://localhost:8983/solr/DefaultCore/update?commit=true -H "Content-Type: text/xml" --data-binary '<optimize/>'</pre></div> |
| <p><i>Hint: When using Windows, replace all ' marks with "</i>. |
| </p> |
| <!-- |
| NewPP limit report |
| Preprocessor node count: 102/1000000 |
| Post-expand include size: 2229/2097152 bytes |
| Template argument size: 1697/2097152 bytes |
| #ifexist count: 0/100 |
| --> |
| |
| <!-- Saved in parser cache with key wikidb:pcache:idhash:39273-0!1!0!!en!2!edit=0 and timestamp 20130416060936 --> |
| <div class="printfooter"> |
| Retrieved from "<a href="How_to_setup_SMILA_in_a_cluster.html">http://wiki.eclipse.org/SMILA/Documentation/HowTo/How_to_setup_SMILA_in_a_cluster</a>"</div> |
| <!-- end content --> |
| <div class="visualClear"></div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Yoink of toolbox for phoenix moved up --> |
| |
| |
| </div> |
| </div> |
| <div id="clearFooter"/> |
| <div id="footer" > |
| <ul id="footernav"> |
| <li class="first"><a href="http://www.eclipse.org/">Home</a></li> |
| <li><a href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a></li> |
| <li><a href="http://www.eclipse.org/legal/termsofuse.php">Terms of Use</a></li> |
| <li><a href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a></li> |
| <li><a href="http://www.eclipse.org/org/foundation/contact.php">Contact</a></li> |
| <li><a href="http://wiki.eclipse.org/Eclipsepedia:About" title="Eclipsepedia:About">About Eclipsepedia</a></li> |
| </ul> |
| <span id="copyright">Copyright © 2013 The Eclipse Foundation. All Rights Reserved</span> |
| <p id="footercredit">This page was last modified 14:26, 19 October 2012 by <a href="http://wiki.eclipse.org/User:Andreas.weber.empolis.com" title="User:Andreas.weber.empolis.com">Andreas Weber</a>. Based on work by <a href="http://wiki.eclipse.org/index.php?title=User:Nadine.auslaender.empolis.com&action=edit" class="new" title="User:Nadine.auslaender.empolis.com">Nadine Ausländer</a> and <a href="http://wiki.eclipse.org/User:Juergen.schumacher.empolis.com" title="User:Juergen.schumacher.empolis.com">Juergen Schumacher</a>.</p> |
| <p id="footerviews">This page has been accessed 392 times.</p> |
| </div> |
| |
| <script type="text/javascript"> |
| var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); |
| document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); |
| </script> |
| <script type="text/javascript"> |
| var pageTracker = _gat._getTracker("UA-910670-4"); |
| pageTracker._trackPageview(); |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- <div class="visualClear"></div> --> |
| |
| <script type="text/javascript">if (window.runOnloadHook) runOnloadHook();</script> |
| </div> |
| |
| <!-- Served in 0.051 secs. --></body></html> |
