blob: 04163eb307b915c682cdb04fc1efc9cf11fb36e1 [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="keywords" content="installationkubernetes, kubernetes, installation, pvc, deployment">
<title>Multi-User&#58 Deploy to Kubernetes | Eclipse Che Documentation</title>
<link rel="stylesheet" href="css/syntax.css">
<link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" crossorigin="anonymous">
<!--<link rel="stylesheet" type="text/css" href="css/bootstrap.min.css">-->
<link rel="stylesheet" href="css/modern-business.css">
<!-- Latest compiled and minified CSS -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<link rel="stylesheet" href="css/customstyles.css">
<link rel="stylesheet" href="css/boxshadowproperties.css">
<!-- most color styles are extracted out to here -->
<link rel="stylesheet" href="css/theme-che.css">
<link rel="stylesheet" href="/css/coderay.css" media="screen" type="text/css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-cookie/1.4.1/jquery.cookie.min.js" crossorigin="anonymous"></script>
<script src="js/jquery.navgoco.min.js"></script>
<!-- Latest compiled and minified JavaScript -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!-- Anchor.js -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/2.0.0/anchor.min.js" crossorigin="anonymous"></script>
<script src="js/toc.js"></script>
<script src="js/customscripts.js"></script>
<link rel="shortcut icon" href="che/docs/images/favicon.ico">
<!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
<script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
<![endif]-->
<link rel="alternate" type="application/rss+xml" title="che" href="http://0.0.0.0:4000/feed.xml">
<script>
$(document).ready(function() {
// Initialize navgoco with default options
$("#mysidebar").navgoco({
caretHtml: '',
accordion: true,
openClass: 'active', // open
save: false, // leave false or nav highlighting doesn't work right
cookie: {
name: 'navgoco',
expires: false,
path: '/'
},
slide: {
duration: 400,
easing: 'swing'
}
});
$("#collapseAll").click(function(e) {
e.preventDefault();
$("#mysidebar").navgoco('toggle', false);
});
$("#expandAll").click(function(e) {
e.preventDefault();
$("#mysidebar").navgoco('toggle', true);
});
});
</script>
<script>
$(function () {
$('[data-toggle="tooltip"]').tooltip()
})
</script>
<script>
$(document).ready(function() {
$("#tg-sb-link").click(function() {
$("#tg-sb-sidebar").toggle();
$("#tg-sb-content").toggleClass('col-md-9');
$("#tg-sb-content").toggleClass('col-md-12');
$("#tg-sb-icon").toggleClass('fa-toggle-on');
$("#tg-sb-icon").toggleClass('fa-toggle-off');
});
});
</script>
</head>
<body>
<!-- Navigation -->
<nav class="navbar navbar-inverse navbar-static-top">
<div class="container topnavlinks">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="fa fa-home fa-lg navbar-brand" href="index.html">&nbsp;<span class="projectTitle"> Eclipse Che Documentation</span></a>
</div>
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav navbar-right">
<!-- toggle sidebar button -->
<li><a id="tg-sb-link" href="#"><i id="tg-sb-icon" class="fa fa-toggle-on"></i> Nav</a></li>
<!-- entries without drop-downs appear here -->
<li><a href="https://medium.com/eclipse-che-blog/" target="_blank">Blog</a></li>
<li><a href="https://github.com/eclipse/che" target="_blank">Source Code</a></li>
<!-- entries with drop-downs appear here -->
<!-- conditional logic to control which topnav appears for the audience defined in the configuration file.-->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Get Support<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="https://github.com/eclipse/che/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Akind%2Fbug" target="_blank">Known Bugs</a></li>
<li><a href="https://github.com/eclipse/che/issues/new" target="_blank">File an Issue</a></li>
<li><a href="https://stackoverflow.com/questions/tagged/eclipse-che" target="_blank">Che on StackOverflow</a></li>
</ul>
</li>
<!--
<li>
<a class="email" title="Submit feedback" href="#" onclick="javascript:window.location='mailto:?subject= feedback&body=I have some feedback about the Multi-User&#58 Deploy to Kubernetes page: ' + window.location.href;"><i class="fa fa-envelope-o"></i> Feedback</a>
</li>
-->
<!--comment out this block if you want to hide search-->
<li>
<!--start search-->
<div id="search-demo-container">
<input type="text" id="search-input" placeholder="search...">
<ul id="results-container"></ul>
</div>
<script src="js/jekyll-search.js" type="text/javascript"></script>
<script type="text/javascript">
SimpleJekyllSearch.init({
searchInput: document.getElementById('search-input'),
resultsContainer: document.getElementById('results-container'),
dataSource: 'search.json',
searchResultTemplate: '<li><a href="{url}" title="Multi-User&amp;#58 Deploy to Kubernetes">{title}</a></li>',
noResultsText: 'No results found.',
limit: 10,
fuzzy: true,
})
</script>
<!--end search-->
</li>
</ul>
</div>
</div>
<!-- /.container -->
</nav>
<!-- Page Content -->
<div class="container">
<div id="main">
<!-- Content Row -->
<div class="row">
<!-- Sidebar Column -->
<div class="col-md-3" id="tg-sb-sidebar">
<ul id="mysidebar" class="nav">
<li class="sidebarTitle"> </li>
<li>
<a href="#">Overview</a>
<ul>
<li><a href="index.html">Introduction</a></li>
<li><a href="quick-start.html">Getting Started</a></li>
<li><a href="single-multi-user.html">Single and Multi-User Flavors</a></li>
<li><a href="infra-support.html">Supported Infrastructures</a></li>
</ul>
</li>
<li>
<a href="#">Che on Docker</a>
<ul>
<li><a href="docker-single-user.html">Docker - Single User</a></li>
<li><a href="docker-multi-user.html">Docker - Multi User</a></li>
<li><a href="docker-config.html">Docker - Configuration</a></li>
<li><a href="docker-cli.html">Docker - CLI Reference</a></li>
</ul>
</li>
<li>
<a href="#">Che on Kubernetes</a>
<ul>
<li><a href="kubernetes-single-user.html">Kubernetes - Single User</a></li>
<li class="active"><a href="kubernetes-multi-user.html">Kubernetes - Multi User</a></li>
<li><a href="kubernetes-config.html">Kubernetes - Configuration</a></li>
<li><a href="kubernetes-admin-guide.html">Kubernetes - Admin Guide</a></li>
</ul>
</li>
<li>
<a href="#">Che on OpenShift</a>
<ul>
<li><a href="openshift-single-user.html">OpenShift - Single User</a></li>
<li><a href="openshift-multi-user.html">OpenShift - Multi User</a></li>
<li><a href="openshift-config.html">OpenShift - Configuration</a></li>
<li><a href="openshift-admin-guide.html">OpenShift - Admin Guide</a></li>
</ul>
</li>
<li>
<a href="#">User Management</a>
<ul>
<li><a href="user-management.html">Authentication and Authorization</a></li>
<li><a href="authentication.html">Security Model</a></li>
<li><a href="permissions.html">Permissions</a></li>
<li><a href="organizations.html">Organizations in UD</a></li>
<li><a href="resource-management.html">Resource Management</a></li>
</ul>
</li>
<li>
<a href="#">User Guides</a>
<ul>
<li><a href="creating-starting-workspaces.html">Creating and starting Workspaces</a></li>
<li><a href="ide-projects.html">Projects</a></li>
<li><a href="editor-code-assistance.html">Editor and Code-Assistance</a></li>
<li><a href="dependency-management.html">Dependency Management</a></li>
<li><a href="commands-ide-macro.html">Commands and IDE Macros</a></li>
<li><a href="version-control.html">Version Control</a></li>
<li><a href="debug.html">Debug</a></li>
</ul>
</li>
<li>
<a href="#">Workspace Administration</a>
<ul>
<li><a href="what-are-workspaces.html">Workspace Overview</a></li>
<li><a href="stacks.html">Workspace - Stacks</a></li>
<li><a href="recipes.html">Workspace - Recipes</a></li>
<li><a href="servers.html">Workspace - Servers</a></li>
<li><a href="installers.html">Workspace - Installers</a></li>
<li><a href="volumes.html">Workspace - Volumes Mount</a></li>
<li><a href="env-variables.html">Workspace - Environment Variables</a></li>
<li><a href="projects.html">Workspace - Projects</a></li>
<li><a href="workspaces-troubleshooting.html">Workspace - Troubleshooting</a></li>
<li><a href="workspace-data-model.html">Workspace Data Model</a></li>
</ul>
</li>
<li>
<a href="#">Portable Workspaces</a>
<ul>
<li><a href="chedir-getting-started.html">Chedir - Getting Started</a></li>
<li><a href="why-chedir.html">Chedir - Why Chedir?</a></li>
<li><a href="chedir-installation.html">Chedir - Installation</a></li>
<li><a href="chedir-project-setup.html">Chedir - Project Setup</a></li>
<li><a href="chedir-up-and-down.html">Chedir - Up and Down</a></li>
<li><a href="chefile.html">Chedir - Chefile</a></li>
<li><a href="chedir-ssh.html">Chedir - SSH</a></li>
<li><a href="factories-getting-started.html">Factory - Getting Started</a></li>
<li><a href="creating-factories.html">Factory - Creating</a></li>
<li><a href="factories_json_reference.html">Factory - JSON Reference</a></li>
</ul>
</li>
<li>
<a href="#">Developer Guides</a>
<ul>
<li><a href="framework-overview.html">Overview</a></li>
<li><a href="rest-api.html">SDK - REST API</a></li>
<li><a href="che-in-che-quickstart.html">SDK - Your First Plugin</a></li>
<li><a href="build-reqs.html">SDK - Building Che</a></li>
<li><a href="assemblies.html">SDK - Assemblies</a></li>
<li><a href="logging.html">SDK - Logging</a></li>
<li><a href="ide-extensions-gwt.html">SDK - GWT IDE Extensions</a></li>
<li><a href="server-side-extensions.html">SDK - Server Side Extensions</a></li>
<li><a href="custom-installers.html">SDK - Installers</a></li>
<li><a href="project-types.html">SDK - Project Types</a></li>
<li><a href="language-servers.html">SDK - Language Support</a></li>
<li><a href="parts.html">IDE UI&#58 Parts</a></li>
<li><a href="actions.html">IDE UI&#58 Actions</a></li>
</ul>
</li>
<li>
<a href="#">Dev Essentials</a>
<ul>
<li><a href="guice.html">Dependency Injection</a></li>
<li><a href="dto.html">Transport&#58 DTO</a></li>
<li><a href="json-rpc.html">Communication&#58 JSON-RPC</a></li>
<li><a href="handling-projects-in-plugins.html">Handling Projects in Plugins</a></li>
<li><a href="dao.html">Persistence, DAO</a></li>
<li><a href="properties.html">Properties</a></li>
</ul>
</li>
<li>
<a href="#">Infrastructure and SPI</a>
<ul>
<li><a href="spi_overview.html">Overview</a></li>
<li><a href="spi-implementation.html">Implementation Notes</a></li>
</ul>
</li>
<!-- if you aren't using the accordion, uncomment this block:
<p class="external">
<a href="#" id="collapseAll">Collapse All</a> | <a href="#" id="expandAll">Expand All</a>
</p>
-->
</ul>
<!-- this highlights the active parent class in the navgoco sidebar. this is critical so that the parent expands when you're viewing a page. This must appear below the sidebar code above. Otherwise, if placed inside customscripts.js, the script runs before the sidebar code runs and the class never gets inserted.-->
<script>$("li.active").parents('li').toggleClass("active");</script>
</div>
<!-- Content Column -->
<div class="col-md-9" id="tg-sb-content">
<div class="post-header">
<h1 class="post-title-main">Multi-User&#58 Deploy to Kubernetes</h1>
</div>
<div class="post-content">
<!-- this handles the automatic toc. use ## for subheads to auto-generate the on-page minitoc. if you use html tags, you must supply an ID for the heading element in order for it to appear in the minitoc. -->
<script>
$( document ).ready(function() {
// Handler for .ready() called.
$('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2' });
/* this offset helps account for the space taken up by the floating toolbar. */
$('#toc').on('click', 'a', function() {
var target = $(this.getAttribute('href'))
, scroll_target = target.offset().top
$(window).scrollTop(scroll_target - 10);
return false
})
});
</script>
<div id="toc"></div>
<!--
-->
<div class="sect1">
<h2 id="prerequisites">Prerequisites</h2>
<div class="sectionbody">
<div class="ulist">
<ul>
<li>
<p>A Kubernetes cluster with at least 4GB RAM and RBAC:</p>
</li>
<li>
<p>for MiniKube 0.26.0 and above <code>minikube start --cpus 2 --memory 4096 --extra-config=apiserver.authorization-mode=RBAC</code></p>
</li>
<li>
<p>for MiniKube 0.25.2 and lower <code>minikube start --cpus 2 --memory 4096 --extra-config=apiserver.Authorization.Mode=RBAC</code></p>
</li>
<li>
<p><code>kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default</code></p>
</li>
<li>
<p>Install the <a href="https://github.com/kubernetes/helm/blob/master/docs/install.md">Helm</a> CLI</p>
</li>
<li>
<p>Set your default Kubernetes context (this is required to use helm):</p>
<div class="ulist">
<ul>
<li>
<p>In MiniKube this is set for you automatically</p>
</li>
<li>
<p>Otherwise, you may have to modify the KUBECONFIG environment variable and then type <code>kubectl config use-context &lt;my-context&gt;</code></p>
</li>
</ul>
</div>
</li>
<li>
<p>Install tiller on your cluster:</p>
<div class="ulist">
<ul>
<li>
<p>Create a <a href="https://github.com/kubernetes/helm/blob/master/docs/rbac.md">tiller serviceAccount</a>: <code>kubectl create serviceaccount tiller --namespace kube-system</code></p>
</li>
<li>
<p>Bind it to the almighty cluster-admin role: <code>kubectl apply -f ./tiller-rbac.yaml</code></p>
</li>
<li>
<p>Install tiller itself: <code>helm init --service-account tiller</code></p>
</li>
</ul>
</div>
</li>
<li>
<p>Ensure that you have an NGINX-based ingress controller. Note: This is the default ingress controller on MiniKube. You can start it with <code>minikube addons enable ingress</code></p>
</li>
<li>
<p>DNS discovery should be enabled. Note: It is enabled by default in minikube.</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect1">
<h2 id="cluster-ip">Cluster IP</h2>
<div class="sectionbody">
<div class="ulist">
<ul>
<li>
<p>If your cluster is running on MiniKube, simply type <code>minikube ip</code> at your terminal</p>
</li>
<li>
<p>If your cluster is in the cloud, obtain the hostname or ip address from your cloud provider</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>In production, you should specify a hostname (see <a href="https://github.com/eclipse/che/issues/8694">here</a> why). In case you don’t have a hostname (e.g. during development), and would still want to use a host-based configuration, you can use services such as nip.io or xip.io.</p>
</div>
<div class="paragraph">
<p>In case you’re specifying a hostname, simply pass it as the value of the <code>ingressDomain</code> parameter below.</p>
</div>
<div class="paragraph">
<p>If you must use IP address (e.g. your corporate policy prevents you from using nip.io), you would also have to set <code>isHostBased</code> to <code>false</code>.</p>
</div>
</div>
</div>
<div class="sect1">
<h2 id="deploy-syntax">Deploy Syntax</h2>
<div class="sectionbody">
<div class="paragraph">
<p>The context of the commands below is <code>che/deploy/kubernetes/helm/che</code></p>
</div>
<div class="sect2">
<h3 id="to-deploy-with-dedicated-keycloak-as-authentication-server">To deploy with dedicated keycloak as authentication server:</h3>
<div class="ulist">
<ul>
<li>
<p>Override default values by changing the values.yaml file and then typing:</p>
<div class="listingblock">
<div class="content">
<pre>helm upgrade --install &lt;my-che-installation&gt; --namespace &lt;my-che-namespace&gt; -f ./values/multi-user.yaml ./</pre>
</div>
</div>
</li>
<li>
<p>Or, you can override default values during installation, using the <code>--set</code> flag:</p>
<div class="listingblock">
<div class="content">
<pre>helm upgrade --install &lt;my-che-installation&gt; --namespace &lt;my-che-namespace&gt; -f ./values/multi-user.yaml --set global.ingressDomain=&lt;my-hostname&gt; --set cheImage=&lt;my-image&gt; ./</pre>
</div>
</div>
</li>
<li>
<p>Master: <code><a href="https://che-&lt;che-namespace&gt;.domain" class="bare">https://che-&lt;che-namespace&gt;.domain</a></code></p>
</li>
<li>
<p>Keycloak: <code><a href="https://keycloak-&lt;che-namespace&gt;.domain" class="bare">https://keycloak-&lt;che-namespace&gt;.domain</a></code></p>
</li>
<li>
<p>Workspaces servers: <code><a href="https://server-host.domain" class="bare">https://server-host.domain</a></code></p>
</li>
</ul>
</div>
</div>
<div class="sect2">
<h3 id="to-deploy-without-keycloak-in-multi-user-mode">To deploy without keycloak in multi-user mode</h3>
<div class="paragraph">
<p>In this case you have to supply custom OpenIdConnect provider (see <a href="https://github.com/eclipse/che-docs/blob/b2310017b1a75901cbec3b9c665d7ffa1cb23177/src/main/pages/setup-openshift/openshift-config.md">here</a> for more details), using the following flags:</p>
</div>
<div class="listingblock">
<div class="content">
<pre>helm upgrade --install &lt;my-che-installation&gt; --namespace &lt;my-che-namespace&gt; -f ./values/multi-user.yaml --set global.ingressDomain=&lt;my-hostname&gt;,cheImage=&lt;my-image&gt;,global.cheDedicatedKeycloak=false,customOidcProvider=&lt;oidc-url&gt;,cheKeycloakClientId=&lt;oidc_clientId&gt;,customOidcUsernameClaim=&lt;user_name_claim&gt; ./</pre>
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>cheKeycloakClientId - is your authentication server client ID</p>
</li>
<li>
<p>customOidcUsernameClaim - is optional, and should be used if your authentication server does not support 'preferred_username' claim in the JWT token (This can be checked by accessing oidc-url/.well-known/openid-configuration response). This parameter defines, which claim should be used to determine user name instead.</p>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="default-host">Default Host</h2>
<div class="sectionbody">
<div class="paragraph">
<p>All Ingress specs are created without a host attribute (defaults to <code>*</code>). Path based routing to all components. Multi User configuration is enabled.</p>
</div>
<div class="listingblock">
<div class="content">
<pre>helm upgrade --install &lt;che-release-name&gt; --namespace &lt;che-namespace&gt; -f ./values/default-host.yaml --set global.ingressDomain=&lt;domain&gt; ./</pre>
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>Master: <code><a href="http://&lt;domain&gt;/" class="bare">http://&lt;domain&gt;/</a></code></p>
</li>
<li>
<p>Keycloak: <code><a href="http://&lt;domain&gt;/auth/" class="bare">http://&lt;domain&gt;/auth/</a></code></p>
</li>
<li>
<p>Workspaces servers: <code><a href="http://&lt;domain&gt;/&lt;path-to-server&gt" class="bare">http://&lt;domain&gt;/&lt;path-to-server&gt</a>;</code></p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect1">
<h2 id="tls">TLS</h2>
<div class="sectionbody">
<div class="paragraph">
<p>Cert-Manager is used to issue LetsEncrypt certificates. To avoid rate-limit issues, we use a single hostname for all ingresses. Path based routing to all components. Multi User configuration is enabled.</p>
</div>
<div class="listingblock">
<div class="content">
<pre>helm install --name &lt;cert-manager-release-name&gt; stable/cert-manager
helm upgrade --install &lt;che-release-name&gt; --namespace &lt;che-namespace&gt; -f ./values/tls.yaml --set global.ingressDomain=&lt;domain&gt; ./</pre>
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>Master: <code><a href="https://&lt;domain&gt;/" class="bare">https://&lt;domain&gt;/</a></code></p>
</li>
<li>
<p>Keycloak: <code><a href="https://&lt;domain&gt;/auth/" class="bare">https://&lt;domain&gt;/auth/</a></code></p>
</li>
<li>
<p>Workspaces servers: <code><a href="https://&lt;domain&gt;/&lt;path-to-server&gt" class="bare">https://&lt;domain&gt;/&lt;path-to-server&gt</a>;</code></p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect1">
<h2 id="delete-che-deployment">Delete Che Deployment</h2>
<div class="sectionbody">
<div class="paragraph">
<p>You can delete a deployment using the following command:</p>
</div>
<div class="listingblock">
<div class="content">
<pre>helm delete &lt;che-release-name&gt;</pre>
</div>
</div>
</div>
</div>
<div class="tags">
<b>Tags: </b>
<a href="tag_installation.html" class="btn btn-default navbar-btn cursorNorm" role="button">installation</a>
<a href="tag_kubernetes.html" class="btn btn-default navbar-btn cursorNorm" role="button">kubernetes</a>
</div>
<!--
-->
</div>
<hr class="shaded"/>
<footer>
<div class="row">
<div class="col-lg-12 footer">
Eclipse Che - Documentation <br/>
Site last generated: Sep 13, 2018 <br/>
<hr>
<a href="http://www.eclipse.org" target="_blank">Eclipse Foundation</a><br/>
<a href="http://www.eclipse.org/legal/privacy.php" target="_blank">Privacy Policy</a><br/>
<a href="http://www.eclipse.org/legal/termsofuse.php" target="_blank">Terms of Use</a><br/>
<a href="https://www.eclipse.org/legal/epl-2.0/" target="_blank">Eclipse Public License</a><br/>
<a href="http://www.eclipse.org/legal" target="_blank">Legal Resources</a><br/>
</div>
</div>
</footer>
<!-- /.row -->
</div>
<!-- /.container -->
</div>
<!-- /#main -->
</div>
</body>
</html>