blob: 7d5a8e918c2aa48910f66e38286c9cc478f6ff60 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-112407000-2"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag() {
dataLayer.push(arguments);
}
gtag('js', new Date());
gtag('config', 'UA-112407000-2');
</script>
<!-- Google Tag Manager -->
<script>
(function (w, d, s, l, i) {
w[l] = w[l] || [];
w[l].push({
'gtm.start': new Date().getTime(),
event: 'gtm.js'
});
var f = d.getElementsByTagName(s)[0],
j = d.createElement(s),
dl = l != 'dataLayer' ? '&l=' + l : '';
j.async = true;
j.src =
'https://www.googletagmanager.com/gtm.js?id=' + i + dl;
f.parentNode.insertBefore(j, f);
})(window, document, 'script', 'dataLayer', 'GTM-KS8HHSF');
</script>
<!-- End Google Tag Manager -->
<head>
<title>A technical autopsy of a containerized ‘Node.js dependency insights’ microservice application | Codewind</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="This blog focuses on the code to implement a cloud-native serverless Node.js microservice developed using Eclispe Codewind.">
<meta name="keywords" content="Eclipse, docker, container, devops, applications, development, iteration, microservices, cloud, services, rapid, integrated"/>
<link rel="icon" type="image/png" sizes="16x16" href="images/favicon/favicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="images/favicon/favicon-32x32.png">
<link href="https://fonts.googleapis.com/css?family=IBM+Plex+Sans:300,400,600&display=swap" rel="stylesheet">
<!-- Bootstrap CSS CDN -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
<link rel="stylesheet" href="css/styles.css">
<link rel="stylesheet" href="css/docs.css">
<link rel="stylesheet" href="css/learn.css">
<link rel="stylesheet" href="css/blog.css">
<link rel="stylesheet" href="css/guides.css">
<link rel="stylesheet" href="css/search.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/themes/prism.min.css">
</head>
<body data-spy="scroll" data-target="#toc">
<!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KS8HHSF"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->
<div class="main">
<!-- Bootstrap NavBar -->
<nav class="navbar navbar-expand-xl navbar-light cw-banner fixed-top" aria-label="topnav">
<button class="navbar-toggler navbar-toggler-right" type="button" data-toggle="collapse"
data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false"
aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<a class="navbar-brand" href="/codewind/">
<img alt="Codewind logo image" title="Codewind logo image" src="images/header/header-logo.svg" class="cw-header-logo" alt="">
</a>
<div class="collapse navbar-collapse justify-content-end cw-navbar-padding" id="navbarNavDropdown">
<ul class="navbar-nav cw-navbar-nav">
<li class="nav-item cw-navbar-item cw-header-link-docs">
<a class="nav-link cw-nav-link cw-header-link-text" href="learn.html">Learn</a>
</li>
<li class="nav-item cw-navbar-item cw-header-link-news">
<a class="nav-link cw-nav-link cw-header-link-text" href="news.html">News</a>
</li>
<li class="nav-item cw-navbar-item cw-header-link-blog">
<a class="nav-link cw-nav-link cw-header-link-text" href="blog.html">Blog</a>
</li>
<li class="nav-item cw-navbar-item cw-header-link-guides">
<a class="nav-link cw-nav-link cw-header-link-text" href="guides.html">Guides</a>
</li>
<form class="form-inline my-2 my-lg-0 cw-navbar-item" action="/codewind/search.html" method="get">
<svg class="bi bi-search" width="1em" height="1em" viewBox="0 0 16 16" fill="black" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" d="M10.442 10.442a1 1 0 011.415 0l3.85 3.85a1 1 0 01-1.414 1.415l-3.85-3.85a1 1 0 010-1.415z" clip-rule="evenodd"/>
<path fill-rule="evenodd" d="M6.5 12a5.5 5.5 0 100-11 5.5 5.5 0 000 11zM13 6.5a6.5 6.5 0 11-13 0 6.5 6.5 0 0113 0z" clip-rule="evenodd"/>
</svg>
<input id="nav-search" class="form-control mr-sm-2" type="text" id="search-box" name="query" placeholder="Search">
</form>
<li class="nav-item cw-navbar-item cw-header-link">
<a class="nav-link cw-nav-link" href="https://github.com/eclipse/codewind"><img alt="Codewind Github" class="banner-image" title="Codewind Github" data-toggle="tooltip" data-placement="top" id="cw_github_stars" title="..." src="images/header/github.svg"/></a>
</li>
<li class="nav-item cw-navbar-item cw-header-link">
<a class="nav-link cw-nav-link" href="https://twitter.com/EclipseCodewind"><img alt="Codewind Twitter" class="banner-image" title="Codewind Twitter" src="images/header/twitter.png"/></a>
</li>
<li class="nav-item cw-navbar-item cw-header-link">
<a class="nav-link cw-nav-link" href="https://mattermost.eclipse.org/eclipse/channels/eclipse-codewind"><img alt="Codewind Mattermost" class="banner-image" title="Codewind Mattermost" src="images/header/mattermost.png"/></a>
</li>
<li class="nav-item cw-navbar-item cw-header-link">
<a class="nav-link cw-nav-link" href="https://www.youtube.com/channel/UCnKCVK6RFDyHFqUmXlAhCHQ"><img alt="Codewind YouTube" class="banner-image" title="Codewind YouTube" src="images/header/youtube.png"/></a>
</li>
<!-- li class="nav-item cw-navbar-item" id="download-li">
<button onClick="window.location.href='https://microclimate.dev/download/codewind';" type="button" class="btn cw-download-button">Download</button>
<a href="https://microclimate.dev/download/codewind" class="nav-link cw-nav-link cw-download-link" href="#">Download</a>
</li> -->
<!-- Smaller devices menu END -->
</ul>
</div>
</nav>
<!-- End Bootstrap NavBar -->
<div class="row" id="post-row">
<div class="col-xs-12 col-lg-1"></div>
<div class="py-5 px-5 col-xs-12 col-lg-10">
<div class="cw-blog-spacer"></div>
<div id="post-content">
<h1>A technical autopsy of a containerized ‘Node.js dependency insights’ microservice application</h1>
<p>30 Oct 2019 - Nik Canvin</p>
<div role="main"><p>This blog focuses on the code to implement a cloud-native serverless Node.js microservice developed using Eclispe Codewind.</p>
<p>Note: To see a quick overview and demonstration of the running microservice from an end users point of view, you may like this <a href="/codewind/a-new-microservice-to-provide-node-js-sub-dependency-license-insights.html">previous blog</a>.</p>
<p>An overview of flow and microservice files:</p>
<p><img src="images/blog/technicalautopsy_1.gif" alt="image of the microservice flow and files" /></p>
<p>A more in-depth look at the key files in the microservice:</p>
<p><strong>‘routers/processNow.js’</strong> and <strong>‘public/uploads.html’</strong> files</p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_2.png" alt="image of code in VS Code" width="800px" />
<em>code extract</em></p>
<ol>
<li>processNode.js defines the root ‘/‘ application endpoint, which loads the uploads.html form, from the ‘/public’ dir.</li>
<li>After the user has entered the details of the package name and version they would like to check, the community ‘<a href="https://www.npmjs.com/package/multer" target="_blank">multer</a>’ module is used to store the details in an ‘uploadData’ constant.</li>
<li>That constant is then passed to the following service for processing:</li>
</ol>
<p><strong>‘services/handleUpload.js’</strong> file (using asynchronous calls throughout to control the following work flow):</p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_3.png" alt="image of code in VS Code" width="800px" />
<em>code extract</em></p>
<ol>
<li>Removes the packageDir if it pre-exists (makes sure we do a fresh check even if package name/version previously checked, in case and sub-dependency details have changed).</li>
<li>Creates working directories for the packageDir and an analysisDir.</li>
<li>Calls GetPackageAndDependencies which downloads and decompresses the package from NPM. It the calls CleanNodePackageInstall to install all the sub-dependencies. If the user specified a ‘clean’ install of sub-dependencies (default) and a package-lock.json exists, then NPM installs all the production sub-dependencies at the versions specified in the package-lock.json file. If not, then the none explicit semver versions declared in the package.json are used instead.</li>
<li>Calls PerformLicenseCheck, with uses the community ‘license-checker’ module to determine either the declared license for all production dependencies to a best guess by analysing a few targeted files.</li>
<li>Now it’s time to determine package usage safety. Initially, all packages are added to the ‘Green list’ (no further action needs to be taken), then the following steps are run to determine any increased usage risk level that is detected.It then calls CompareResultsWithExistingClearance which checks which packages have been previously cleared for use (this assumes the developer is part of a project following a formal IP clearance package approval process, such as the Eclipse Codewind project following the Eclipse IP Process). Any packages not previously cleared for use, are moved to the ‘amber list’ (warning users of packages not previously cleared).</li>
<li>Next os to call CompareResultsWithApprovedLicenses, which checks which packages are NOT licensed according to a pre-approved safe license type list (as above, this also assumes the developer is part of a project following a formal IP clearance package approval process, such as the Eclipse Codewind project following the Eclipse IP Process). Any packages without pre-approved license types are moved to the ‘red list’ (alerting users of packages with potentially blocking usage licenses). Finally is to check if any dependency files contain any risky secondary licensed code by calling ‘PerformGrep’, which spawns a child process to run a ‘grep command’ which scans every file in every package looking for risky non-case sensitive ‘gpl’ keyword. All results are passed too ‘ProcessGrepResults, which gets the line and line number of any risky secondary licensed code. It then calls CompareResultsWithGPLMentions, which moves any ‘amber’ packages to be ‘red’ packages, if risky keywords are found.</li>
<li>All the results are then sent to sent to ‘RenderPugResultsFile’ (see below) to format them as html</li>
<li>Then the html is returned to the users browser.</li>
</ol>
<p><strong>‘results.pug’</strong> file</p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_4.png" alt="image of code in VS Code" width="800px" />
<em>code extract</em></p>
<ol>
<li>A ‘results.css’ stylesheet is linked to help format the html</li>
<li>A banner is added, linking to the ‘Eclipse Codewind’ page and the specific OpenSourceUsageManager’ project GitHub page</li>
<li>Then a summary of overall results is displayed</li>
<li>If packages (on the ‘red’ list) with issues were found, then an alert immediately informs the user.</li>
<li>Else if packages (on the ‘amber’ list) with warnings were found, then this is conveyed to the user.</li>
<li>Else a ‘green’ bill of health informs the user that they may use their package without further action.</li>
<li>Then in a collapsable format, first all the ‘red’ packages are displayed (with a drill to salient comments), then the ‘amber’ ones and finally the ‘green’ ones at the bottom.</li>
</ol>
<p>To get a new containerized microservice up and running immediately without any Kubernetes or Docker skills at all, I used Eclispe Codewind. See this <a href="/codewind/introduction-to-eclipse-codewind-build-high-quality-cloud-native-applications-faster.html">introduction</a> to follow in my footsteps.</p>
<p><strong>All projects created using the Eclipse Codewind Express Node.js project template that I used, contain the following key files:</strong></p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_5.png" alt="image of code in VS Code" width="800px" />
<em>code extract</em></p>
<ul>
<li>A ‘dockerfile’ which drives the docker build process when Eclipse Codewind containerises the application. It defines the base operating system (node:12-stretch in this case) to put in the container and updates all the operating system dependencies. It then copies all the application root files into the container, installs all the application sub-dependencies (node modules in this case), configures/exposes port 3000, then starts the application in the container.</li>
<li>A ‘server.js’ file which creates the ‘express’ server and defines an ‘appmetrics-dash’ endpoint that Eclipse Codewind uses to visualise application metrics such as CPU, memory and http response time/throughput.</li>
<li>A ‘router/health.js’ file which defines a ‘/health’ endpoint which displays a ‘status: ‘UP” message and is used by Eclipse Codewind to determine if the app is running.</li>
</ul>
<p>Two other noteworthy features (available by right-clicking a project in Eclipse Codewind) that were used to ensure the microservice delivered acceptable behaviour and performance that come built in with Eclipse Codewind are:</p>
<p><strong>Application Monitor</strong></p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_6.gif" alt="image of code in VS Code" />
<em>realtime metrics for a running containerised application</em></p>
<p>This was used to check CPU, memory and http metrics during development iterations, to ensure nothing untoward had occured in the latest code changes.</p>
<p><strong>Performance Dashboard</strong></p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_7.png" alt="image of performance dashboard" width="800px" />
<em>the performance dashboard</em></p>
<p>This was used to drive simulated load through the microservice to check overall throughput performance. Several runs were stored and compared during development iterations and to verify the final code.</p>
<h3 id="in-a-nutshell">In a nutshell</h3>
<p>Using the Eclipse Codewind develop tools instantly created a fully formed and started containerized Node.js microservice application, doing all the heavy containerization lifting automatically.</p>
<p style="text-align: center;"><img src="images/blog/technicalautopsy_8.png" alt="image of Codewind" width="800px" />
<em>the performance dashboard</em></p>
<p>The application was completed by adding the code in files described above.
To see the final microservice in action, you may like to visit this previous blog: <a href="/codewind/a-new-microservice-to-provide-node-js-sub-dependency-license-insights.html">“A new microservice to provide ‘Node.js sub-dependency license insights’”</a></p>
</div>
</div>
</div>
<div class="col-xs-12 col-lg-1"></div>
</div>
<!-- footer row -->
<footer>
<div id="footer-div-mobile">
<div class="row">
<div class="col-sm-12 text-center">
<span>Useful Links:</span>
<br/><br/>
<a class="cw-footer-links" href="http://www.eclipse.org">Eclipse Foundation</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/termsofuse.php">Website Terms of Use</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal">Legal</a><br/>
</div>
</div>
<div class="cw_footer_display_flex cw-footer-same-height cw-footer-center">
<div class="cw_footer_display_icons row">
<div class="cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="mailto:codewind-dev@eclipse.org"><img alt="Send us an email" title="Send us an email" src="images/footer/email-icon.svg" class="cw-logo" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="https://twitter.com/EclipseCodewind"><img alt="Codewind Twitter" title="Codewind Twitter" src="images/footer/twitter-logo.svg" class="cw-logo" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="https://github.com/eclipse/codewind"><img alt="Codewind Github" title="Codewind Github" src="images/footer/github-logo.svg"
class="cw-logo" /></a>
</div>
</div>
</div>
<div class=" cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="https://mattermost.eclipse.org/eclipse/channels/eclipse-codewind"><img alt="Codewind Mattermost" title="Codewind Mattermost" src="images/footer/mattermost-logo.png" class="cw-logo-mm" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="https://www.youtube.com/channel/UCnKCVK6RFDyHFqUmXlAhCHQ"><img alt="Codewind YouTube" title="Codewind YouTube" src="images/footer/youtube-logo-dark.svg"
class="cw-logo" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center col-md-3 col-sm-6 col-xs-12">
<div>
<div>
<a href="http://www.eclipse.org"><img class="cw-logo-eclipse-mobile" alt="Eclipse" title="Eclipse" src="images/footer/eclipse.svg"/></a>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="row cw-footer-desktop" id="footer-div">
<div class="cw-footer-left">
<div class="px-5 cw-font-12>
<span class="cw-font-14">Useful Links:</span>
<br/><br/>
<a class="cw-footer-links" href="http://www.eclipse.org">Eclipse Foundation</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/privacy.php">Privacy Policy</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/termsofuse.php">Website Terms of Use</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal/copyright.php">Copyright Agent</a><br/>
<a class="cw-footer-links" href="http://www.eclipse.org/legal">Legal</a><br/>
</div>
</div>
<div class="cw-footer-border-right"></div>
<div class="cw_footer_display_flex cw-footer-same-height cw-footer-center">
<div class="cw_footer_display_icons">
<div class="cw-footer-col text-center">
<div>
<div>
<a href="mailto:codewind-dev@eclipse.org"><img alt="Send us an email" title="Send us an email" src="images/footer/email-icon.svg" class="cw-logo" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center">
<div>
<div>
<a href="https://twitter.com/EclipseCodewind"><img alt="Codewind Twitter" title="Codewind Twitter" src="images/footer/twitter-logo.svg" class="cw-logo" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center">
<div>
<div>
<a href="https://github.com/eclipse/codewind"><img alt="Codewind Github" title="Codewind Github" src="images/footer/github-logo.svg"
class="cw-logo" /></a>
</div>
</div>
</div>
<div class=" cw-footer-col text-center">
<div class="cw-logo-mm" >
<div class="cw-logo-mm" >
<a href="https://mattermost.eclipse.org/eclipse/channels/eclipse-codewind"><img alt="Codewind Mattermost" title="Codewind Mattermost" src="images/footer/mattermost-logo.png" class="cw-logo-mm" /></a>
</div>
</div>
</div>
<div class="cw-footer-col text-center">
<div>
<div>
<a href="https://www.youtube.com/channel/UCnKCVK6RFDyHFqUmXlAhCHQ"><img alt="Codewind YouTube" title="Codewind YouTube" src="images/footer/youtube-logo-dark.svg"
class="cw-logo" /></a>
</div>
</div>
</div>
</div>
</div>
<div class="cw-footer-border-right"></div>
<div class="cw-footer-right cw-footer-same-height cw-footer-vcenter">
<div class="cw-footer-eclipse-img cw-footer-same-height px-5 ">
<a href="http://www.eclipse.org">
<img alt="Eclipse" title="Eclipse" src="images/footer/eclipse.svg"/>
</a>
</div>
</div>
</div>
</footer>
<!-- footer row END -->
<!-- Jquery -->
<script
src="https://code.jquery.com/jquery-3.4.1.min.js"
integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo="
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous">
</script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous">
</script>
<!-- Font Awesome JS -->
<script defer src="https://use.fontawesome.com/releases/v5.0.13/js/solid.js"
integrity="sha384-tzzSw1/Vo+0N5UhStP3bvwWPq+uvzCMfrN1fEFe+xBmv1C/AtVX5K0uZtmcHitFZ" crossorigin="anonymous">
</script>
<script defer src="https://use.fontawesome.com/releases/v5.0.13/js/fontawesome.js"
integrity="sha384-6OIrr52G08NpOFSZdxxz1xdNSndlD4vdcf/q2myIUVO0VsqaGHJsB0RaBE01VTOY" crossorigin="anonymous">
</script>
<script src="js/jquery.matchHeight-min.js"></script>
<script src="js/index.js"></script>
<script src="js/docs.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/prism.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/components/prism-docker.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/components/prism-json.min.js"></script>
</div>
</body>
</html>