blob: 187d01604bcad2a6e056b9d7284d4aa2a840d19c [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="keywords" content="blogconnectivity, ">
<title> Digital twins of devices connected via LoRaWAN to TTN </title>
<link rel="stylesheet" href="css/syntax.css">
<link rel="stylesheet" type="text/css" href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" crossorigin="anonymous">
<link rel="stylesheet" href="css/modern-business.css">
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" crossorigin="anonymous">
<link rel="stylesheet" href="css/customstyles.css">
<link rel="stylesheet" href="css/boxshadowproperties.css">
<link rel="stylesheet" href="css/theme-ditto.css">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700">
<script src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" crossorigin="anonymous"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" crossorigin="anonymous"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/2.0.0/anchor.min.js" crossorigin="anonymous"></script>
<script src="js/toc.js"></script>
<script src="js/customscripts.js"></script>
<script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "Organization",
"url": "https://eclipse.org/ditto/",
"logo": "https://eclipse.org/ditto/images/ditto.svg"
}
</script>
<link rel="icon" type="image/png" href="images/favicon-16x16.png" sizes="16x16">
<link rel="icon" type="image/png" href="images/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="images/favicon-96x96.png" sizes="96x96">
<link rel="alternate" type="application/rss+xml" title="Eclipse Ditto Blog" href="https://www.eclipse.org/ditto/feed.xml">
<!-- Eclipse Foundation cookie consent: -->
<link rel="stylesheet" type="text/css" href="//www.eclipse.org/eclipse.org-common/themes/solstice/public/stylesheets/vendor/cookieconsent/cookieconsent.min.css" />
<script src="//www.eclipse.org/eclipse.org-common/themes/solstice/public/javascript/vendor/cookieconsent/default.min.js"></script>
<script>
$(document).ready(function() {
$("#tg-sb-link").click(function() {
$("#tg-sb-sidebar").toggle();
$("#tg-sb-content").toggleClass('col-md-9');
$("#tg-sb-content").toggleClass('col-md-12');
$("#tg-sb-icon").toggleClass('fa-toggle-on');
$("#tg-sb-icon").toggleClass('fa-toggle-off');
});
});
</script>
</head>
<script>
(function(w,d,s,l,i){
w[l]=w[l]||[];
w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});
var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),
dl=l!='dataLayer'?'&l='+l:'';
j.async=true;
j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;
f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-5WLCZXC');
</script>
<body>
<!-- Navigation -->
<nav class="navbar navbar-inverse navbar-fixed-top">
<div class="container topnavlinks">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-ditto-home" href="index.html">&nbsp;<img src="images/ditto_allwhite_symbolonly.svg" class="ditto-navbar-symbol" alt="Home"> <img src="images/ditto_allwhite_textonly.svg" class="ditto-navbar-symbol-text" alt="Ditto"></a>
</div>
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav navbar-right">
<!-- toggle sidebar button -->
<!--<li><a id="tg-sb-link" href="#"><i id="tg-sb-icon" class="fa fa-toggle-on"></i> Nav</a></li>-->
<!-- entries without drop-downs appear here -->
<li><a href="blog.html">Blog</a></li>
<li><a href="intro-overview.html">Documentation</a></li>
<li><a href="http-api-doc.html">HTTP API</a></li>
<li><a href="sandbox.html">Sandbox</a></li>
<li><a href="https://github.com/eclipse/ditto" target="_blank">GitHub</a></li>
<li><a href="https://github.com/eclipse/ditto-examples" target="_blank">GitHub examples</a></li>
<!-- entries with drop-downs appear here -->
<!-- conditional logic to control which topnav appears for the audience defined in the configuration file.-->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Links<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="https://projects.eclipse.org/projects/iot.ditto" target="_blank">Eclipse Ditto Project</a></li>
<li><a href="https://www.eclipse.org/forums/index.php/f/364/" target="_blank">Forum</a></li>
<li><a href="https://ci.eclipse.org/ditto/" target="_blank">Jenkins</a></li>
<li><a href="https://dev.eclipse.org/mhonarc/lists/ditto-dev/" target="_blank">Mailing list archives</a></li>
<li><a href="https://gitter.im/eclipse/ditto" target="_blank">Gitter.im chat</a></li>
</ul>
</li>
<!--comment out this block if you want to hide search-->
<li>
<!--start search-->
<div id="search-demo-container">
<input type="text" id="search-input" placeholder="search...">
<ul id="results-container"></ul>
</div>
<script src="//cdnjs.cloudflare.com/ajax/libs/simple-jekyll-search/0.0.9/jekyll-search.js" type="text/javascript"></script>
<script type="text/javascript">
SimpleJekyllSearch.init({
searchInput: document.getElementById('search-input'),
resultsContainer: document.getElementById('results-container'),
dataSource: 'search.json',
searchResultTemplate: '<li><a href="{url}" title="Digital twins of devices connected via LoRaWAN to TTN">{title}</a></li>',
noResultsText: 'No results found.',
limit: 10,
fuzzy: true,
})
</script>
<!--end search-->
</li>
</ul>
</div>
</div>
<!-- /.container -->
</nav>
<!-- Page Content -->
<div class="container">
<div id="main">
<!-- Content Row -->
<div class="row">
<!-- Content Column -->
<div class="col-md-12" id="tg-sb-content">
<!-- Look the author details up from the site config. -->
<!-- Output author details if some exist. -->
<!-- Output author details if some exist. -->
<!---->
<!--<span>-->
<!--&lt;!&ndash; Mugshot. &ndash;&gt;-->
<!--<img src="https://www.gravatar.com/avatar/19a9fd49b6778aef898249fb4f11bd24?s=135" alt="A photo of Thomas Jäckle" />-->
<!--&lt;!&ndash; Personal Info. &ndash;&gt;-->
<!--Written by <a href="https://github.com/thjaeckle" target="_blank">Thomas Jäckle</a>-->
<!--</span>-->
<!---->
<article class="post" itemscope itemtype="http://schema.org/BlogPosting">
<header class="post-header">
<h1 class="post-title" itemprop="name headline">Digital twins of devices connected via LoRaWAN to TTN</h1>
<p class="post-meta">Published by <img src="https://www.gravatar.com/avatar/19a9fd49b6778aef898249fb4f11bd24?s=135" alt="A photo of Thomas Jäckle" style="width:50px;border-radius:50%;display:inline-block;margin-right:5px;" /><span itemprop="author" itemscope itemtype="http://schema.org/Person"><span itemprop="name"><a href="https://github.com/thjaeckle" target="_blank">Thomas Jäckle</a> </span></span> on <time datetime="2020-04-16T00:00:00+00:00" itemprop="datePublished">Apr 16, 2020</time> - Tags:
<a href="tag_blog.html">blog</a>,
<a href="tag_connectivity.html">connectivity</a>
</p>
</header>
<div class="post-content" itemprop="articleBody">
<!-- this handles the automatic toc. use ## for subheads to auto-generate the on-page minitoc. if you use html tags, you must supply an ID for the heading element in order for it to appear in the minitoc. -->
<script>
$( document ).ready(function() {
// Handler for .ready() called.
$('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2,h3,h4' });
/* this offset helps account for the space taken up by the floating toolbar. */
$('#toc').on('click', 'a', function() {
var target = $(this.getAttribute('href'))
, scroll_target = target.offset().top
$(window).scrollTop(scroll_target - 10);
return false
})
});
</script>
<div id="toc"></div>
<figure><img class="docimage" src="images/2020-04-16-Logo_TTVC_color.png" alt="TTVC logo" style="max-width:300px;padding-left:1em;float:right" /></figure>
<p><br /></p>
<p>A workshop of the <a href="https://www.thethingsnetwork.org/article/the-things-virtual-conference">2020 The Things Virtual Conference</a>
on April 16th 2020 is/was about how to connect Eclipse Ditto to “The Things Network” via TTN’s MQTT broker in order to
automatically update digital twins of devices connected via LoRaWAN to the TTN backend.</p>
<p>You can find the slides <a href="slides/2020_04_16-ttn-virtual-conference/index.html">here</a>.</p>
<p>This blogpost helps setting up this kind of connection and shall also be used as a step-by-step tutorial during
the workshop.</p>
<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/D33JrN2RWiI" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe>
<h2 id="requirements">Requirements</h2>
<p>You’ll need:</p>
<ul>
<li>an operating system capable of running Docker (best use a Linux distribution)</li>
<li>4 CPU cores and 4GB of RAM are advised (less can work, but the Ditto cluster startup is more fragile then)</li>
<li>to have installed: <code class="highlighter-rouge">curl</code> and <code class="highlighter-rouge">git</code></li>
</ul>
<p>Also, you’ll need a TTN account and an existing application with at least one device if you want to follow the hands-on
part and want to create digital twins of your devices connected to TTN.</p>
<h2 id="preparation">Preparation</h2>
<p>Please follow these initial preparation steps (if you don’t already have Docker and Docker Compose installed).</p>
<p>When you have access to a Kubernetes cluster and already have worked with <a href="https://helm.sh">Helm</a> (the package manager
for Kubernetes), you can alternatively install Ditto via its official
<a href="https://hub.helm.sh/charts/eclipse-iot/ditto">Helm chart</a>.</p>
<h3 id="install-docker">Install Docker</h3>
<p>Assumption: You’re running a Debian or Ubuntu based Linux distribution containing the <code class="highlighter-rouge">apt</code> package manager.</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">sudo </span>apt install docker.io
<span class="nb">sudo </span>service docker start
<span class="nb">sudo </span>usermod <span class="nt">-a</span> <span class="nt">-G</span> docker &lt;your-username&gt;
</code></pre></div></div>
<p>Logout and login again so that your user gets the “docker” group.</p>
<h3 id="install-docker-compose">Install Docker Compose</h3>
<p><a href="https://docs.docker.com/compose/install/">Follow the installation guide here</a>, in short:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">sudo </span>curl <span class="nt">-L</span> <span class="s2">"https://github.com/docker/compose/releases/download/1.25.4/docker-compose-</span><span class="k">$(</span>uname <span class="nt">-s</span><span class="k">)</span><span class="s2">-</span><span class="k">$(</span>uname <span class="nt">-m</span><span class="k">)</span><span class="s2">"</span> <span class="nt">-o</span> /usr/local/bin/docker-compose
<span class="nb">sudo </span>chmod +x /usr/local/bin/docker-compose
</code></pre></div></div>
<h3 id="clone-ditto-codebase">Clone Ditto codebase</h3>
<p>That is required to get the <code class="highlighter-rouge">docker-compose.yaml</code> file and other resources required to run Ditto with Docker Compose.</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>git clone <span class="nt">--depth</span> 1 https://github.com/eclipse/ditto.git
</code></pre></div></div>
<h2 id="startup-ditto-cluster">Startup Ditto cluster</h2>
<p>Change directory into the just cloned git repository - optionally adjust the <code class="highlighter-rouge">DITTO_EXTERNAL_PORT</code> variable to where
Ditto is reachable after the start:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">cd </span>ditto/deployment/docker/
<span class="nb">export </span><span class="nv">DITTO_EXTERNAL_PORT</span><span class="o">=</span>80
docker-compose up <span class="nt">-d</span>
</code></pre></div></div>
<p>Verify that Ditto is running:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>docker-compose ps
</code></pre></div></div>
<p>The output should look similar like this:</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code> Name Command State Ports
----------------------------------------------------------------------------------------
docker_concierge_1 /sbin/tini -- java -jar st ... Up 8080/tcp
docker_connectivity_1 /sbin/tini -- java -jar st ... Up 8080/tcp
docker_gateway_1 /sbin/tini -- java -Dditto ... Up 0.0.0.0:8081-&gt;8080/tcp
docker_mongodb_1 docker-entrypoint.sh mongo ... Up 27017/tcp
docker_nginx_1 nginx -g daemon off; Up 0.0.0.0:80-&gt;80/tcp
docker_policies_1 /sbin/tini -- java -jar st ... Up 8080/tcp
docker_swagger-ui_1 nginx -g daemon off; Up 80/tcp, 8080/tcp
docker_things-search_1 /sbin/tini -- java -jar st ... Up 8080/tcp
docker_things_1 /sbin/tini -- java -jar st ... Up 8080/tcp
</code></pre></div></div>
<p>Verify that your Ditto cluster is healthy. Please give it ~1 minute in order to properly start up.</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl <span class="nt">-u</span> devops:foobar http://localhost:<span class="k">${</span><span class="nv">DITTO_EXTERNAL_PORT</span><span class="k">}</span>/status/health
</code></pre></div></div>
<p>The returned output should start with:</p>
<div class="language-json highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">{</span><span class="s2">"label"</span><span class="p">:</span><span class="s2">"roles"</span><span class="p">,</span><span class="s2">"status"</span><span class="p">:</span><span class="s2">"UP"</span><span class="p">,</span><span class="w"> </span><span class="err">...</span><span class="w"> </span><span class="p">}</span><span class="w">
</span></code></pre></div></div>
<p>If your Ditto cluster has trouble starting up (e.g. because you only have less CPU cores than advised), try the following
startup command instead:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>docker-compose start mongodb<span class="p">;</span> sleep 30<span class="p">;</span> docker-compose start policies things<span class="p">;</span> sleep 60<span class="p">;</span> docker-compose start concierge<span class="p">;</span> sleep 60<span class="p">;</span> docker-compose start things-search<span class="p">;</span> sleep 60<span class="p">;</span> docker-compose start connectivity<span class="p">;</span> sleep 60<span class="p">;</span> docker-compose up <span class="nt">-d</span>
</code></pre></div></div>
<h2 id="configure-connection-to-ttn-mqtt-broker">Configure connection to TTN MQTT broker</h2>
<p>The Things Network provides a built in MQTT broker which you can connect to using your TTN application credentials.
For a more detailed description on that topic, please refer to the
<a href="https://www.thethingsnetwork.org/docs/applications/mqtt/quick-start.html">TTN MQTT Quick Start</a>.</p>
<p>Eclipse Ditto can establish connections to MQTT brokers. This is a schematic picture of what we now will do:</p>
<figure><img class="docimage" src="images/../slides/images/ttn-ditto-via-mqtt.png" alt="TTN to Ditto via MQTT" style="max-width:600px;padding-left:1em" /></figure>
<p>In order to connect to your own TTN application, perform the following steps.</p>
<p>You can find the <code class="highlighter-rouge">&lt;AppId&gt;</code> (application ID) and <code class="highlighter-rouge">&lt;AppKey&gt;</code> (access key) in your TTN console of your application.
For <code class="highlighter-rouge">&lt;Region&gt;</code>, e.g. choose <code class="highlighter-rouge">'eu'</code> when your application is in handled by the Handler ‘ttn-handler-eu’.</p>
<p>Please export your application’s credentials locally to environment variables:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span><span class="nv">TTN_REGION</span><span class="o">=</span><span class="s1">'&lt;Region&gt;'</span>
<span class="nb">export </span><span class="nv">TTN_APP_ID</span><span class="o">=</span><span class="s1">'&lt;AppID&gt;'</span>
<span class="nb">export </span><span class="nv">TTN_APP_KEY</span><span class="o">=</span><span class="s1">'&lt;AppKey&gt;'</span>
</code></pre></div></div>
<p>After having done that, you can already create the connection of Ditto to the TTN MQTT broker:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl <span class="nt">-X</span> POST <span class="nt">-u</span> devops:foobar <span class="nt">-H</span> <span class="s1">'Content-Type: application/json'</span> <span class="nt">-d</span> <span class="s1">'{
"targetActorSelection": "/system/sharding/connection",
"headers": {
"aggregate": false
},
"piggybackCommand": {
"type": "connectivity.commands:createConnection",
"connection": {
"id": "ttn-connection-via-mqtt",
"name": "TTN-MQTT",
"connectionType": "mqtt",
"connectionStatus": "open",
"uri": "tcp://'</span><span class="s2">"</span><span class="k">${</span><span class="nv">TTN_APP_ID</span><span class="k">}</span><span class="s2">"</span><span class="s1">':'</span><span class="s2">"</span><span class="k">${</span><span class="nv">TTN_APP_KEY</span><span class="k">}</span><span class="s2">"</span><span class="s1">'@'</span><span class="s2">"</span><span class="k">${</span><span class="nv">TTN_REGION</span><span class="k">}</span><span class="s2">"</span><span class="s1">'.thethings.network:1883",
"failoverEnabled": true,
"clientCount": 1,
"validateCertificates": false,
"sources": [{
"addresses": [
"'</span><span class="s2">"</span><span class="k">${</span><span class="nv">TTN_APP_ID</span><span class="k">}</span><span class="s2">"</span><span class="s1">'/devices/+/up"
],
"consumerCount": 1,
"qos": 0,
"authorizationContext": [
"pre-authenticated:ttn-connection"
],
"enforcement": {
"input": "{{ source:address }}",
"filters": [
"'</span><span class="s2">"</span><span class="k">${</span><span class="nv">TTN_APP_ID</span><span class="k">}</span><span class="s2">"</span><span class="s1">'/devices/{{ thing:name }}/up"
]
},
"replyTarget": {
"enabled": false
},
"payloadMapping": [
"ttn-demo-mapping"
]
}],
"mappingDefinitions": {
"ttn-demo-mapping": {
"mappingEngine": "JavaScript",
"options": {
"incomingScript": "function mapToDittoProtocolMsg(\n headers,\n textPayload,\n bytePayload,\n contentType\n) {\n\n let ttnJson = JSON.parse(textPayload);\n let deviceId = ttnJson['</span><span class="s2">"'"</span><span class="s1">'dev_id'</span><span class="s2">"'"</span><span class="s1">'];\n let payloadFields = ttnJson['</span><span class="s2">"'"</span><span class="s1">'payload_fields'</span><span class="s2">"'"</span><span class="s1">'];\n \n let attributesObj = {\n hardwareSerial: ttnJson['</span><span class="s2">"'"</span><span class="s1">'hardware_serial'</span><span class="s2">"'"</span><span class="s1">'],\n ttnCounter: ttnJson['</span><span class="s2">"'"</span><span class="s1">'counter'</span><span class="s2">"'"</span><span class="s1">']\n };\n \n let featuresObj = {\n temperature: {\n properties: {\n value: payloadFields['</span><span class="s2">"'"</span><span class="s1">'temperature_7'</span><span class="s2">"'"</span><span class="s1">']\n }\n },\n pressure: {\n properties: {\n value: payloadFields['</span><span class="s2">"'"</span><span class="s1">'barometric_pressure_10'</span><span class="s2">"'"</span><span class="s1">']\n }\n },\n humidity: {\n properties: {\n value: payloadFields['</span><span class="s2">"'"</span><span class="s1">'relative_humidity_8'</span><span class="s2">"'"</span><span class="s1">']\n }\n }\n };\n \n let thing = {\n attributes: attributesObj,\n features: featuresObj\n };\n \n let dittoHeaders = {\n '</span><span class="s2">"'"</span><span class="s1">'response-required'</span><span class="s2">"'"</span><span class="s1">': false,\n '</span><span class="s2">"'"</span><span class="s1">'If-Match'</span><span class="s2">"'"</span><span class="s1">': '</span><span class="s2">"'"</span><span class="s1">'*'</span><span class="s2">"'"</span><span class="s1">'\n };\n\n return Ditto.buildDittoProtocolMsg(\n '</span><span class="s2">"'"</span><span class="s1">'org.eclipse.ditto.ttn.demo'</span><span class="s2">"'"</span><span class="s1">',\n deviceId,\n '</span><span class="s2">"'"</span><span class="s1">'things'</span><span class="s2">"'"</span><span class="s1">',\n '</span><span class="s2">"'"</span><span class="s1">'twin'</span><span class="s2">"'"</span><span class="s1">',\n '</span><span class="s2">"'"</span><span class="s1">'commands'</span><span class="s2">"'"</span><span class="s1">',\n '</span><span class="s2">"'"</span><span class="s1">'modify'</span><span class="s2">"'"</span><span class="s1">',\n '</span><span class="s2">"'"</span><span class="s1">'/'</span><span class="s2">"'"</span><span class="s1">',\n dittoHeaders,\n thing\n );\n}",
"outgoingScript": "function mapFromDittoProtocolMsg() { return null; }",
"loadBytebufferJS": "false",
"loadLongJS": "false"
}
}
}
}
}
}'</span> http://localhost:<span class="k">${</span><span class="nv">DITTO_EXTERNAL_PORT</span><span class="k">}</span>/devops/piggyback/connectivity?timeout<span class="o">=</span>8s
</code></pre></div></div>
<p>Explanation - what is done here:</p>
<ul>
<li>using curl with the <code class="highlighter-rouge">devops</code> (admin) user and its initial password <code class="highlighter-rouge">foobar</code> we create a connection of type <code class="highlighter-rouge">mqtt</code>
(you can find further information on that in Ditto’s <a href="connectivity-protocol-bindings-mqtt.html">MQTT docs</a>)</li>
<li>we use the TTN application credentials in the configured <code class="highlighter-rouge">"uri"</code>, connect via plain TCP
(SSL is also possible but in this case a little more complicated as the server certificate of the TTN MQTT broker
would have to be imported)</li>
<li>we add an entry in <code class="highlighter-rouge">"sources"</code>:
<ul>
<li>defining the MQTT topic (<code class="highlighter-rouge">"addresses"</code>) to subscribe to</li>
<li>specifying in which <code class="highlighter-rouge">"authorizationContext"</code> messages from this connection shall be executed</li>
<li>defining in the <code class="highlighter-rouge">"enforcement"</code> that, based on the MQTT topic, a device may only update the Ditto twin having the same name</li>
<li>declaring that a custom payload mapping shall be applied for each incoming message</li>
</ul>
</li>
<li>in the <code class="highlighter-rouge">"mappingDefinitions"</code> we define the previously used “ttn-demo-mapping” as JavaScript based mapping:
<ul>
<li>only an “incoming” script is defined as we don’t handle downstream messages to TTN in this example</li>
<li>when you want to understand the script in more depth, please take a look at the <a href="#javascript-payload-mapping-script-in-detail">details about it</a></li>
</ul>
</li>
</ul>
<div class="alert alert-success" role="alert"><i class="fa fa-check-square-o"></i> <b>Tip:</b> As you have other custom <code class="highlighter-rouge">payload_fields</code> for your TTN devices, please adjust the script
if you want to see the device’s custom payload fields in your Ditto twins.</div>
<h2 id="create-a-common-policy-for-the-twins-to-be-created">Create a common policy for the twins to be created</h2>
<p>Eclipse Ditto secures each API access to the managed twins by applying authorization of the authenticated user.<br />
Those “rules” which authenticated user may access which twins are defined in <a href="basic-policy.html">Policies</a>.</p>
<p>In order to proceed with our scenario, we create a single Policy which shall be used for all twins we create in a later
step:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl <span class="nt">-X</span> PUT <span class="nt">-u</span> ditto:ditto <span class="nt">-H</span> <span class="s1">'Content-Type: application/json'</span> <span class="nt">-d</span> <span class="s1">'{
"policyId": "org.eclipse.ditto.ttn.demo:twin-policy",
"entries": {
"USER": {
"subjects": {
"nginx:ditto": {
"type": "basic auth user authenticated via nginx"
}
},
"resources": {
"thing:/": {
"grant": ["READ", "WRITE"],
"revoke": []
},
"policy:/": {
"grant": ["READ", "WRITE"],
"revoke": []
},
"message:/": {
"grant": ["READ", "WRITE"],
"revoke": []
}
}
},
"TTN": {
"subjects": {
"pre-authenticated:ttn-connection": {
"type": "used in the connections authorizationContext to the TTN MQTT"
}
},
"resources": {
"thing:/": {
"grant": ["WRITE"],
"revoke": []
}
}
}
}
}'</span> http://localhost:<span class="k">${</span><span class="nv">DITTO_EXTERNAL_PORT</span><span class="k">}</span>/api/2/policies/org.eclipse.ditto.ttn.demo:twin-policy
</code></pre></div></div>
<p>Explanation - what is done here:</p>
<ul>
<li>we create a new Policy with the ID <code class="highlighter-rouge">"org.eclipse.ditto.ttn.demo:twin-policy"</code></li>
<li>it contains 2 entries:
<ul>
<li><code class="highlighter-rouge">"USER"</code>: this Policy entry contains the authorization information of the user of the twin APIs (authenticated via the contained “nginx” acting as reverse proxy).
This user may READ+WRITE the things (twins), this created policy and may also send and receive messages.</li>
<li><code class="highlighter-rouge">"TTN"</code>: this Policy entry contains the authorization information of the connection to the TTN MQTT broker (the subject was configured as <code class="highlighter-rouge">"authorizationContext"</code> when we created the connection.
This connection may only WRITE (update) the things (twins).</li>
</ul>
</li>
</ul>
<h2 id="create-digital-twins">Create digital twins</h2>
<p>Now we have everything in place in order to create digital twins for our devices connected to TTN.</p>
<p>Please export all device ids you want to create digital twins for as comma separated environment variable:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">export </span><span class="nv">TTN_DEVICE_IDS</span><span class="o">=</span><span class="s1">'&lt;comma-separated-list-of-your-device-ids&gt;'</span>
</code></pre></div></div>
<p>After having done that, we can already create the twins in Ditto as the <code class="highlighter-rouge">ditto</code> user:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">for </span>dev_id <span class="k">in</span> <span class="k">${</span><span class="nv">TTN_DEVICE_IDS</span><span class="p">//,/ </span><span class="k">}</span>
<span class="k">do</span>
<span class="c"># call your procedure/other scripts here below</span>
<span class="nb">echo</span> <span class="s2">"Creating digital twin with Thing ID: org.eclipse.ditto.ttn.demo:</span><span class="nv">$dev_id</span><span class="s2">"</span>
curl <span class="nt">-X</span> PUT <span class="nt">-u</span> ditto:ditto <span class="nt">-H</span> <span class="s1">'Content-Type: application/json'</span> <span class="nt">-d</span> <span class="s1">'{
"policyId": "org.eclipse.ditto.ttn.demo:twin-policy"
}'</span> http://localhost:<span class="k">${</span><span class="nv">DITTO_EXTERNAL_PORT</span><span class="k">}</span>/api/2/things/org.eclipse.ditto.ttn.demo:<span class="nv">$dev_id</span>
<span class="k">done</span>
</code></pre></div></div>
<p>Explanation - what is done here:</p>
<ul>
<li>we split the passed in <code class="highlighter-rouge">TTN_DEVICE_IDS</code> environment variable by <code class="highlighter-rouge">,</code> and iterate over all contained device ids</li>
<li>for each device ID we create a new Thing (twin) referencing the already previously created Policy</li>
</ul>
<h2 id="access-your-digital-twins-via-api">Access your digital twins via API</h2>
<p>Congratulations, if you have done it so far your TTN devices do now have digital twin representations in Eclipse Ditto.</p>
<div class="alert alert-success" role="alert"><i class="fa fa-check-square-o"></i> <b>Tip:</b> Install the command line tool <code class="highlighter-rouge">jq</code> and pipe the output of the below <code class="highlighter-rouge">curl</code> commands to it in order to get
prettified and colored JSON</div>
<div class="alert alert-info" role="alert"><i class="fa fa-info-circle"></i> <b>Note:</b> Alternatively to <code class="highlighter-rouge">curl</code>, you may also use the locally deployed swagger-ui at http://localhost:${DITTO_EXTERNAL_PORT}/apidoc/
in order to try out Ditto’s HTTP API - <em>make sure to select</em> <code class="highlighter-rouge">/api/2 - local Ditto</code> in the ‘Servers’
section - when asked for credentials, use username ‘ditto’ and password ‘ditto’</div>
<p>You can now, for example, use Ditto’s HTTP APIs in order</p>
<ul>
<li>to retrieve the latest reported values: <code class="highlighter-rouge">curl -u ditto:ditto http://localhost:${DITTO_EXTERNAL_PORT}/api/2/things/org.eclipse.ditto.ttn.demo:&lt;dev_id&gt;</code></li>
<li>to get a live stream of updates to the twins using SSE (Server Sent Events): <code class="highlighter-rouge">curl --http2 -u ditto:ditto -H 'Accept:text/event-stream' -N http://localhost:${DITTO_EXTERNAL_PORT}/api/2/things</code></li>
<li>to list all available twins via the search API: <code class="highlighter-rouge">curl -u ditto:ditto http://localhost:${DITTO_EXTERNAL_PORT}/api/2/search/things</code>
<ul>
<li>alternatively, use your browser and open http://localhost:${DITTO_EXTERNAL_PORT}/api/2/search/things</li>
<li>when asked for credentials, use username “ditto” and password “ditto”</li>
</ul>
</li>
<li>formulate a search query, e.g. only searching for twins with a temperature above 24°, sorted by the last modification, the most recent first
to get the most active twin as first result:
<ul>
<li><code class="highlighter-rouge">curl -u ditto:ditto "http://localhost:${DITTO_EXTERNAL_PORT}/api/2/search/things?filter=gt(features/temperature/properties/value,24.0)&amp;option=sort(-_modified),size(5)&amp;fields=thingId,policyId,attributes,features,_modified,_revision"</code></li>
</ul>
</li>
</ul>
<h2 id="which-other-possibilities-do-we-now-have">Which other possibilities do we now have?</h2>
<p>Now you have all the possibilities Eclipse Ditto as digital twin framework provides, e.g.:</p>
<ul>
<li>directly use your device’s data in a web application consuming Ditto’s HTTP API</li>
<li>directly use your device’s data in a mobile app using Ditto’s bidirectional <a href="httpapi-protocol-bindings-websocket.html">WebSocket</a></li>
<li>make use of the <a href="https://github.com/eclpise/ditto-clients/">Eclipse Ditto Java or JavaScript clients</a> which also use the WebSocket to integrate your device’s data</li>
<li>create another connection (optionally also applying JavaScript based payload mapping)
<ul>
<li>to e.g. <a href="connectivity-protocol-bindings-kafka2.html">Apache Kafka</a> and forward all the modifications made to your devices to there</li>
<li>or using <a href="connectivity-protocol-bindings-http.html">HTTP push</a> in order to call another HTTP API (e.g. insert time series data into an InfluxDB via its HTTP API)</li>
</ul>
</li>
</ul>
<p><br />
<br /></p>
<p>For time reasons we do not go deeper into additional topics, they are possible however, please consult the Ditto
documentation:</p>
<ul>
<li>the WebSocket channel and subscribing for change notifications</li>
<li>sending downward messages to devices</li>
<li>live commands (not retrieving persisted data of devices, but live data)</li>
<li>a more detailed introduction into authentication mechanisms (<a href="installation-operating.html#openid-connect">OpenID Connect with OAuth2.0 is possible</a>)</li>
<li>possibilities to configure your Policies on every resource level, e.g. allowing individuals to only access certain values of a twin</li>
<li>and many other things..</li>
</ul>
<h2 id="additional-resources">Additional resources</h2>
<h3 id="cleanup-after-the-workshop">Cleanup after the workshop</h3>
<p>Simply perform in the <code class="highlighter-rouge">ditto/deployment/docker</code> folder:</p>
<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>docker-compose down
</code></pre></div></div>
<p>And unistall docker + docker-compose (for docker-compose, just remove the downloaded file) again, if you don’t need it.</p>
<h3 id="javascript-payload-mapping-script-in-detail">JavaScript payload mapping script in detail</h3>
<p>Similar to the TTN console’s decoding/converting capabilities of “Payload Formats” of an TTN application, Ditto is able
to apply a custom JavaScript function for each consumed message.<br />
That is necessary in order to convert the received data into a <a href="protocol-overview.html">Ditto Protocol</a> message
including the JSON hierarchy of a so called <a href="basic-thing.html">Thing</a> being the representation of a digital twin.</p>
<p>As the above injected JavaScript payload mapping script is formatted in a single line, this is the script we used pretty
formatted, including the jsdoc of the provided function and some other inline comments.</p>
<p>If you need to adjust the script in order to use your own <code class="highlighter-rouge">payload_fields</code>, please replace all newlines with <code class="highlighter-rouge">\n</code> and
escape the single quotes <code class="highlighter-rouge">'</code> in the script with the following replacement: <code class="highlighter-rouge">'"'"'</code>. Otherwise the single quotes won’t get
correctly escaped in the bash. You can remove the comments before making a single line of the script.</p>
<div class="language-javascript highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="cm">/**
* Maps the passed parameters to a Ditto Protocol message.
* @param {Object.&lt;string, string&gt;} headers - The headers Object containing all received header values
* @param {string} [textPayload] - The String to be mapped
* @param {ArrayBuffer} [bytePayload] - The bytes to be mapped as ArrayBuffer
* @param {string} [contentType] - The received Content-Type, e.g. "application/json"
* @returns {(DittoProtocolMessage|Array&lt;DittoProtocolMessage&gt;)} dittoProtocolMessage(s) -
* The mapped Ditto Protocol message,
* an array of Ditto Protocol messages or
* &lt;code&gt;null&lt;/code&gt; if the message could/should not be mapped
*/</span>
<span class="kd">function</span> <span class="nx">mapToDittoProtocolMsg</span><span class="p">(</span>
<span class="nx">headers</span><span class="p">,</span>
<span class="nx">textPayload</span><span class="p">,</span>
<span class="nx">bytePayload</span><span class="p">,</span>
<span class="nx">contentType</span>
<span class="p">)</span> <span class="p">{</span>
<span class="kd">let</span> <span class="nx">ttnJson</span> <span class="o">=</span> <span class="nx">JSON</span><span class="p">.</span><span class="nx">parse</span><span class="p">(</span><span class="nx">textPayload</span><span class="p">);</span> <span class="c1">// we simply parse the incoming TTN message as JSON</span>
<span class="kd">let</span> <span class="nx">deviceId</span> <span class="o">=</span> <span class="nx">ttnJson</span><span class="p">[</span><span class="s1">'dev_id'</span><span class="p">];</span> <span class="c1">// and extract some fields we require</span>
<span class="kd">let</span> <span class="nx">payloadFields</span> <span class="o">=</span> <span class="nx">ttnJson</span><span class="p">[</span><span class="s1">'payload_fields'</span><span class="p">];</span> <span class="c1">// the 'payload_fields' content is - obviously - different for your application</span>
<span class="kd">let</span> <span class="nx">attributesObj</span> <span class="o">=</span> <span class="p">{</span> <span class="c1">// the attributes of a Thing are meant for unstructured data </span>
<span class="na">hardwareSerial</span><span class="p">:</span> <span class="nx">ttnJson</span><span class="p">[</span><span class="s1">'hardware_serial'</span><span class="p">],</span>
<span class="na">ttnCounter</span><span class="p">:</span> <span class="nx">ttnJson</span><span class="p">[</span><span class="s1">'counter'</span><span class="p">]</span>
<span class="p">};</span>
<span class="kd">let</span> <span class="nx">featuresObj</span> <span class="o">=</span> <span class="p">{</span> <span class="c1">// the features of a Thing e.g. contain sensor data of devices</span>
<span class="na">temperature</span><span class="p">:</span> <span class="p">{</span>
<span class="na">properties</span><span class="p">:</span> <span class="p">{</span>
<span class="na">value</span><span class="p">:</span> <span class="nx">payloadFields</span><span class="p">[</span><span class="s1">'temperature_7'</span><span class="p">]</span>
<span class="p">}</span>
<span class="p">},</span>
<span class="na">pressure</span><span class="p">:</span> <span class="p">{</span>
<span class="na">properties</span><span class="p">:</span> <span class="p">{</span>
<span class="na">value</span><span class="p">:</span> <span class="nx">payloadFields</span><span class="p">[</span><span class="s1">'barometric_pressure_10'</span><span class="p">]</span>
<span class="p">}</span>
<span class="p">},</span>
<span class="na">humidity</span><span class="p">:</span> <span class="p">{</span>
<span class="na">properties</span><span class="p">:</span> <span class="p">{</span>
<span class="na">value</span><span class="p">:</span> <span class="nx">payloadFields</span><span class="p">[</span><span class="s1">'relative_humidity_8'</span><span class="p">]</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">};</span>
<span class="kd">let</span> <span class="nx">thing</span> <span class="o">=</span> <span class="p">{</span> <span class="c1">// a Thing can contain both attributes and features</span>
<span class="na">attributes</span><span class="p">:</span> <span class="nx">attributesObj</span><span class="p">,</span>
<span class="na">features</span><span class="p">:</span> <span class="nx">featuresObj</span>
<span class="p">};</span>
<span class="kd">let</span> <span class="nx">dittoHeaders</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'response-required'</span><span class="p">:</span> <span class="kc">false</span><span class="p">,</span> <span class="c1">// we don't expect a response sent back to TTN</span>
<span class="s1">'If-Match'</span><span class="p">:</span> <span class="s1">'*'</span> <span class="c1">// we only want to update the thing if it already exists</span>
<span class="p">};</span>
<span class="k">return</span> <span class="nx">Ditto</span><span class="p">.</span><span class="nx">buildDittoProtocolMsg</span><span class="p">(</span>
<span class="s1">'org.eclipse.ditto.ttn.demo'</span><span class="p">,</span> <span class="c1">// this is the namespace used as prefix for Ditto Thing IDs</span>
<span class="nx">deviceId</span><span class="p">,</span> <span class="c1">// the TTN device ID is used as "name" part of the Ditto Thing ID </span>
<span class="s1">'things'</span><span class="p">,</span>
<span class="s1">'twin'</span><span class="p">,</span>
<span class="s1">'commands'</span><span class="p">,</span>
<span class="s1">'modify'</span><span class="p">,</span>
<span class="s1">'/'</span><span class="p">,</span>
<span class="nx">dittoHeaders</span><span class="p">,</span>
<span class="nx">thing</span>
<span class="p">);</span>
<span class="p">}</span>
</code></pre></div></div>
<p>An example message received from the TTN MQTT broker:</p>
<div class="language-json highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">{</span><span class="w">
</span><span class="s2">"app_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"iot-campus-be12"</span><span class="p">,</span><span class="w">
</span><span class="s2">"dev_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"node0"</span><span class="p">,</span><span class="w">
</span><span class="s2">"hardware_serial"</span><span class="p">:</span><span class="w"> </span><span class="s2">"70B3D5499A2D3954"</span><span class="p">,</span><span class="w">
</span><span class="s2">"port"</span><span class="p">:</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w">
</span><span class="s2">"counter"</span><span class="p">:</span><span class="w"> </span><span class="mi">9449</span><span class="p">,</span><span class="w">
</span><span class="s2">"payload_raw"</span><span class="p">:</span><span class="w"> </span><span class="s2">"B2cA6AhoKwpzJ8oEAwH4"</span><span class="p">,</span><span class="w">
</span><span class="s2">"payload_fields"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"analog_out_4"</span><span class="p">:</span><span class="w"> </span><span class="mf">5.04</span><span class="p">,</span><span class="w">
</span><span class="s2">"barometric_pressure_10"</span><span class="p">:</span><span class="w"> </span><span class="mf">1018.6</span><span class="p">,</span><span class="w">
</span><span class="s2">"relative_humidity_8"</span><span class="p">:</span><span class="w"> </span><span class="mf">21.5</span><span class="p">,</span><span class="w">
</span><span class="s2">"temperature_7"</span><span class="p">:</span><span class="w"> </span><span class="mf">23.2</span><span class="w">
</span><span class="p">},</span><span class="w">
</span><span class="s2">"metadata"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="err">...</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span></code></pre></div></div>
<p>would be transformed to the following Ditto Protocol message:</p>
<div class="language-json highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">{</span><span class="w">
</span><span class="s2">"topic"</span><span class="p">:</span><span class="w"> </span><span class="s2">"org.eclipse.ditto/node0/things/twin/commands/modify"</span><span class="p">,</span><span class="w">
</span><span class="s2">"path"</span><span class="p">:</span><span class="w"> </span><span class="s2">"/"</span><span class="p">,</span><span class="w">
</span><span class="s2">"value"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"attributes"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"hardwareSerial"</span><span class="p">:</span><span class="w"> </span><span class="s2">"70B3D5499A2D3954"</span><span class="p">,</span><span class="w">
</span><span class="s2">"ttnCounter"</span><span class="p">:</span><span class="w"> </span><span class="mi">9449</span><span class="w">
</span><span class="p">},</span><span class="w">
</span><span class="s2">"features"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"temperature"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"properties"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"value"</span><span class="p">:</span><span class="w"> </span><span class="mf">23.2</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">},</span><span class="w">
</span><span class="s2">"pressure"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"properties"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"value"</span><span class="p">:</span><span class="w"> </span><span class="mf">1018.6</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">},</span><span class="w">
</span><span class="s2">"humidity"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"properties"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="s2">"value"</span><span class="p">:</span><span class="w"> </span><span class="mf">21.5</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span></code></pre></div></div>
<p><br />
<br /></p>
<figure><img class="docimage" src="images/ditto.svg" alt="Ditto" style="max-width: 500px" /></figure>
<p><br />
The Eclipse Ditto team</p>
</div>
</article>
<hr class="shaded"/>
<footer>
<div class="row">
<div class="col-lg-12 footer">
<div class="logo">
<a href="https://eclipse.org"><img src="images/eclipse_foundation_logo.svg" alt="Eclipse logo"/></a>
</div>
<p class="notice">
&copy;2021 Eclipse Ditto.
Site last generated: Mar 22, 2021 <br />
</p>
<div class="quickLinks">
<a href="https://www.eclipse.org/legal/privacy.php" target="_blank">
&gt; Privacy Policy
</a>
<a href="https://www.eclipse.org/legal/termsofuse.php" target="_blank">
&gt; Terms of Use
</a>
<a href="https://www.eclipse.org/legal/copyright.php" target="_blank">
&gt; Copyright Agent
</a>
<a href="https://www.eclipse.org/legal" target="_blank">
&gt; Legal
</a>
<a href="https://www.eclipse.org/legal/epl-2.0/" target="_blank">
&gt; License
</a>
<a href="https://eclipse.org/security" target="_blank">
&gt; Report a Vulnerability
</a>
</div>
</div>
</div>
</footer>
</div>
<!-- /.row -->
</div>
<!-- /.container -->
</div>
<!-- /#main -->
</div>
</body>
</html>