blob: fa1903ab8ee7997a5f0a123f234df3a1b776eeff [file] [log] [blame]
<!DOCTYPE html>
<html lang="dev" class="js csstransforms3d">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="generator" content="Hugo 0.58.3" />
<meta name="description" content="A set of micro-services for connecting millions of devices.">
<meta name="author" content="The Eclipse Hono Project">
<link rel="apple-touch-icon" sizes="180x180" href="/hono/docs/favicon/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="48x48" href="/hono/docs/favicon/favicon-48x48.png">
<link rel="icon" type="image/png" sizes="32x32" href="/hono/docs/favicon/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/hono/docs/favicon/favicon-16x16.png">
<link rel="manifest" href="/hono/docs/favicon/site.webmanifest">
<link rel="mask-icon" href="/hono/docs/favicon/safari-pinned-tab.svg" color="#5bbad5">
<link rel="shortcut icon" href="/hono/docs/favicon/favicon.ico">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/hono/docs/favicon/browserconfig.xml">
<meta name="theme-color" content="#ffffff">
<title>Telemetry API for Kafka Specification :: Eclipse Hono&trade; Vers.: dev</title>
<link href="/hono/docs/css/nucleus.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/fontawesome-all.min.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/hybrid.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/featherlight.min.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/perfect-scrollbar.min.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/auto-complete.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/theme.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/hugo-theme.css?1610071840" rel="stylesheet">
<link href="/hono/docs/css/theme-hono.css?1610071840" rel="stylesheet">
<script src="/hono/docs/js/jquery-2.x.min.js?1610071840"></script>
<style type="text/css">
:root #header + #content > #left > #rlblock_left{
display:none !important;
}
:not(pre) > code + span.copy-to-clipboard {
display: none;
}
</style>
<link rel="stylesheet" href="https://www.eclipse.org/eclipse.org-common/themes/solstice/public/stylesheets/vendor/cookieconsent/cookieconsent.min.css">
<link rel="stylesheet" href='/hono/docs/css/hono.css'>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@EclipseHono">
<meta name="twitter:title" content="Telemetry API for Kafka Specification :: Eclipse Hono&amp;trade; Vers.: dev">
<meta name="twitter:image" content="https://www.eclipse.org/hono/docs/images/twitter_image.png">
<meta name="twitter:description" content="A set of micro-services for connecting millions of devices.">
<meta property="og:title" content="Telemetry API for Kafka Specification :: Eclipse Hono&amp;trade; Vers.: dev" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://www.eclipse.org/hono/docs/dev/api/kafka/telemetry//" />
<meta property="og:image" content="https://www.eclipse.org/hono/docs/images/twitter_image.png" />
</head>
<body class="" data-url="/hono/docs/dev/api/kafka/telemetry/">
<nav id="sidebar" class="">
<div id="header-wrapper">
<div id="header">
<a href="https://www.eclipse.org/hono/">
<img src="/hono/docs/images/HONO-Logo_Bild-Wort_quer-w-310x120px.svg" alt="Hono logo" class="logo-img">
</a>
</div>
<div class="searchbox">
<label for="search-by"><i class="fas fa-search"></i></label>
<input data-search-input id="search-by" type="search" placeholder="Search...">
<span data-search-clear=""><i class="fas fa-times"></i></span>
</div>
<script type="text/javascript" src="/hono/docs/js/lunr.min.js?1610071840"></script>
<script type="text/javascript" src="/hono/docs/js/auto-complete.js?1610071840"></script>
<script type="text/javascript">
var baseurl = "https:\/\/www.eclipse.org\/hono\/docs\/\/dev";
</script>
<script type="text/javascript" src="/hono/docs/js/search.js?1610071840"></script>
</div>
<div class="highlightable">
<ul class="topics">
<li data-nav-id="/hono/docs/dev/concepts/" title="Concepts" class="dd-item
">
<a href="/hono/docs/dev/concepts/">
<i class="far fa-lightbulb"></i> Concepts
</a>
<ul>
<li data-nav-id="/hono/docs/dev/concepts/device-identity/" title="Device Identity" class="dd-item ">
<a href="/hono/docs/dev/concepts/device-identity/">
Device Identity
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/tenancy/" title="Multi-Tenancy" class="dd-item ">
<a href="/hono/docs/dev/concepts/tenancy/">
Multi-Tenancy
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/device-provisioning/" title="Device Provisioning" class="dd-item ">
<a href="/hono/docs/dev/concepts/device-provisioning/">
Device Provisioning
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/connecting-devices/" title="Connecting Devices" class="dd-item ">
<a href="/hono/docs/dev/concepts/connecting-devices/">
Connecting Devices
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/device-notifications/" title="Device Notifications" class="dd-item ">
<a href="/hono/docs/dev/concepts/device-notifications/">
Device Notifications
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/command-and-control/" title="Command &amp; Control" class="dd-item ">
<a href="/hono/docs/dev/concepts/command-and-control/">
Command &amp; Control
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/resource-limits/" title="Resource limits" class="dd-item ">
<a href="/hono/docs/dev/concepts/resource-limits/">
Resource limits
</a>
</li>
<li data-nav-id="/hono/docs/dev/concepts/connection-events/" title="Connection Events" class="dd-item ">
<a href="/hono/docs/dev/concepts/connection-events/">
Connection Events
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/" title="User Guide" class="dd-item
">
<a href="/hono/docs/dev/user-guide/">
<i class="fas fa-book-reader"></i> User Guide
</a>
<ul>
<li data-nav-id="/hono/docs/dev/user-guide/mongodb-based-device-registry/" title="MongoDB Based Device Registry" class="dd-item ">
<a href="/hono/docs/dev/user-guide/mongodb-based-device-registry/">
MongoDB Based Device Registry
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/file-based-device-registry/" title="File Based Device Registry" class="dd-item ">
<a href="/hono/docs/dev/user-guide/file-based-device-registry/">
File Based Device Registry
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/http-adapter/" title="HTTP Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/http-adapter/">
HTTP Adapter
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/mqtt-adapter/" title="MQTT Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/mqtt-adapter/">
MQTT Adapter
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/amqp-adapter/" title="AMQP Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/amqp-adapter/">
AMQP Adapter
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/coap-adapter/" title="CoAP Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/coap-adapter/">
CoAP Adapter
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/kura-adapter/" title="Kura Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/kura-adapter/">
Kura Adapter
</a>
</li>
<li data-nav-id="/hono/docs/dev/user-guide/sigfox-adapter/" title="Sigfox Adapter" class="dd-item ">
<a href="/hono/docs/dev/user-guide/sigfox-adapter/">
Sigfox Adapter
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/" title="Admin Guide" class="dd-item
">
<a href="/hono/docs/dev/admin-guide/">
<i class="fas fa-sliders-h"></i> Admin Guide
</a>
<ul>
<li data-nav-id="/hono/docs/dev/admin-guide/common-config/" title="Common Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/common-config/">
Common Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/auth-server-config/" title="Auth Server Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/auth-server-config/">
Auth Server Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/jdbc-device-registry-config/" title="JDBC Based Device Registry Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/jdbc-device-registry-config/">
JDBC Based Device Registry Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/mongodb-device-registry-config/" title="MongoDB Based Device Registry Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/mongodb-device-registry-config/">
MongoDB Based Device Registry Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/file-based-device-registry-config/" title="File Based Device Registry Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/file-based-device-registry-config/">
File Based Device Registry Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/command-router-config/" title="Configuring the Command Router Service" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/command-router-config/">
Command Router Service Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/device-connection-config/" title="Configuring the Device Connection Service" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/device-connection-config/">
Device Connection Service Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/http-adapter-config/" title="HTTP Adapter Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/http-adapter-config/">
HTTP Adapter Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/mqtt-adapter-config/" title="MQTT Adapter Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/mqtt-adapter-config/">
MQTT Adapter Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/amqp-adapter-config/" title="AMQP Adapter Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/amqp-adapter-config/">
AMQP Adapter Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/coap-adapter-config/" title="CoAP Adapter Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/coap-adapter-config/">
CoAP Adapter Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/kura-adapter-config/" title="Kura Adapter Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/kura-adapter-config/">
Kura Adapter Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/hono-client-configuration/" title="Hono Client Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/hono-client-configuration/">
Hono Client Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/hono-kafka-client-configuration/" title="Hono Kafka Client Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/hono-kafka-client-configuration/">
Hono Kafka Client Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/amqp-network-config/" title="AMQP 1.0 Messaging Network Configuration" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/amqp-network-config/">
AMQP 1.0 Messaging Network Configuration
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/secure_communication/" title="Secure Communication" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/secure_communication/">
Secure Communication
</a>
</li>
<li data-nav-id="/hono/docs/dev/admin-guide/monitoring-tracing-config/" title="Monitoring &amp; Tracing" class="dd-item ">
<a href="/hono/docs/dev/admin-guide/monitoring-tracing-config/">
Monitoring &amp; Tracing
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/dev-guide/" title="Developer Guide" class="dd-item
">
<a href="/hono/docs/dev/dev-guide/">
<i class="fas fa-tools"></i> Developer Guide
</a>
<ul>
<li data-nav-id="/hono/docs/dev/dev-guide/building_hono/" title="Building from Source" class="dd-item ">
<a href="/hono/docs/dev/dev-guide/building_hono/">
Building from Source
</a>
</li>
<li data-nav-id="/hono/docs/dev/dev-guide/amqp_adapter_client/" title="AMQP Adapter Client for Java" class="dd-item ">
<a href="/hono/docs/dev/dev-guide/amqp_adapter_client/">
AMQP Adapter Client for Java
</a>
</li>
<li data-nav-id="/hono/docs/dev/dev-guide/java_client_consumer/" title="Consuming Messages from Java" class="dd-item ">
<a href="/hono/docs/dev/dev-guide/java_client_consumer/">
Consuming Messages from Java
</a>
</li>
<li data-nav-id="/hono/docs/dev/dev-guide/custom_http_adapter/" title="Implement a Custom Hono HTTP Protocol Adapter" class="dd-item ">
<a href="/hono/docs/dev/dev-guide/custom_http_adapter/">
Implement a Custom Hono HTTP Protocol Adapter
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/api/" title="API" class="dd-item
parent
">
<a href="/hono/docs/dev/api/">
&nbsp;<i class='fas fa-plug'></i>&nbsp;API
</a>
<ul>
<li data-nav-id="/hono/docs/dev/api/telemetry/" title="Telemetry API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/telemetry/">
Telemetry API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/event/" title="Event API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/event/">
Event API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/command-and-control/" title="Command &amp; Control API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/command-and-control/">
Command &amp; Control API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/kafka/telemetry/" title="Telemetry API for Kafka Specification" class="dd-item active">
<a href="/hono/docs/dev/api/kafka/telemetry/">
Telemetry API - Kafka
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/kafka/event/" title="Event API for Kafka Specification" class="dd-item ">
<a href="/hono/docs/dev/api/kafka/event/">
Event API - Kafka
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/tenant/" title="Tenant API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/tenant/">
Tenant API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/command-router/" title="Command Router API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/command-router/">
Command Router API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/device-connection/" title="Device Connection API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/device-connection/">
Device Connection API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/device-registration/" title="Device Registration API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/device-registration/">
Device Registration API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/credentials/" title="Credentials API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/credentials/">
Credentials API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/authentication/" title="Authentication API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/authentication/">
Authentication API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/management/" title="Device Registry Management API Specification" class="dd-item ">
<a href="/hono/docs/dev/api/management/">
Device Registry Management API
</a>
</li>
<li data-nav-id="/hono/docs/dev/api/metrics/" title="Metrics" class="dd-item ">
<a href="/hono/docs/dev/api/metrics/">
Metrics
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/deployment/" title="Deployment" class="dd-item
">
<a href="/hono/docs/dev/deployment/">
<i class="fas fa-shipping-fast"></i> Deployment
</a>
<ul>
<li data-nav-id="/hono/docs/dev/deployment/helm-based-deployment/" title="Helm based Deployment" class="dd-item ">
<a href="/hono/docs/dev/deployment/helm-based-deployment/">
Helm based Deployment
</a>
</li>
<li data-nav-id="/hono/docs/dev/deployment/openshift/" title="OpenShift / OKD" class="dd-item ">
<a href="/hono/docs/dev/deployment/openshift/">
OpenShift / OKD
</a>
</li>
<li data-nav-id="/hono/docs/dev/deployment/create-kubernetes-cluster/" title="Setting up a Kubernetes Cluster" class="dd-item ">
<a href="/hono/docs/dev/deployment/create-kubernetes-cluster/">
Setting up a Kubernetes Cluster
</a>
</li>
<li data-nav-id="/hono/docs/dev/deployment/resource-limitation/" title="Limiting Resource Usage" class="dd-item ">
<a href="/hono/docs/dev/deployment/resource-limitation/">
Limiting Resource Usage
</a>
</li>
</ul>
</li>
<li data-nav-id="/hono/docs/dev/architecture/" title="Architecture" class="dd-item
">
<a href="/hono/docs/dev/architecture/">
<i class="fas fa-landmark"></i> Architecture
</a>
<ul>
<li data-nav-id="/hono/docs/dev/architecture/component-view/" title="Component View" class="dd-item ">
<a href="/hono/docs/dev/architecture/component-view/">
Component View
</a>
</li>
<li data-nav-id="/hono/docs/dev/architecture/auth/" title="Authentication/Authorization" class="dd-item ">
<a href="/hono/docs/dev/architecture/auth/">
Authentication/Authorization
</a>
</li>
</ul>
</li>
</ul>
<section id="shortcuts">
<h3></h3>
<ul>
<li>
<a class="padding" href="https://www.eclipse.org/hono/" title="Hono&#39;s Homepage"><i class='fas fa-home'></i> Hono Home</a>
</li>
<li>
<a class="padding" href="https://www.eclipse.org/hono/getting-started/" title="Getting started with Eclipse Hono"><i class='fas fa-plane-departure'></i> Getting Started</a>
</li>
</ul>
</section>
<section id="prefooter">
<hr/>
<ul>
<li>
<div id="select-box-wrapper">
<div id="select-box">
<a class="padding">
Version:&nbsp;
<div class="select-style">
<select id="select-language" onchange="location = this.value;">
<option id="dev" value="/hono/docs/dev/api/kafka/telemetry/" selected>dev</option>
</select>
<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="255px" height="255px" viewBox="0 0 255 255" style="enable-background:new 0 0 255 255;" xml:space="preserve">
<g>
<g id="arrow-drop-down">
<polygon points="0,63.75 127.5,191.25 255,63.75 " />
</g>
</g>
</svg>
</div>
</a>
</div>
</div>
</li>
</ul>
</section>
<section id="footer">
<p>&copy; 2021 <a href="https://www.eclipse.org/hono/">The Eclipse Hono Project</a></p>
<p>
Documentation built with
<a href="https://gohugo.io/" target="_blank">Hugo</a>
using the
<a href="https://github.com/matcornic/hugo-theme-learn" target="_blank">Learn</a> theme.
</p>
<div class="eclipse-logo">
<a href="https://www.eclipse.org" target="_blank">
<img src="https://www.eclipse.org/hono/docs/images/eclipse_foundation_logo.svg"/>
</a>
</div>
</section>
</div>
</nav>
<section id="body">
<div id="overlay"></div>
<div class="old-version-hint">
<p>This page refers to version <em>dev</em>.
You might want to use the <a href="https://www.eclipse.org/hono/docs/">current stable</a> version.
</p>
</div>
<div class="padding highlightable">
<div>
<div id="top-bar">
<div id="top-github-link">
<a class="github-link" title='Edit this page' href="https://github.com/eclipse/hono/edit/master/site/documentation/content/api/kafka/telemetry/index.md" target="blank">
<i class="fas fa-code-branch"></i>
<span id="top-github-link-text">Edit this page</span>
</a>
</div>
<div id="breadcrumbs" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
<span id="sidebar-toggle-span">
<a href="#" id="sidebar-toggle" data-sidebar-toggle="">
<i class="fas fa-bars"></i>
</a>
</span>
<span id="toc-menu"><i class="fas fa-list-alt"></i></span>
<span class="links">
<a href='/hono/docs/dev/'>Documentation</a> > <a href='/hono/docs/dev/api/'>API</a> > Telemetry API for Kafka Specification
</span>
</div>
<div class="progress">
<div class="wrapper">
<nav id="TableOfContents">
<ul>
<li>
<ul>
<li><a href="#kafka-based-messaging">Kafka-based Messaging</a>
<ul>
<li><a href="#quality-of-service">Quality of Service</a></li>
<li><a href="#message-ordering">Message Ordering</a></li>
<li><a href="#at-least-once-producers">AT LEAST ONCE Producers</a></li>
<li><a href="#at-most-once-producers">AT MOST ONCE Producers</a></li>
</ul></li>
<li><a href="#southbound-operations">Southbound Operations</a>
<ul>
<li><a href="#produce-telemetry-data">Produce Telemetry Data</a></li>
</ul></li>
<li><a href="#northbound-operations">Northbound Operations</a>
<ul>
<li><a href="#consume-telemetry-data">Consume Telemetry Data</a></li>
</ul></li>
</ul></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="body-inner">
<h1>Telemetry API for Kafka Specification</h1>
<p>The <em>Telemetry</em> API is used by <em>Protocol Adapters</em> to send telemetry data downstream.
<em>Business Applications</em> and other consumers use the API to receive data published by devices belonging to a particular tenant.</p>
<p>The Telemetry API for Kafka is an alternative to the <a href="/hono/docs/dev/api/telemetry/">Telemetry API for AMQP</a>.
With this API clients publish telemetry data to an Apache Kafka&reg; cluster instead of an AMQP Messaging Network.</p>
<div class="alert alert-notice">
<h4 class="alert-heading"><i class="fas fa-info-circle"></i> Tech preview</h4>
<div>The support of Kafka as a messaging system is currently a preview and not yet ready for production. The APIs may change with the next version.</div>
</div>
<h2 id="kafka-based-messaging">Kafka-based Messaging</h2>
<p>Using Kafka instead of AMQP comes with slightly different behavior. Kafka provides a Publish/Subscribe messaging style.
Every message is sent by a <em>producer</em> to a <em>topic</em> in a Kafka cluster, where it will be persisted by (multiple) <em>brokers</em>.
Each topic consists of a configurable number of <em>partitions</em>. The <em>key</em> of a message determines to which partition it will be written.
Each partition is <em>replicated</em> to a configurable number of brokers in the cluster to provide fault-tolerance if a broker goes down.</p>
<p>A Kafka message (also called record) consists of a key, a value, a timestamp, and headers.</p>
<p>Multiple <em>consumers</em> can read the messages at the same time and they can read them repeatedly (if an error occurred).
This decoupling of producer and consumers has the consequence that a producer does not get feedback about the consumption of messages,
it does not &ldquo;know&rdquo; <em>when</em> a message will be read by a consumer (or if there are many or no consumer at all).
For example, a protocol adapter can only confirm to the device that the Kafka cluster successfully persisted a telemetry message,
not if a <em>Business Application</em> received it.</p>
<p>Messages are usually deleted from a topic at some point &ndash; regardless of whether they have been processed by a consumer.
The duration of this <em>log retention time</em> should be configured in Kafka to reasonable values for each topic.</p>
<p>See the <a href="https://kafka.apache.org/documentation/#configuration">Kafka documentation</a> for details about Kafka&rsquo;s configuration properties.</p>
<h3 id="quality-of-service">Quality of Service</h3>
<p>The configuration property <a href="https://kafka.apache.org/documentation/#acks"><em>acks</em></a> of the Kafka client is used to
configure what acknowledgements a producer expects from the cluster for each message.
The value of this property determines the maximum <em>Quality of Service</em> level the producer can achieve.
Kafka supports the following settings:</p>
<ul>
<li><code>0</code>: the producer does not wait for any acknowledgements. This provides no guarantees.</li>
<li><code>1</code>: the producer requests only an acknowledgement from one broker. If the broker goes down before others finished
replication, then the message will be lost.</li>
<li><code>all</code> (or equivalent <code>-1</code>): the producer requests an acknowledgement that confirms that the message has successfully
been replicated to the required number of brokers. This guarantees that the message will not be lost as long as
at least one of the replicas remains alive.</li>
</ul>
<h3 id="message-ordering">Message Ordering</h3>
<p>Hono requires that producers send messages in the right order. Each producer MUST always send messages with the same key
to the same partition.
For example, each protocol adapter must always send telemetry data reported by a particular device to the same partition
(e.g. by using the same <a href="https://kafka.apache.org/documentation/#partitioner.class">partitioner</a> implementation)
in the same order that they have been received in.</p>
<p>The following producer configuration properties influence if the order of the messages can be guaranteed by Kafka:</p>
<ul>
<li><a href="https://kafka.apache.org/documentation/#retries"><em>retries</em></a></li>
<li><a href="https://kafka.apache.org/documentation/#max.in.flight.requests.per.connection"><em>max.in.flight.requests.per.connection</em></a></li>
<li><a href="https://kafka.apache.org/documentation/#enable.idempotence"><em>enable.idempotence</em></a></li>
</ul>
<p>Setting <em>acks</em> to <code>0</code> effectively disables retries. If <em>retries</em> are set to a value greater than zero and
<em>max.in.flight.requests.per.connection</em> is set to a value greater than <code>1</code>, Kafka no longer guarantees that messages
are stored in the order they have been sent.
The only exception is the <em>idempotent</em> producer, which can handle <em>max.in.flight.requests.per.connection</em> to be up to <code>5</code>
with retries enabled while still maintaining the message order (since Kafka version 1.0.0).
Note that the <em>idempotent</em> producer option requires special privileges for the producer&rsquo;s user to be configured.</p>
<h3 id="at-least-once-producers">AT LEAST ONCE Producers</h3>
<p>To provide <em>AT LEAST ONCE</em> delivery semantics, a producer MUST wait for the acknowledgements from all <em>in-sync replicas</em>
before sending the acknowledgement back to the client.
This can be achieved e.g. by setting <em>acks</em> to <code>all</code> or, implicitly, by setting <em>enable.idempotence</em> to <code>true</code>.</p>
<p>The producer MUST retain the order in which it received messages from a client.
This requires that if <em>retries</em> is set to a value greater than zero, <em>max.in.flight.requests.per.connection</em> must be set to <code>1</code>.
Alternatively, <em>enable.idempotence</em> can be set to <code>true</code>.
Disabling retries might cause messages to fail in case of high load or transient transmission failures, so this is not recommended.</p>
<p>The recommended configuration for <em>AT LEAST ONCE</em> producers has the following properties:
<code>enable.idempotence=true</code>, <code>acks=all</code>, <code>max.in.flight.requests.per.connection=5</code>, leaving <em>retries</em> unset (which defaults to <code>Integer.MAX</code>),
and setting <em>delivery.timeout.ms</em> to a reasonable value. This configuration is supported from Kafka version 1.0.0 on.</p>
<h3 id="at-most-once-producers">AT MOST ONCE Producers</h3>
<p>Every producer that does not wait for acknowledgements from all <em>in-sync replicas</em> or does not consider them before sending
an acknowledgement back to the client, is considered to provide only <em>AT MOST ONCE</em> delivery semantics.
To achieve this, several strategies are possible:</p>
<ol>
<li>The producer can disable acknowledgements (<code>acks=0</code>). This is the fastest mode of delivery but has the drawback of a potential loss of messages without notice.</li>
<li>The producer can enable acknowledgements, but not wait for the acknowledgements from the Kafka cluster before acknowledging the message to <em>its</em> client.<br /></li>
</ol>
<p>The producer MUST retain the order in which it received messages from a client.
This requires to either set <em>retries</em> to <code>0</code> or to set <em>max.in.flight.requests.per.connection</em> to <code>1</code>.</p>
<p><strong>NB:</strong> To send messages with both delivery semantics with the same producer, it MUST be configured for <em>AT LEAST ONCE</em>.
Such a producer may ignore the outcome of the <em>produce</em> operation for <em>AT MOST ONCE</em> messages.</p>
<h2 id="southbound-operations">Southbound Operations</h2>
<p>The following operation can be used by <em>Protocol Adapters</em> to send telemetry data received from devices to downstream consumers like <em>Business Applications</em>.</p>
<h3 id="produce-telemetry-data">Produce Telemetry Data</h3>
<p>The protocol adapter writes messages to the tenant-specific topic <code>hono.telemetry.${tenant_id}</code> where <code>${tenant_id}</code> is the ID of the tenant that the client wants to upload telemetry data for.</p>
<p><strong>Preconditions</strong></p>
<ol>
<li>Either the topic <code>hono.telemetry.${tenant_id}</code> exists, or the broker is configured to automatically create topics on demand.</li>
<li>The client is authorized to write to the topic.</li>
<li>The device for which the adapter wants to send telemetry data has been registered (see <a href="/hono/docs/dev/api/device-registration/">Device Registration API</a>).</li>
</ol>
<p><strong>Message Flow</strong></p>
<p>It is up to the discretion of the protocol adapter whether it wants to use <em>AT LEAST ONCE</em> or <em>AT MOST ONCE</em> delivery semantics
according to <a href="#at-least-once-producers">AT LEAST ONCE Producers</a> respectively <a href="#at-most-once-producers">AT MOST ONCE Producers</a>.</p>
<p>The recommended Kafka producer properties for <em>AT LEAST ONCE</em> delivery are: <code>enable.idempotence=true</code>, <code>acks=all</code>, <code>max.in.flight.requests.per.connection=5</code>,
leaving <em>retries</em> unset, and setting <em>delivery.timeout.ms</em> to a reasonable value (supported from Kafka version 1.0.0 on).</p>
<p>The following sequence diagram illustrates the flow of messages involved in the <em>HTTP Adapter</em> producing a telemetry data message to the Kafka cluster implementing <em>AT MOST ONCE</em> delivery semantics.</p>
<figure>
<img src="produce_kafka_qos0.svg"/> <figcaption>
<h4>Produce telemetry data flow (AT MOST ONCE)</h4>
</figcaption>
</figure>
<ol>
<li><em>Device</em> <code>4711</code> PUTs telemetry data to the <em>HTTP Adapter</em>
<ol>
<li><em>HTTP Adapter</em> produces telemetry data to <em>Kafka Cluster</em>.</li>
<li><em>HTTP Adapter</em> acknowledges the reception of the data to the <em>Device</em>.</li>
</ol></li>
</ol>
<p>The following sequence diagram illustrates the flow of messages involved in the <em>HTTP Adapter</em> producing a telemetry data message to the Kafka cluster implementing <em>AT LEAST ONCE</em> delivery semantics.</p>
<figure>
<img src="produce_kafka_qos1.svg"/> <figcaption>
<h4>Produce telemetry data flow (AT LEAST ONCE)</h4>
</figcaption>
</figure>
<ol>
<li><em>Device</em> <code>4711</code> PUTs telemetry data to the <em>HTTP Adapter</em>, indicating <em>QoS Level</em> 1.
<ol>
<li><em>HTTP Adapter</em> produces telemetry data to <em>Kafka Cluster</em>.</li>
<li><em>Kafka Cluster</em> acknowledges reception of the message.</li>
<li><em>HTTP Adapter</em> acknowledges the reception of the data to the <em>Device</em>.</li>
</ol></li>
</ol>
<p>When a Kafka producer raises an exception while sending a telemetry message to Kafka, the protocol adapter MUST NOT try to re-send such rejected messages but SHOULD indicate the failed transfer to the device if the transport protocol provides means to do so.</p>
<p><strong>Use <em>AT LEAST ONCE</em> and <em>AT MOST ONCE</em> in a Protocol Adapter</strong></p>
<p>If a protocol adapter should support both delivery semantics, a single producer MUST be used and it MUST be configured
for <em>AT LEAST ONCE</em>. It may not wait for acknowledgements of <em>AT MOST ONCE</em> messages.</p>
<p>A protocol adapter MUST NOT use two producers sending telemetry data for the same device.
Otherwise, the messages from a device with different QoS levels could be out of order, as Kafka does not guarantee the
message order between multiple producer instances.</p>
<p><strong>Message Format</strong></p>
<p>The key of the message MUST be the device ID.</p>
<p>Metadata MUST be set as Kafka headers on a message.
The following table provides an overview of the headers a client needs to set on a <em>Produce Telemetry Data</em> message.</p>
<table>
<thead>
<tr>
<th align="left">Name</th>
<th align="center">Mandatory</th>
<th align="left">Type</th>
<th align="left">Description</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left"><em>content-type</em></td>
<td align="center">yes</td>
<td align="left"><em>string</em></td>
<td align="left">A content type indicating the type and characteristics of the data contained in the message value as a valid MIME type, e.g. <code>text/plain; charset=&quot;utf-8&quot;</code> for a text message or <code>application/json</code> etc. The value may be set to <code>application/octet-stream</code> if the message message value is to be considered <em>opaque</em> binary data. See <a href="http://www.ietf.org/rfc/rfc2046.txt">RFC 2046</a> for details.</td>
</tr>
<tr>
<td align="left"><em>content-encoding</em></td>
<td align="center">no</td>
<td align="left"><em>string</em></td>
<td align="left">The content encoding as defined in section 3.5 of <a href="http://www.ietf.org/rfc/rfc2616.txt">RFC 2616</a>.</td>
</tr>
<tr>
<td align="left"><em>creation-time</em></td>
<td align="center">no</td>
<td align="left"><em>long</em></td>
<td align="left">The instant in time (milliseconds since the Unix epoch) when the message has been created. This header is mandatory if <em>ttd</em> or <em>ttl</em> is set, otherwise, it is optional.</td>
</tr>
<tr>
<td align="left"><em>device_id</em></td>
<td align="center">yes</td>
<td align="left"><em>string</em></td>
<td align="left">The identifier of the device that the data in the message value is originating from.</td>
</tr>
<tr>
<td align="left"><em>qos</em></td>
<td align="center">no</td>
<td align="left"><em>int</em></td>
<td align="left">The quality of service level of the message. Supported values are <code>0</code> for AT MOST ONCE and <code>1</code> for AT LEAST ONCE.</td>
</tr>
<tr>
<td align="left"><em>ttd</em></td>
<td align="center">no</td>
<td align="left"><em>int</em></td>
<td align="left">The <em>time &lsquo;til disconnect</em> indicates the number of seconds that the device will remain connected to the protocol adapter. The value of this header must be interpreted relative to the message&rsquo;s <em>creation-time</em>. A value of <code>-1</code> is used to indicate that the device will remain connected until further notice, i.e. until another message indicates a <em>ttd</em> value of <code>0</code>. In absence of this property, the connection status of the device is to be considered indeterminate. <em>Backend Applications</em> might use this information to determine a time window during which the device will be able to receive a command.</td>
</tr>
<tr>
<td align="left"><em>ttl</em></td>
<td align="center">no</td>
<td align="left"><em>long</em></td>
<td align="left">The <em>time-to-live</em> in milliseconds. A consumer of the message SHOULD discard the message if the sum of <em>creation-time</em> and <em>ttl</em> is greater than the current time (milliseconds since the Unix epoch).</td>
</tr>
</tbody>
</table>
<p>Protocol adapters MAY add additional headers to the Kafka record.</p>
<p>The value of the message MUST consist of a byte array containing the telemetry data. The format and encoding of the data MUST be indicated by the <em>content-type</em> and (optional) <em>content-encoding</em> headers of the message.</p>
<h2 id="northbound-operations">Northbound Operations</h2>
<p>The following operation can be used by <em>Business Applications</em> to receive telemetry data from Kafka.</p>
<h3 id="consume-telemetry-data">Consume Telemetry Data</h3>
<p>Hono delivers messages containing telemetry data reported by a particular device in the same order that they have been received in (using the <a href="#produce-telemetry-data">Produce Telemetry Data</a> operation).
<em>Business Applications</em> consume messages from the tenant-specific topic <code>hono.telemetry.${tenant_id}</code> where <code>${tenant_id}</code> represents the ID of the tenant the client wants to retrieve telemetry data for.</p>
<p><strong>Preconditions</strong></p>
<ol>
<li>Either the topic <code>hono.telemetry.${tenant_id}</code> exists, or the broker is configured to automatically create topics on demand.</li>
<li>The client is authorized to read from the topic.</li>
<li>The client subscribes to the topic with a Kafka consumer.</li>
</ol>
<p>A consumer can provide <em>AT MOST ONCE</em> or <em>AT LEAST ONCE</em> delivery semantics depending on the offset commit strategy it implements.
If a message has the <em>qos</em> Kafka header with the value <code>1</code>, the consumer is expected to provide proper
<em>AT LEAST ONCE</em> semantics, and therefore MUST NOT commit the message <em>offset</em> before it has been fully processed.</p>
<p><strong>Message Flow</strong></p>
<p>The following sequence diagram illustrates the flow of messages involved in a <em>Business Application</em> consuming a telemetry data message from the Kafka cluster. The delivery mode used is <em>AT LEAST ONCE</em>.</p>
<figure>
<img src="consume_kafka.svg"/> <figcaption>
<h4>Consume Telemetry Data</h4>
</figcaption>
</figure>
<ol>
<li><em>Business Application</em> polls telemetry messages from <em>Kafka Cluster</em>.
<ol>
<li><em>Kafka Cluster</em> returns a batch of messages.</li>
</ol></li>
<li><em>Business Application</em> processes the messages.</li>
<li><em>Business Application</em> commits the offset of the last processed telemetry message.
<ol>
<li><em>Kafka Cluster</em> acknowledges the commit.</li>
</ol></li>
</ol>
<p><strong>Message Format</strong></p>
<p>The format of the messages containing the telemetry data is the same as for the <a href="#produce-telemetry-data">Produce Telemetry Data operation</a>.</p>
<footer class=" footline" >
</footer>
</div>
</div>
<div id="navigation">
</div>
</section>
<div style="left: -1000px; overflow: scroll; position: absolute; top: -1000px; border: none; box-sizing: content-box; height: 200px; margin: 0px; padding: 0px; width: 200px;">
<div style="border: none; box-sizing: content-box; height: 200px; margin: 0px; padding: 0px; width: 200px;"></div>
</div>
<script src="/hono/docs/js/clipboard.min.js?1610071841"></script>
<script src="/hono/docs/js/perfect-scrollbar.min.js?1610071841"></script>
<script src="/hono/docs/js/perfect-scrollbar.jquery.min.js?1610071841"></script>
<script src="/hono/docs/js/jquery.sticky.js?1610071841"></script>
<script src="/hono/docs/js/featherlight.min.js?1610071841"></script>
<script src="/hono/docs/js/html5shiv-printshiv.min.js?1610071841"></script>
<script src="/hono/docs/js/highlight.pack.js?1610071841"></script>
<script>hljs.initHighlightingOnLoad();</script>
<script src="/hono/docs/js/modernizr.custom.71422.js?1610071841"></script>
<script src="/hono/docs/js/learn.js?1610071841"></script>
<script src="/hono/docs/js/hugo-learn.js?1610071841"></script>
<link href="/hono/docs/mermaid/mermaid.css?1610071841" type="text/css" rel="stylesheet" />
<script src="/hono/docs/mermaid/mermaid.js?1610071841"></script>
<script>
mermaid.initialize({ startOnLoad: true });
</script>
<script>
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-5WLCZXC');
</script>
<script src="https://www.eclipse.org/eclipse.org-common/themes/solstice/public/javascript/vendor/cookieconsent/default.min.js"></script>
</body>
</html>