add new mqtt connector app

apps/mqtt-connector

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+cd $(dirname ${0})
+
+. ./prep-env.sh
+
+cd mqttConnector
+
+PYTHONPATH=${PYTHONPATH}:src exec python -m mqttconnector $@

apps/mqttConnector/MANIFEST.in

@@ -0,0 +1 @@
+include utils.py

apps/mqttConnector/README.md

@@ -0,0 +1,183 @@
+# Introduction
+![](mqtt_connector_diagram.png)
+
+MQTTConnector is able to connect a MQTT-Broker with OpenMTC. The subscribes to specific topics on the broker in order to forward all published data to OpenMTC.
+
+The MQTTConnector will subscribe itself on the MQTT-Broker (e.g. MeshBlu) to a specific first topics level. Published data is expected to contain at least two more topic levels, with one representing the location and one the device. This information will later be used to create an entity within the OpenMTC resource tree. If data is published, the MQTTConnector app will be notified via the subscription and will forward the data to an instance of OpenMTC (gateway or backend).
+
+# Getting started
+
+In order to get an overview of the possible paramters of the app:
+
+```
+./apps/mqtt-connector -v
+```
+
+The most relevant paramters are the following:
+
+EP (Endpoint): URL of the OpenMTC endpoint to use (backend or gateway)
+Example: "http://localhost:18000"
+
+BROKER_EP (MQTT Endpoint): URL of the MQTT-Broker
+Example: "http://localhost:1883"
+
+TOPIC_PRE: definition of the first topic level in order to filter for relevant topics
+Example: "OfficeBuilding"
+
+TOPIC_INDEX_LOCATION: topic level representing the location
+Example: 1
+
+TOPIC_INDEX_DEVICE: topic level representing the device
+Example: 2
+
+BROKER_USER: username to connect to the MQTT Broker
+BROKER_USER_PW: password to connect to the MQTT Broker
+
+If used to together with the OpenMTC OrionContextBroker app, it is also possible to configure a specific fiware service.
+
+# Required Data Format
+
+In order to work data needs to be published to the MQTT-Broker in an onem2m specific format. This will look like the following:
+
+```json
+{
+    "m2m:cin": {
+        "con": B64_ENCODED_SENML_PAYLOAD
+    }
+}
+```
+
+where the B64_ENCODED_SENML_PAYLOAD could look like this:
+
+decoded:
+```json
+[
+    {
+        "bn": "23",
+        "v": 27,
+        "u": "Celsius",
+        "t": 1527757260000,
+        "n": "Temperatur"
+    }
+]
+```
+
+# Example
+
+![](mqtt_connector_ocb_example.png)
+
+In this example setup, we are using the MQTTConnector to forward data of a temperature sensor connected to the Mosquitto MQTT-Broker via OpenMTC to an instance of the Fiware Orion Context Broker. The following docker-compose configuration shows the setup in more detail.
+
+```
+version: "2"
+
+services:
+    backend:
+        image: openmtc/backend-amd64
+        ports:
+            - "18000:18000"
+        environment:
+            - ONEM2M_NOTIFICATION_DISABLED=false
+            - ONEM2M_HTTP_TRANSPORT_DISABLED=false
+            - ONEM2M_HTTP_TRANSPORT_SSL_ENABLED=false
+            - ONEM2M_HTTP_TRANSPORT_REQUIRE_CERT=false
+
+    mosquitto:
+        image: eclipse-mosquitto
+        ports:
+            - "1883:1883"
+
+    mqtt-ipe:
+        image: openmtc/mqttconnector-amd64
+        environment:
+            - EP=http://backend:18000
+            - BROKER_EP=mosquitto:1883
+            - TOPIC_PRE=OfficeBuilding
+            - TOPIC_INDEX_LOCATION=1
+            - TOPIC_INDEX_DEVICE=2
+        links:
+            - mosquitto
+            - backend
+
+    orioncontextbroker-app:
+        image: openmtc/orion-context-broker-app-amd64
+        container_name: orioncontextbroker-app
+        links:
+            - backend
+            - orion
+        ports:
+            - "8086:8086"
+            - "8080:8080"
+        environment:
+            - EP=http://backend:18000
+            - ORION_HOST=http://orion:1026
+            - ORION_API=v2
+            - ACCUMULATE_ADDRESS=http://orioncontextbroker-app:8080
+            - LABELS=["openmtc:sensor_data"]
+
+    mongo:
+        image: mongo:3.4
+        command: --nojournal
+
+    orion:
+        image: fiware/orion
+        ports:
+            - "1026:1026"
+        command: -dbhost mongo -logLevel debug
+        links:
+            - mongo
+```
+
+The following will be published to topic OfficeBuilding/MeetingRoom/temperature:
+
+```json
+{
+    "m2m:cin": {
+        "con": "W3tibjogMjMsIHY6IDI3LCB1OiBDZWxzaXVzLCB0OiAxNTI3NzU3MjYwMDAwLCBuOiBUZW1wZXJhdHVyfV0K"
+    }
+}
+```
+
+The string is in "con" is the base64 representation of:
+
+```json
+[
+    {
+        "bn": "23",
+        "v": 27,
+        "u": "Celsius",
+        "t": 1527757260000,
+        "n": "temperature"
+    }
+]
+```
+
+If the data is successfully published to the MQTT-Broker it will be forwarded to the OCB. Therefore we are able to request the data from the OCB.
+
+```
+curl localhost:1026/v2/entities/MeetingRoom-temperature/attrs -H -s
+```
+
+```json
+{
+  "temperature": {
+    "type": "Float",
+    "value": 27,
+    "metadata": {
+      "bn": {
+        "type": "String",
+        "value": 23
+      },
+      "timestamp": {
+        "type": "String",
+        "value": 152775726000
+      },
+      "unit": {
+        "type": "String",
+        "value": "Celsius"
+      }
+    }
+  }
+}
+
+```

apps/mqttConnector/bin/openmtc-mqtt-connector

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec python -m mqttconnector $@

apps/mqttConnector/config.json

@@ -0,0 +1,28 @@
+{
+    "name": "mqttConnector",
+    "ep": "http://localhost:18000",
+    "cse_base": "onem2m",
+    "poas": [
+        "http://auto:21753"
+    ],
+    "originator_pre": "//openmtc.org/in-cse-1",
+    "ssl_certs": {
+        "cert_file": null,
+        "key_file": null,
+        "ca_certs": null
+    },
+    "broker_ep": "localhost:1883",
+    "topic_pre": "exampleTopic",
+    "topic_index_location": 1,
+    "topic_index_device": -1,
+    "fiware_service": null,
+    "broker_user": "foo",
+    "broker_user_pw": "bar",
+    "mqtts_ca_certs": null,
+    "mqtts_certfile": null,
+    "mqtts_keyfile": null,
+    "logging": {
+        "level": "ERROR",
+        "file": null
+    }
+}

apps/mqttConnector/docker/configure-mqttconnector-and-start

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+CONFIG_FILE="/etc/openmtc/mqttconnector/config.json"
+
+NAME=${NAME-"mqttConnector"}
+EP=${EP-"http://localhost:8000"}
+CSE_BASE=${CSE_BASE-"onem2m"}
+POAS=${POAS-'["http://auto:21753"]'}
+ORIGINATOR_PRE=${ORIGINATOR_PRE-"//openmtc.org/mn-cse-1"}
+SSL_CRT=${SSL_CRT-"/etc/openmtc/certs/mqttconnector.cert.pem"}
+SSL_KEY=${SSL_KEY-"/etc/openmtc/certs/mqttconnector.key.pem"}
+SSL_CA=${SSL_CA-"/etc/openmtc/certs/ca-chain.cert.pem"}
+BROKER_EP=${BROKER_EP-"localhost:1883"}
+TOPIC_PRE=${TOPIC_PRE-"exampleTopic"}
+TOPIC_INDEX_LOCATION=${TOPIC_INDEX_LOCATION}
+TOPIC_INDEX_DEVICE=${TOPIC_INDEX_DEVICE}
+FIWARE_SERVICE=${FIWARE_SERVICE}
+BROKER_USER=${BROKER_USER-"foo"}
+BROKER_USER_PW=${BROKER_USER_PW-"bar"}
+MQTTS_CA_CERTS=${MQTTS_CA_CERTS}
+MQTTS_CERTFILE=${MQTTS_CERTFILE}
+MQTTS_KEYFILE=${MQTTS_KEYFILE}
+
+# defaults logging
+LOGGING_FILE=${LOGGING_FILE-"/var/log/openmtc/mqttconnector.log"}
+LOGGING_LEVEL=${LOGGING_LEVEL-"ERROR"}
+
+# ensure correct level
+case ${LOGGING_LEVEL} in
+    FATAL|ERROR|WARN|INFO|DEBUG)
+    ;;
+    *)
+    LOGGING_LEVEL="ERROR"
+    ;;
+esac
+
+# local ip
+LOCAL_IP=$(ip r get 8.8.8.8 | awk 'NR==1 {print $NF}')
+
+# set hostname
+HOST_NAME=${EXTERNAL_IP-${LOCAL_IP}}
+
+# Configuration of the service.
+CONFIG_TEMP=${CONFIG_FILE}".tmp"
+echo -n "Configuring M2M mqttconnector..."
+JQ_STRING='.'
+
+# basics
+JQ_STRING=${JQ_STRING}' |
+    .name = "'${NAME}'" |
+    .ep = "'${EP}'" |
+    .cse_base = "'${CSE_BASE}'" |
+    .poas = '${POAS}' |
+    .originator_pre = "'${ORIGINATOR_PRE}'" |
+    .ssl_certs.cert_file = "'${SSL_CRT}'" |
+    .ssl_certs.key_file = "'${SSL_KEY}'" |
+    .ssl_certs.ca_certs = "'${SSL_CA}'" |
+    .broker_ep = "'${BROKER_EP}'" |
+    .topic_pre = "'${TOPIC_PRE}'" |
+    .topic_index_location = '${TOPIC_INDEX_LOCATION}' |
+    .topic_index_device = '${TOPIC_INDEX_DEVICE}' |
+    .fiware_service = "'${FIWARE_SERVICE}'" |
+    .broker_user = "'${BROKER_USER}'" |
+    .broker_user_pw = "'${BROKER_USER_PW}'" |
+    .mqtts_ca_certs = "'${MQTTS_CA_CERTS}'" |
+    .mqtts_certfile = "'${MQTTS_CERTFILE}'" |
+    .mqtts_keyfile = "'${MQTTS_KEYFILE}'" |
+    .logging.file |= "'${LOGGING_FILE}'" |
+    .logging.level |= "'${LOGGING_LEVEL}'"
+'
+
+cat ${CONFIG_FILE} | jq -M "${JQ_STRING}"> ${CONFIG_TEMP}
+mv ${CONFIG_TEMP} ${CONFIG_FILE}
+
+echo "done"
+
+exec python -m mqttconnector $@

apps/mqttConnector/docker/mqttconnector-amd64

@@ -0,0 +1,30 @@
+############################################################
+# Dockerfile to run openmtc mqttconnector binary
+############################################################
+
+# Set the base image to use openmtc/sdk
+FROM openmtc/sdk-amd64:latest
+
+ENV MOD_NAME=mqttconnector
+
+# Set the file maintainer
+MAINTAINER rst
+
+# install openmtc dependencies
+COPY tmp/$MOD_NAME-dependencies.txt /tmp/requirements.txt
+RUN pip install --upgrade --requirement /tmp/requirements.txt
+
+# install openmtc-mqttconnector
+COPY tmp/openmtc-$MOD_NAME.tar.gz /tmp/openmtc-$MOD_NAME.tar.gz
+RUN tar xzf /tmp/openmtc-$MOD_NAME.tar.gz -C / \
+    --owner root --group root --no-same-owner --no-overwrite-dir \
+    --transform 's/json\.dist/json/' --show-transformed
+
+RUN mkdir -p /var/log/openmtc
+
+# add change config
+COPY configure-$MOD_NAME-and-start /usr/local/bin/configure-and-start
+
+# entry point
+ENTRYPOINT ["/usr/local/bin/configure-and-start"]
+CMD [""]

apps/mqttConnector/docker/mqttconnector-arm

@@ -0,0 +1,30 @@
+############################################################
+# Dockerfile to run openmtc mqttconnector binary
+############################################################
+
+# Set the base image to use openmtc/sdk
+FROM openmtc/sdk-arm:latest
+
+ENV MOD_NAME=mqttconnector
+
+# Set the file maintainer
+MAINTAINER rst
+
+# install openmtc dependencies
+COPY tmp/$MOD_NAME-dependencies.txt /tmp/requirements.txt
+RUN pip install --upgrade --requirement /tmp/requirements.txt
+
+# install openmtc-mqttconnector
+COPY tmp/openmtc-$MOD_NAME.tar.gz /tmp/openmtc-$MOD_NAME.tar.gz
+RUN tar xzf /tmp/openmtc-$MOD_NAME.tar.gz -C / \
+    --owner root --group root --no-same-owner --no-overwrite-dir \
+    --transform 's/json\.dist/json/' --show-transformed
+
+RUN mkdir -p /var/log/openmtc
+
+# add change config
+COPY configure-$MOD_NAME-and-start /usr/local/bin/configure-and-start
+
+# entry point
+ENTRYPOINT ["/usr/local/bin/configure-and-start"]
+CMD [""]

apps/mqttConnector/etc/conf/config.json.dist

@@ -0,0 +1,28 @@
+{
+    "name": "mqttConnector",
+    "ep": "http://localhost:18000",
+    "cse_base": "onem2m",
+    "poas": [
+        "http://auto:21753"
+    ],
+    "originator_pre": "//openmtc.org/in-cse-1",
+    "ssl_certs": {
+        "cert_file": "/etc/openmtc/certs/mqttconnector.cert.pem",
+        "key_file": "/etc/openmtc/certs/mqttconnector.key.pem",
+        "ca_certs": "/etc/openmtc/certs/ca-chain.cert.pem"
+    },
+    "broker_ep": "localhost:8883",
+    "topic_pre": "exampleTopic",
+    "topic_index_location": 1,
+    "topic_index_device": -1,
+    "fiware_service": null,
+    "broker_user": "foo",
+    "broker_user_pw": "bar",
+    "mqtts_ca_certs": null,
+    "mqtts_certfile": null,
+    "mqtts_keyfile": null,
+    "logging": {
+        "level": "INFO",
+        "file": "/var/log/openmtc/mqttconnector.log"
+    }
+}

apps/mqttConnector/etc/systemd/system/openmtc-mqttconnector.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=OpenMTC mqttConnector
+After=network.target
+Wants=ntp.service
+
+[Service]
+ExecStart=/usr/local/bin/mqtt-connector
+
+[Install]
+WantedBy=multi-user.target

apps/mqttConnector/setup-mqttconnector.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+from distutils.core import setup
+from glob import glob
+import sys
+
+from utils import get_packages, get_pkg_files, OpenMTCSdist, move_config_files
+
+# name and dir
+NAME = "mqttconnector"
+BASE_DIR = "."
+
+# import pkg
+sys.path.append(BASE_DIR + "/src")
+pkg = __import__(NAME)
+
+# setup name and version
+SETUP_NAME = "openmtc-" + NAME
+SETUP_VERSION = pkg.__version__
+SETUP_DESCRIPTION = pkg.__description__
+
+# meta
+SETUP_AUTHOR = pkg.__author_name__
+SETUP_AUTHOR_EMAIL = pkg.__author_mail__
+SETUP_URL = "http://www.openmtc.org"
+SETUP_LICENSE = "Fraunhofer FOKUS proprietary"
+
+# requirements
+SETUP_REQUIRES = pkg.__requires__
+SETUP_INSTALL_REQUIRES = pkg.__requires__
+
+# packages
+PACKAGES = [NAME]
+PACKAGE_DIR = {"": BASE_DIR + "/src"}
+all_packages = []
+for package in PACKAGES:
+    all_packages.extend(get_packages(package, PACKAGE_DIR))
+
+# scripts
+SETUP_SCRIPTS = glob(BASE_DIR + "/bin/*")
+
+# package data
+PACKAGE_DATA = {NAME: get_pkg_files(BASE_DIR, NAME)}
+
+# data files
+CONFIG_FILES = ("config.json",)
+CONFIG_DIR = "/etc/openmtc/" + NAME
+CONFIG_DIST_FILES = (BASE_DIR + "/etc/conf/config.json.dist",)
+DATA_FILES = [(CONFIG_DIR, CONFIG_DIST_FILES)]
+
+# cmd class
+CMD_CLASS = {'sdist': OpenMTCSdist}
+
+if __name__ == "__main__":
+    if 'bdist_wheel' in sys.argv:
+        raise RuntimeError("This setup.py does not support wheels")
+
+    ############################################################################
+    # setup
+    setup(name=SETUP_NAME,
+          version=SETUP_VERSION,
+          description=SETUP_DESCRIPTION,
+          author=SETUP_AUTHOR,
+          author_email=SETUP_AUTHOR_EMAIL,
+          url=SETUP_URL,
+          license=SETUP_LICENSE,
+          requires=SETUP_REQUIRES,
+          install_requires=SETUP_INSTALL_REQUIRES,
+          package_dir=PACKAGE_DIR,
+          packages=all_packages,
+          scripts=SETUP_SCRIPTS,
+          package_data=PACKAGE_DATA,
+          data_files=DATA_FILES,
+          cmdclass=CMD_CLASS
+          )
+
+    ############################################################################
+    # install
+    if "install" in sys.argv:
+        # only do this during install
+        move_config_files(CONFIG_DIR, CONFIG_FILES)

apps/mqttConnector/src/mqttconnector/__init__.py

@@ -0,0 +1,9 @@
+"""
+TODO: Add description here
+"""
+
+__version__ = "ADD-VERSION-HERE"
+__description__ = "mqttConnector"
+__author_name__ = "ADD_AUTHOR_HERE"
+__author_mail__ = "ADD_MAIL_HERE"
+__requires__ = ["paho_mqtt"]

apps/mqttConnector/src/mqttconnector/__main__.py

@@ -0,0 +1,95 @@
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+
+from openmtc_app.util import prepare_app, get_value
+from openmtc_app.runner import AppRunner as Runner
+from .mqtt_connector import mqttConnector
+
+# defaults
+default_name = "mqttConnector"
+default_ep = "http://localhost:8000"
+default_topic_pre = "exampleTopic"
+default_topic_index_location = 1
+default_topic_index_device = -1
+default_fiware_service = None
+default_broker_user = "foo"
+default_broker_user_pw = "bar"
+default_mqtts_ca_certs = None
+default_mqtts_certfile = None
+default_mqtts_keyfile = None
+
+# args parser
+parser = ArgumentParser(
+    description="An IPE called mqttConnector",
+    prog="mqttConnector",
+    formatter_class=ArgumentDefaultsHelpFormatter)
+parser.add_argument("-n", "--name", help="Name used for the AE.")
+parser.add_argument("-s", "--ep", help="URL of the local Endpoint.")
+parser.add_argument("--topic_pre", help="Topic you want to be subscribed to")
+parser.add_argument(
+    "--topic_index_location", help="Index of location in topic string")
+parser.add_argument(
+    "--topic_index_device", help="Index of device name in topic string")
+parser.add_argument("--broker_user", help="credentials for MQTT broker")
+parser.add_argument("--broker_user_pw", help="credentials for MQTT broker")
+parser.add_argument("--mqtts", dest='mqtts_enabled', action='store_true')
+parser.add_argument("--no-mqtts", dest='mqtts_enabled', action='store_false')
+parser.set_defaults(mqtts_enabled=False)
+parser.add_argument(
+    "--mqtts_ca_certs", help="Path to CA certs or tuple of paths")
+parser.add_argument("--mqtts_certfile", help="Path to own mqtts cert")
+parser.add_argument("--mqtts_keyfile", help="Path to own mqtts key")
+# args, config and logging
+args, config = prepare_app(parser, __loader__, __name__, "config.json")
+
+# variables
+nm = get_value("name", (unicode, str), default_name, args, config)
+cb = config.get("cse_base", "onem2m")
+ep = get_value("ep", (unicode, str), default_ep, args, config)
+poas = config.get("poas", ["http://auto:21753"])
+originator_pre = config.get("originator_pre", "//openmtc.org/mn-cse-1")
+ssl_certs = config.get("ssl_certs", {})
+
+broker_ep = config.get("broker_ep", "openmtc.smartorchestra.de:8883")
+
+topic_pre = get_value("topic_pre", (unicode, str), default_topic_pre, args,
+                      config)
+topic_index_location = get_value("topic_index_location", (int),
+                                 default_topic_index_location, args, config)
+topic_index_device = get_value("topic_index_device", (int),
+                               default_topic_index_device, args, config)
+fiware_service = get_value("fiware_service", (unicode, str),
+                           default_fiware_service, args, config)
+broker_user = get_value("broker_user", (unicode, str), default_broker_user,
+                        args, config)
+broker_user_pw = get_value("broker_user_pw", (unicode, str),
+                           default_broker_user_pw, args, config)
+user_pw = get_value("broker_user_pw", (unicode, str), default_broker_user_pw,
+                    args, config)
+mqtts_enabled = get_value("mqtts_enabled", (bool), False, args, config)
+mqtts_ca_certs = get_value("mqtts_ca_certs", (unicode, str),
+                           default_mqtts_ca_certs, args, config)
+mqtts_certfile = get_value("mqtts_certfile", (unicode, str),
+                           default_mqtts_certfile, args, config)
+mqtts_keyfile = get_value("mqtts_keyfile", (unicode, str),
+                          default_mqtts_keyfile, args, config)
+# start
+app = mqttConnector(
+    broker_ep=broker_ep,
+    topic_pre=topic_pre,
+    broker_user=broker_user,
+    broker_user_pw=broker_user_pw,
+    topic_index_location=topic_index_location,
+    topic_index_device=topic_index_device,
+    fiware_service=fiware_service,
+    mqtts_enabled=mqtts_enabled,
+    mqtts_ca_certs=mqtts_ca_certs,
+    mqtts_certfile=mqtts_certfile,
+    mqtts_keyfile=mqtts_keyfile,
+    name=nm,
+    cse_base=cb,
+    poas=poas,
+    originator_pre=originator_pre,
+    **ssl_certs)
+Runner(app).run(ep)
+
+print("Exiting....")

apps/mqttConnector/src/mqttconnector/mqtt_connector.py

@@ -0,0 +1,127 @@
+import re
+from base64 import b64decode as base64decode
+from itertools import groupby
+from json import loads as json_decode
+from os import path as ospath
+
+from paho.mqtt import client as mqtt
+
+from openmtc_app.onem2m import XAE
+
+
+class mqttConnector(XAE):
+    interval = 10
+    _location_containers = {}
+    _device_containers = {}
+    _sensor_containers = {}
+    client = mqtt.Client()
+
+    def __init__(self,
+                 broker_ep,
+                 topic_pre,
+                 broker_user,
+                 broker_user_pw,
+                 topic_index_location=1,
+                 topic_index_device=-1,
+                 fiware_service=None,
+                 mqtts_enabled=False,
+                 mqtts_ca_certs=None,
+                 mqtts_certfile=None,
+                 mqtts_keyfile=None,
+                 *args,
+                 **kw):
+        host, port = broker_ep.split(":")
+        self._broker_host = host
+        self._broker_port = int(port)
+
+        self.topic_pre = topic_pre
+        self.topic_index_location = topic_index_location
+        self.topic_index_device = topic_index_device
+        self.fiware_service = fiware_service
+        self.broker_user = broker_user
+        self.broker_user_pw = broker_user_pw
+        self.mqtts_enabled = mqtts_enabled
+        self.mqtts_ca_certs = mqtts_ca_certs
+        self.mqtts_certfile = mqtts_certfile
+        self.mqtts_keyfile = mqtts_keyfile
+
+        super(mqttConnector, self).__init__(*args, **kw)
+
+    def _on_register(self):
+        def on_connect(*_):
+            def callback(*args):
+                (_, _, message) = args
+                self.logger.info(
+                    'Received message on topic %s' % message.topic)
+                self._handle_data(message.topic, message.payload)
+
+            topics = ['%s/#' % self.topic_pre]
+
+            for topic in topics:
+                self.client.message_callback_add(topic, callback)
+
+            self.client.subscribe([(topic, 1) for topic in topics])
+
+        self.client.on_connect = on_connect
+
+        # TODO(rst): this needs to be handled more general and from config
+        self.client.username_pw_set(self.broker_user, self.broker_user_pw)
+        if self.mqtts_enabled:
+            self.client.tls_set(
+                ca_certs=self.mqtts_ca_certs,
+                certfile=self.mqtts_certfile,
+                keyfile=self.mqtts_keyfile)
+            self.client.tls_insecure_set(True)
+        self.client.connect(self._broker_host, self._broker_port)
+        # TODO let gevent handle this
+        self.client.loop_forever()
+
+    def _on_shutdown(self):
+        self.client.disconnect()
+
+    def _get_target_container(self, location, device, sensor):
+        try:
+            return self._sensor_containers[(location, device, sensor)]
+        except KeyError:
+            try:
+                device_cnt = self._device_containers[(location, device)]
+            except KeyError:
+                try:
+                    location_cnt = self._location_containers[location]
+                except KeyError:
+                    location_cnt = self.create_container(None, location)
+                    self._location_containers[location] = location_cnt
+                device_cnt = self.create_container(
+                    location_cnt, device, labels=["openmtc:device"])
+                self._device_containers[(location, device)] = device_cnt
+            openmtc_id = "%s/%s/%s" % (
+                (self.fiware_service + '~' if self.fiware_service else '') +
+                location, device, sensor)
+            labels = ['openmtc:sensor_data', 'openmtc:id:%s' % openmtc_id]
+            sensor_cnt = self.create_container(
+                device_cnt, sensor, labels=labels)
+            self._sensor_containers[(location, device, sensor)] = sensor_cnt
+            return sensor_cnt
+
+    def _handle_data(self, topic, payload):
+        # get location and device
+        try:
+            location = topic.split('/')[self.topic_index_location]
+            device = topic.split('/')[self.topic_index_device]
+        except (AttributeError, ValueError):
+            self.logger.error("Topic '%s' not valid. Dropping." % topic)
+            return
+
+        # check payload
+        try:
+            readings = json_decode(
+                base64decode(json_decode(payload)['m2m:cin']['con']))
+        except (ValueError, KeyError, TypeError):
+            self.logger.error('Damaged payload; discarding')
+            return
+
+        # push data
+        for _, values in groupby(readings, key=lambda x: x['n']):
+            sensor_cnt = self._get_target_container(location, device, 'number')
+            for value in sorted(values, key=lambda x: x['t']):
+                self.push_content(sensor_cnt, [value])

apps/mqttConnector/utils.py

@@ -0,0 +1,148 @@
+import distutils.command.sdist
+import distutils.command.build_py
+import os
+import subprocess
+import sys
+
+
+def echo(msg, *args):
+    if args:
+        msg = msg % args
+    sys.stdout.write(msg + "\n")
+
+
+def get_packages(package, package_dir, excluded_list=None, included_list=None):
+    included_list = included_list or []
+    excluded_list = excluded_list or []
+
+    try:
+        root = package_dir[package]
+    except KeyError:
+        root = package_dir.get("", ".") + "/" + package
+
+    if not os.path.exists(root):
+        sys.stderr.write(
+            "Directory for package %s does not exist: %s\n" % (package, root))
+        sys.exit(1)
+
+    def on_error(error):
+        sys.stderr.write(
+            "Error while collecting packages for %s: %s\n" % (package, error))
+        sys.exit(1)
+
+    packages = [package]
+
+    r_prefix = len(root) + 1
+    for path, dirs, files in os.walk(root, onerror=on_error):
+        is_module = "__init__.py" in files and path != root
+        excluded = any(map(lambda x: x in path, excluded_list))
+        included = any(map(lambda x: x in path, included_list))
+        if is_module and (not excluded or included):
+            packages.append(package + "." + path[r_prefix:].replace("/", "."))
+
+    return packages
+
+
+def get_pkg_files(base_dir, name):
+    package_files = []
+    pkg_dir = os.path.join(base_dir, 'src', name)
+    pkg_data_dir = os.path.join(pkg_dir, 'static')
+    for (path, directories, filenames) in os.walk(pkg_data_dir):
+        for filename in filenames:
+            package_files.append(os.path.join(os.path.relpath(path, pkg_dir),
+                                              filename))
+    return package_files
+
+
+def enable_init_files(init_dir, init_dist_files):
+    for f in init_dist_files:
+        os.chmod(os.path.join(init_dir, os.path.basename(f)), 0755)
+
+
+def move_config_files(config_dir, config_files):
+    for f in config_files:
+        target_file = os.path.join(config_dir, f)
+        if not os.path.exists(target_file):
+            echo("Installing config file %s", target_file)
+            os.rename(target_file + ".dist", target_file)
+            # os.chmod(target_file, 0644)
+        else:
+            echo("Not overwriting config file %s", target_file)
+
+
+def create_openmtc_user(db_dir=None, log_dir=None):
+    try:
+        from pwd import getpwnam
+    except ImportError:
+        print "Could not import the 'pwd' module. Skipping user management"
+    else:
+        # assuming DB_DIR was created by setup already
+        try:
+            pw = getpwnam('openmtc')
+        except KeyError as e:
+            try:
+                # add system user openmtc:openmtc
+                # useradd --system -UM openmtc
+                useradd = "useradd --system -UM openmtc"
+                retcode = subprocess.call(useradd, shell=True)
+                if retcode:
+                    raise Exception("Failed to add user 'openmtc'")
+                pw = getpwnam('openmtc')
+            except Exception as e:
+                sys.stderr.write("Error creating user: %s\n" % (e, ))
+                sys.exit(1)
+        uid = pw.pw_uid
+        gid = pw.pw_gid
+
+        # set path permissions
+        if db_dir:
+            os.chown(db_dir, uid, gid)
+        if log_dir:
+            os.chown(log_dir, uid, gid)
+
+
+class OpenMTCSdist(distutils.command.sdist.sdist):
+    def make_release_tree(self, base_dir, files):
+        distutils.command.sdist.sdist.make_release_tree(self, base_dir, files)
+
+        script_name = os.path.basename(sys.argv[0])
+
+        if script_name != "setup.py":
+            os.rename(base_dir + "/" + script_name, base_dir + "/setup.py")
+            self.filelist.files.remove(script_name)
+            self.filelist.files.append("setup.py")
+
+
+class OpenMTCSdistBinary(OpenMTCSdist, object):
+    def make_release_tree(self, base_dir, files):
+        super(OpenMTCSdistBinary, self).make_release_tree(base_dir, files)
+
+        script_name = os.path.basename(sys.argv[0])
+
+        build_py = self.get_finalized_command('build_py')
+        build_py.compile = 1
+        build_py.optimize = 2
+        build_py.retain_init_py = 1
+        build_py.build_lib = base_dir
+        build_py.byte_compile(
+            [base_dir + "/" + f for f in self.filelist.files if
+             f != script_name and f.endswith(".py")])
+
+
+class OpenMTCBuildPy(distutils.command.build_py.build_py):
+    retain_init_py = 0
+
+    def byte_compile(self, files):
+        distutils.command.build_py.build_py.byte_compile(self, files)
+
+
+class OpenMTCBuildPyBinary(OpenMTCBuildPy, object):
+    retain_init_py = 0
+
+    def byte_compile(self, files):
+        super(OpenMTCBuildPyBinary, self).byte_compile(files)
+
+        for f in files:
+            if (f.endswith('.py') and (os.path.basename(f) != "__init__.py" or
+                                       not self.retain_init_py)):
+                os.unlink(f)