initial commit for the persistence feature

Test debian stretch+python3 (#18 )
* changes starting with python3 explicit * removes python modules which are not available for python3 * exchanges fyzz query parsing with rdflib functionality * fixes interop tests * replaces reduce with for loop in nodb driver * simple python2 -> python3 conversions * adds changes for handling different string handling in python3 * test stretch building with travis * installing python-setuptools in docker * installing python-setuptools in docker * changing python2 to python3 in docker makefiles * changing python2 to python3 and some other test changes * push docker only in master branche * running version of openmtc * fix some port problems * porting path library completly now * restoring travis.yml * testing new travis.yml * add sudo * updating travis OS from trusty to xenial * upgrade pip before * show running docker logs * show more logs * for debugging * showlogs of docker after failure * testing new travis.yml * finish travis.yml
2025-06-24 08:20:30 +00:00 · 2019-01-18 16:53:39 +01:00 · 2019-01-18 14:13:03 +01:00 · 2019-01-18 13:47:25 +01:00
23 changed files with 189 additions and 140 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@ -1,23 +0,0 @@
---
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
---
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
-
-**Additional context**
-Add any other context about the problem here.
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@ -1,20 +0,0 @@
---
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
---
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.
--- a/.travis.yml
+++ b/.travis.yml
@ -1,26 +1,26 @@
 services:
-  - docker
+- docker
 dist: xenial
 before_script:
-  - sudo apt update
-  - sudo apt install qemu-user-static python3 python3-pip python3-setuptools
-  - sudo pip3 install --upgrade pip
-  - sudo python3 -m pip install --upgrade setuptools
-  - sudo python3 -m pip install --upgrade pyresttest
-  - "./create-binary-docker backend"
-  - "./create-binary-docker backend -a arm"
-  - "./create-binary-docker gateway"
-  - "./create-binary-docker gateway -a arm"
-  - "./create-binary-docker orioncontextbroker"
-  - "./create-binary-docker orioncontextbroker -a arm"
-  - "./create-binary-docker influxdbapp"
-  - "./create-binary-docker influxdbapp -a arm"
-  - "./create-binary-docker cul868ipe"
-  - "./create-binary-docker cul868ipe -a arm"
-  - docker run -d --name backend -p 0.0.0.0:18000:18000 -e "ONEM2M_CSE_ID=backend" -e "ONEM2M_NOTIFICATION_DISABLED=false" openmtc/backend-amd64 -v
-  - docker logs backend
-  - docker run -d --name gateway -p 0.0.0.0:8000:8000 -e "ONEM2M_HTTP_TRANSPORT_PORT=8000" -e "ONEM2M_CSE_ID=gateway" -e "ONEM2M_REMOTE_CSE_POA=<POA>" -e "ONEM2M_REMOTE_CSE_ID=backend" -e "ONEM2M_NOTIFICATION_DISABLED=false" -e "ONEM2M_REGISTRATION_DISABLED=false" openmtc/gateway-amd64 -v
-  - docker logs gateway
+- sudo apt update
+- sudo apt install qemu-user-static python3 python3-pip python3-setuptools
+- sudo pip3 install --upgrade pip
+- sudo python3 -m pip install --upgrade setuptools
+- sudo python3 -m pip install --upgrade pyresttest
+- "./create-binary-docker backend"
+- "./create-binary-docker backend -a arm"
+- "./create-binary-docker gateway"
+- "./create-binary-docker gateway -a arm"
+- "./create-binary-docker orioncontextbroker"
+- "./create-binary-docker orioncontextbroker -a arm"
+- "./create-binary-docker influxdbapp"
+- "./create-binary-docker influxdbapp -a arm"
+- "./create-binary-docker cul868ipe"
+- "./create-binary-docker cul868ipe -a arm"
+- docker run -d --name backend -p 0.0.0.0:18000:18000 -e "ONEM2M_CSE_ID=backend" -e "ONEM2M_NOTIFICATION_DISABLED=false" openmtc/backend-amd64 -v
+- docker logs backend
+- docker run -d --name gateway -p 0.0.0.0:8000:8000 -e "ONEM2M_HTTP_TRANSPORT_PORT=8000" -e "ONEM2M_CSE_ID=gateway" -e "ONEM2M_REMOTE_CSE_POA=<POA>" -e "ONEM2M_REMOTE_CSE_ID=backend" -e "ONEM2M_NOTIFICATION_DISABLED=false" -e "ONEM2M_REGISTRATION_DISABLED=false" openmtc/gateway-amd64 -v
+- docker logs gateway
 script:
  - docker tag openmtc/orioncontextbroker-amd64 openmtc/orion-context-broker-app-amd64
  - docker tag openmtc/orioncontextbroker-arm openmtc/orion-context-broker-app-arm
@ -35,21 +35,6 @@ after_script:
 env:
  matrix:
    secure: oQe/MxxFrPPArxb6OFzOUwG2ZlA5GYekMR6qn0Y3101v82MdrIVgDILHR41iwykTtk1XpJPV9uabsRsvY4hBmbjYzihYZFOAzFf+/KU7wROtKum0fFfLjCPb8uGV41k1JTquB94FibXboVAP7rNL0Vrpl+FGUvMGatQBtwZnGvha6Ha07qTao+X9+0dJ4YvFlvcba/jfzBZBwHw7KHpRoiyKU0dPmXpHcqnZBFcNqRXhzdVrgx0auP/tghoshW5LQWkpTV11uSx/kYuby4oo8r8nB6L0rW1jYSXs9DHiTQCfCy24xlb9YJjCD9aFcjH0lIkVIqQwJYA67MzKYMK4XV684J/Jr3+jfVOoUt0bpZaTnk+r/uiFCtEsN7q0KUlvHLUAi1YNJhKs6CRrAH3GK25QByh+suzPzZoHP42F2LYP9URlzbDH+/v7CwNuw+9pHSmxEhs18LVmggwkMos3o3ArvzMLRNJ3QG1fdoxL/Ubqxhmvhy4rZi6vGQsiF64oX8PN0sRgXekTU/ma+6CTM6qhgkocMUYzT2r/6qd/9R3jmtxSZnikjkR6Iu9NLzAUY+cqnvvjohAvqYyj+tGlQfOPxAU9H4wXnbQWLBM8yn6yt2Ki+3+Jx4Owdv0oXVJpN0GWjMzIJ1nEus5odkWW+c4wcZb6m3Ak0PrzYPp2BgE=
-after_success:
-  - if [ "$TRAVIS_BRANCH" == "master" ]; then
-    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
-    docker push openmtc/backend-amd64;
-    docker push openmtc/backend-arm;
-    docker push openmtc/gateway-amd64;
-    docker push openmtc/gateway-arm;
-    docker push openmtc/orion-context-broker-app-amd64;
-    docker push openmtc/orion-context-broker-app-arm;
-    docker push openmtc/cul868-ipe-amd64;
-    docker push openmtc/cul868-ipe-arm;
-    docker push openmtc/influxdb-app-amd64;
-    docker push openmtc/influxdb-app-arm;
-    fi
 after_failure:
  - docker logs backend
-  - docker logs gateway
-  - coveralls
+  - docker logs gateway
--- a/CREDITS.md
+++ b/CREDITS.md
@ -1,6 +0,0 @@
-# Contributors ordered by number of commits
-
-Ronald Steinke <rst-fokus>  
-Christian Klopp <ckl-fokus>  
-Alexander Ortlieb <aor-fokus>  
-Jason Fox <jason-fox>  
--- a/README.md
+++ b/README.md
@ -6,14 +6,12 @@

 [![FIWARE IoT Agents](https://nexus.lab.fiware.org/static/badges/chapters/iot-agents.svg)](https://www.fiware.org/developers/catalogue/)
 [![License: Eclipse](https://img.shields.io/github/license/OpenMTC/OpenMTC.svg)](https://www.eclipse.org/legal/epl-v10.html)
-[![](https://img.shields.io/docker/pulls/openmtc/gateway-amd64.svg)](https://hub.docker.com/u/openmtc)
 [![Support badge](https://nexus.lab.fiware.org/repository/raw/public/badges/stackoverflow/iot-agents.svg)](https://stackoverflow.com/questions/tagged/fiware+iot)
 <br/>
 [![Documentation badge](https://img.shields.io/readthedocs/fiware-openmtc.svg)](http://fiware-openmtc.readthedocs.org/en/latest/?badge=latest)
-[![Build Status](https://travis-ci.org/OpenMTC/OpenMTC.svg?branch=master)](https://travis-ci.org/OpenMTC/OpenMTC)
-[![Coverage Status](https://coveralls.io/repos/github/OpenMTC/OpenMTC/badge.svg?branch=master)](https://coveralls.io/github/OpenMTC/OpenMTC?branch=master)
-[![Known Vulnerabilities](https://snyk.io/test/github/OpenMTC/OpenMTC/badge.svg?targetFile=openmtc-gevent%2Frequirements.txt)](https://snyk.io/test/github/OpenMTC/OpenMTC?targetFile=openmtc-gevent%2Frequirements.txt)
 ![Status](https://nexus.lab.fiware.org/static/badges/statuses/iot-openmtc.svg)
+[![](https://img.shields.io/docker/pulls/openmtc/gateway-amd64.svg)](https://hub.docker.com/u/openmtc)
+[![Build Status](https://travis-ci.org/OpenMTC/OpenMTC.svg?branch=master)](https://travis-ci.org/OpenMTC/OpenMTC)

 The OpenMTC SDK aims to provide developers with a convenient yet flexible tool
 to write oneM2M compliant applications. This includes network applications
@ -24,17 +22,12 @@ This project is part of [FIWARE](https://www.fiware.org/). For more information
 check the FIWARE Catalogue entry for the
 [IoT Agents](https://github.com/Fiware/catalogue/tree/master/iot-agents).

-| :octocat: [Git Repository](https://github.com/OpenMTC/OpenMTC) | :whale: [Docker Hub](https://hub.docker.com/u/openmtc) | :books: [Documentation](https://fiware-openmtc.readthedocs.io/) | :page_facing_up: [Site](https://www.openmtc.org/) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/iot-agents/open-mtc) |
-| -------------------------------------------------------------- | ------------------------------------------------------ | --------------------------------------------------------------- | ------------------------------------------------- | ----------------------------------------------------------------------------------------- |
-
 # Content

 -   [Install](#install)
 -   [Usage](#usage)
 -   [API](#api)
-   [Training](#training)
 -   [Quality Assurance](#quality-assurance)
-   [Contributing](#contributing)
 -   [License](#license)


@ -49,11 +42,6 @@ corresponding section of the
 Information about how to use the IoT Agent can be found in the
 [User & Programmers Manual](https://fiware-openmtc.readthedocs.io/en/latest/introduction).

-## Training
-
-Information about our Training can be found in the
-[Training Section](https://fiware-openmtc.readthedocs.io/en/latest/training/training-ipe-sensors/index.html).
-
 ## API

 Information about the REST API can be found in the
@ -101,14 +89,10 @@ follows:

 ---

-## Contributing
-
-Contribution guidelines are detailed in the [CREDITS](CREDITS.md) file.
-
-## License
+## Licence

 The OpenMTC SDK is licensed under the Eclipse Public License (EPL)
 version 1.

-© 2018-2019 OpenMTC
+© 2018 OpenMTC

--- a/apps/InfluxdbApp/src/influxdbapp/init.py
+++ b/apps/InfluxdbApp/src/influxdbapp/init.py
@ -2,7 +2,7 @@
 Transfer OpenMTC Data to an InfluxDB
 """

-__version__ = "1.3.0"
+__version__ = "1.2.0"
 __description__ = "InfluxdbApp"
 __author_name__ = "Christian Klopp"
 __author_mail__ = "christian.klopp@fokus.fraunhofer.de"
--- a/apps/OrionContextBroker/src/orioncontextbroker/init.py
+++ b/apps/OrionContextBroker/src/orioncontextbroker/init.py
@ -3,7 +3,7 @@ This App will forward all incoming sensor traffic to the Fiware Orion Context
 Broker
 """

-__version__ = "1.3.0"
+__version__ = "1.2.0"
 __description__ = "OrionContextBroker"
 __author_name__ = "Christian Klopp"
 __author_mail__ = "christian.klopp@fokus.fraunhofer.de"
--- a/apps/OrionContextBroker/src/orioncontextbroker/orion_api.py
+++ b/apps/OrionContextBroker/src/orioncontextbroker/orion_api.py
@ -1,6 +1,5 @@
 from urllib.parse import urljoin
 import logging
-from datetime import datetime

 import requests

@ -90,8 +89,7 @@ class OrionAPI(LoggerMixin):
                "type": self._get_type(data_senml["v"]),
                "metadata": {
                    "timestamp": {
-                        "value": datetime.fromtimestamp(float(data_senml["t"])).replace(microsecond=0).isoformat()
-                        if data_senml["t"] != "none" else data_senml["t"],
+                        "value": data_senml["t"],
                        "type": "String"
                    },
                    "bn": {
--- a/apps/csvInjector/src/csvinjector/init.py
+++ b/apps/csvInjector/src/csvinjector/init.py
@ -2,7 +2,7 @@
 App to inject data from a csv file to OpenMTC
 """

-__version__ = "1.3.0"
+__version__ = "1.2.0"
 __description__ = "csvInjector"
 __author_name__ = "Christian Klopp"
 __author_mail__ = "christian.klopp@fokus.fraunhofer.de"
--- a/apps/mqttConnector/src/mqttconnector/init.py
+++ b/apps/mqttConnector/src/mqttconnector/init.py
@ -2,7 +2,7 @@
 TODO: Add description here
 """

-__version__ = "1.3.0"
+__version__ = "1.2.0"
 __description__ = "mqttConnector"
 __author_name__ = "Ronald Steinke"
 __author_mail__ = "ronald.steinke@fokus.fraunhofer.de"
--- a/doc/example-apps/IoT-data-visualization.py
+++ b/doc/example-apps/IoT-data-visualization.py
@ -43,6 +43,6 @@ if __name__ == "__main__":
    from openmtc_app.flask_runner import SimpleFlaskRunner as Runner

    ep = "http://localhost:8000"
-    Runner(DataVisualization(poas=['http://localhost:21345'])).run(ep)
+    Runner(DataVisualization(), port=6050, host='auto').run(ep)


--- a/doc/example-apps/data-aggregation.py
+++ b/doc/example-apps/data-aggregation.py
@ -91,4 +91,4 @@ if __name__ == "__main__":
    from openmtc_app.flask_runner import SimpleFlaskRunner as Runner

    ep = "http://localhost:8000"
-    Runner(DataAggregation(poas=['http://localhost:21346'])).run(ep)
+    Runner(DataAggregation(), port=6050, host='auto').run(ep)
--- a/doc/example-apps/simple-decision-2.py
+++ b/doc/example-apps/simple-decision-2.py
@ -19,4 +19,4 @@ if __name__ == "__main__":
    from openmtc_app.flask_runner import SimpleFlaskRunner as Runner

    ep = "http://localhost:8000"
-    Runner(SimpleDecision2(poas=['http://localhost:21387'])).run(ep)
+    Runner(SimpleDecision2(), port=6050, host='auto').run(ep)
--- a/doc/example-apps/simple-decision.py
+++ b/doc/example-apps/simple-decision.py
@ -47,4 +47,4 @@ if __name__ == "__main__":
    from openmtc_app.flask_runner import SimpleFlaskRunner as Runner

    ep = "http://localhost:8000"
-    Runner(SimpleDecision(poas=['http://localhost:22245'])).run(ep)
+    Runner(SimpleDecision(), port=6050, host='auto').run(ep)
--- a/doc/example-apps/start-simple-app
+++ b/doc/example-apps/start-simple-app
@ -31,6 +31,5 @@ app_file=${app_array[$[${choice}-1]]}
 ################################################################################
 # run app_file
 cd ${base_path}
-cd ..
-. ../common/prep-env.sh
-python3 ${app_file}
+. ../../common/prep-env.sh
+python ${app_file}
--- a/ipes/CUL868IPE/src/cul868ipe/init.py
+++ b/ipes/CUL868IPE/src/cul868ipe/init.py
@ -2,7 +2,7 @@
 Interworking Proxy for Cul868 devices.
 """

-__version__ = "1.3.0"
+__version__ = "1.2.0"
 __description__ = "The OpenMTC Cul868IPE"
 __author_name__ = "Ronny Kreuch"
 __author_mail__ = "ronny.kreuch@fokus.fraunhofer.de"
--- a/openmtc-gevent/config-gateway.json
+++ b/openmtc-gevent/config-gateway.json
@ -7,7 +7,7 @@
        "default_content_type": "application/json"
    },
    "database": {
-        "driver": "openmtc_server.db.nodb2.NoDB2",
+        "driver": "openmtc_server.db.mongodb.MongoDB",
        "dropDB": true
    },
    "logging": {
--- a/openmtc-gevent/requirements.txt
+++ b/openmtc-gevent/requirements.txt
@ -1 +0,0 @@
-dependencies.txt
--- a/roadmap.md
+++ b/roadmap.md
@ -1,15 +0,0 @@
-## Short Term
-onem2m Release 2a compatibility  
-Group Resource  
-Pub/Sub improvements (rateLimit and batchNotify)  
-Persistence Enhancement  
-
-## Medium Term
-onem2m Release 3 compatibility  
-Announcements  
-FlexContainer Resource  
-Node Resource  
-
-## Long Term
-CoAP(s) & MQTTS  
-XML  
--- a/server/openmtc-server/src/openmtc_server/db/mongodb.py
+++ b/server/openmtc-server/src/openmtc_server/db/mongodb.py
@ -0,0 +1,148 @@
+
+from openmtc_server.db import DBAdapter, Shelve, DBError
+from openmtc_server.db import BasicSession
+from copy import copy
+from collections import defaultdict, OrderedDict
+from openmtc_server.db.exc import DBConflict, DBNotFound
+from openmtc_onem2m.model import OneM2MResource, CSEBase, AE, Container, ContentInstance, Subscription, RemoteCSE
+from pymongo import MongoClient
+
+res_type = {'2' : AE,
+            '3' : Container,
+            '4' : ContentInstance,
+            '5' : CSEBase,
+            '16' : RemoteCSE,
+            '23' : Subscription}
+
+class MongoDBSession(BasicSession):
+
+
+    def __init__(self, db, std_type, *args, **kw):
+
+        self.db = db
+        self.std_type = std_type
+        self.resources = db['resources']
+
+
+
+
+    def store(self, resource):
+
+        path = resource.path
+        resource_type = type(resource)
+        document = resource.values
+        document ['path'] = path
+
+        self.logger.debug("Adding resource to db: %s -> %s (%s)",
+                          path, resource, resource_type)
+
+        self.resources.insert(document)
+
+    def _get(self, path):
+        pass
+
+
+    def get(self, path):
+
+        document = self.resources.find_one({'path': path}, {'_id': False})
+        if document is None:
+            document = self.resources.find_one({'resourceID':path}, {'_id': False})
+            if document is None:
+                raise DBNotFound
+        res = self.func (document)
+        return res
+
+    def func (self, document):
+        res_type_value = document['resourceType']
+
+        modelclass = res_type[str(res_type_value)]
+        #print modelclass(**document)
+        return modelclass(**document)
+
+
+    def get_collection(self, resource_type, parent, filter_criteria=None):
+        self.logger.debug("Getting %s children of %s (%s)", resource_type,
+                          parent, parent.__model_name__)
+        parentID = parent.resourceID
+        find_children = self.resources.find({'parentID':parentID}, {'_id': False})
+        children = map (self.func, find_children)
+        return children
+
+
+
+    def exists(self, resource_type, fields):
+        self.logger.debug("Checking existence of %s with %s", resource_type,
+                          fields)
+        pass
+
+    def update(self, resource, fields=None):
+
+        document = resource.values
+        path = resource.path
+        #        document ['path'] = path
+
+        #field = document [fields]
+
+        if fields == None or fields == []:
+            document = document
+        else:
+            # change that document
+            pass
+
+
+        modify = self.resources.find_one_and_update({'path': path}, {'$set': document })
+
+        #print doc
+
+
+        #document[fields] = fields
+
+
+
+    def delete(self, resource):
+        document = resource.values
+        self.resources.remove(document)
+
+    def commit(self):
+        pass
+
+    def rollback(self):
+        pass
+
+
+class MongoDBShelve(dict, Shelve):
+    def commit(self):
+        pass
+
+    def rollback(self):
+        pass
+
+
+class MongoDB(DBAdapter):
+    def __init__(self, *args, **kw):
+        super(MongoDB, self).__init__(*args, **kw)
+
+        client = MongoClient('localhost', 27017)
+        client.drop_database('mongodb')
+        self.db = client['mongodb']
+        #self.db.drop_collection()
+
+        self.onem2m_resources = None
+        self.shelves = None
+        self.initialized = False
+
+    def initialize(self, force=False):
+        if not force and self.is_initialized():
+            raise Exception("Already initialized")
+        #self.onem2m_resources = None
+        self.shelves = defaultdict(MongoDBShelve)
+        self.initialized = True
+
+    def get_shelve(self, name):
+        return self.shelves[name]
+
+    def start_session(self, std_type):
+        return MongoDBSession(self.db, std_type)
+
+    def is_initialized(self):
+        return self.initialized
--- a/server/openmtc-server/src/openmtc_server/db/nodb2.py
+++ b/server/openmtc-server/src/openmtc_server/db/nodb2.py
@ -57,7 +57,7 @@ class NoDB2Session(BasicSession):
        assert path is not None
        self.logger.debug("Getting resource: %s", path)
        resource = self._get(path)
-        return copy(resource)
+        return copy(self._get(path))

    def get_collection(self, resource_type, parent, filter_criteria=None):
        self.logger.debug("Getting %s children of %s (%s)", resource_type,
--- a/setup-gevent-all.py
+++ b/setup-gevent-all.py
@ -13,7 +13,7 @@ from utils import (get_packages, OpenMTCSdist, OpenMTCBuildPy,

 # name and version
 SETUP_NAME = "openmtc-all"
-SETUP_VERSION = "1.3.0"
+SETUP_VERSION = "1.2.0"
 SETUP_DESCRIPTION = "The OpenMTC Backend and Gateway (GEvent version)"

 # meta
--- a/setup-sdk.py
+++ b/setup-sdk.py
@ -7,7 +7,7 @@ from utils import get_packages, OpenMTCSdist
 # name and version
 NAME = "sdk"
 SETUP_NAME = "openmtc-" + NAME
-SETUP_VERSION = "1.3.0"
+SETUP_VERSION = "1.2.0"
 SETUP_DESCRIPTION = "The OpenMTC Python SDK"

 # meta