Implement end-to-end app payload encryption.

This implements end-to-end encryption between the end-device and
end-application. The encrypted AppSKey or SessionKeyID is forwarded to
the end-application which should be able to decrypt or request the
AppSKey to decrypt the uplink payload. As well the end-application will
be able to enqueue encrypted application payloads.

Using this mechanism, ChirpStack will never have access to the uplink
and downlink application-payloads.

api/python/proto/chirpstack-api/api/device.proto

@@ -165,6 +165,17 @@ service DeviceService {
       get : "/api/devices/{dev_eui}/queue"
     };
   }
+
+  // GetNextFCntDown returns the next FCntDown to use for enqueing encrypted
+  // downlinks. The difference with the DeviceActivation f_cont_down is that
+  // this method takes potential existing queue-items into account.
+  rpc GetNextFCntDown(GetDeviceNextFCntDownRequest)
+      returns (GetDeviceNextFCntDownResponse) {
+    option (google.api.http) = {
+      post : "/api/devices/{dev_eui}/get-next-f-cnt-down"
+      body : "*"
+    };
+  }
 }
 
 message Device {
@@ -504,12 +515,20 @@ message DeviceQueueItem {
   google.protobuf.Struct object = 6;
 
   // Is pending.
-  // This is set to true when the downlink is pending.
+  // This is set by ChirpStack to true when the downlink is pending (e.g. it
+  // has been sent, but a confirmation is still pending).
   bool is_pending = 7;
 
   // Downlink frame-counter.
-  // This is set when the payload has been sent as downlink.
+  // Do not set this for plain-text data payloads. It will be automatically set
+  // by ChirpStack when the payload has been sent as downlink.
   uint32 f_cnt_down = 8;
+
+  // Is encrypted.
+  // This must be set to true if the end-application has already encrypted
+  // the data payload. In this case, the f_cnt_down field must be set to
+  // the corresponding frame-counter which has been used during the encryption.
+  bool is_encrypted = 9;
 }
 
 message EnqueueDeviceQueueItemRequest { DeviceQueueItem queue_item = 1; }
@@ -544,3 +563,13 @@ message FlushDevNoncesRequest {
   // Device EUI (EUI64).
   string dev_eui = 1;
 }
+
+message GetDeviceNextFCntDownRequest {
+  // Device EUI (EUI64).
+  string dev_eui = 1;
+}
+
+message GetDeviceNextFCntDownResponse {
+  // FCntDown.
+  uint32 f_cnt_down = 1;
+}

api/python/proto/chirpstack-api/integration/integration.proto

@@ -55,6 +55,9 @@ enum LogCode {
 
   // Relay new end-device.
   RELAY_NEW_END_DEVICE = 9;
+
+  // Downlink frame-counter.
+  F_CNT_DOWN = 10;
 }
 
 // Device information.
@@ -111,6 +114,15 @@ message UplinkRelayRxInfo {
   uint32 wor_channel = 6;
 }
 
+// Join-Server context.
+message JoinServerContext {
+  // Session-key ID.
+  string session_key_id = 1;
+
+  // AppSKey envelope.
+  common.KeyEnvelope app_s_key = 2;
+}
+
 // UplinkEvent is the message sent when an uplink payload has been received.
 message UplinkEvent {
   // Deduplication ID (UUID).
@@ -155,6 +167,12 @@ message UplinkEvent {
 
   // Relay info.
   UplinkRelayRxInfo relay_rx_info = 14;
+
+  // Join-Server context.
+  // A non-empty value indicatest that ChirpStack does not have access to
+  // the AppSKey and that the encryption / decryption of the payloads is
+  // the responsibility of the end-application.
+  JoinServerContext join_server_context = 15;
 }
 
 // JoinEvent is the message sent when a device joined the network.
@@ -174,6 +192,12 @@ message JoinEvent {
 
   // Relay info.
   UplinkRelayRxInfo relay_rx_info = 5;
+
+  // Join-Server context.
+  // A non-empty value indicatest that ChirpStack does not have access to
+  // the AppSKey and that the encryption / decryption of the payloads is
+  // the responsibility of the end-application.
+  JoinServerContext join_server_context = 6;
 }
 
 // AckEvent is the message sent when a confirmation on a confirmed downlink

api/python/proto/chirpstack-api/internal/internal.proto

@@ -31,6 +31,9 @@ message DeviceSession {
   // AppSKey envelope.
   common.KeyEnvelope app_s_key = 8;
 
+  // JS Session Key ID.
+  bytes js_session_key_id = 42;
+
   // Uplink frame-counter.
   uint32 f_cnt_up = 9;
 

api/python/src/chirpstack_api/api/device_pb2.pyi

@@ -315,7 +315,7 @@ class GetDeviceLinkMetricsResponse(_message.Message):
     def __init__(self, rx_packets: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ..., gw_rssi: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ..., gw_snr: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ..., rx_packets_per_freq: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ..., rx_packets_per_dr: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ..., errors: _Optional[_Union[_common_pb2.Metric, _Mapping]] = ...) -> None: ...
 
 class DeviceQueueItem(_message.Message):
-    __slots__ = ["id", "dev_eui", "confirmed", "f_port", "data", "object", "is_pending", "f_cnt_down"]
+    __slots__ = ["id", "dev_eui", "confirmed", "f_port", "data", "object", "is_pending", "f_cnt_down", "is_encrypted"]
     ID_FIELD_NUMBER: _ClassVar[int]
     DEV_EUI_FIELD_NUMBER: _ClassVar[int]
     CONFIRMED_FIELD_NUMBER: _ClassVar[int]
@@ -324,6 +324,7 @@ class DeviceQueueItem(_message.Message):
     OBJECT_FIELD_NUMBER: _ClassVar[int]
     IS_PENDING_FIELD_NUMBER: _ClassVar[int]
     F_CNT_DOWN_FIELD_NUMBER: _ClassVar[int]
+    IS_ENCRYPTED_FIELD_NUMBER: _ClassVar[int]
     id: str
     dev_eui: str
     confirmed: bool
@@ -332,7 +333,8 @@ class DeviceQueueItem(_message.Message):
     object: _struct_pb2.Struct
     is_pending: bool
     f_cnt_down: int
-    def __init__(self, id: _Optional[str] = ..., dev_eui: _Optional[str] = ..., confirmed: bool = ..., f_port: _Optional[int] = ..., data: _Optional[bytes] = ..., object: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., is_pending: bool = ..., f_cnt_down: _Optional[int] = ...) -> None: ...
+    is_encrypted: bool
+    def __init__(self, id: _Optional[str] = ..., dev_eui: _Optional[str] = ..., confirmed: bool = ..., f_port: _Optional[int] = ..., data: _Optional[bytes] = ..., object: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., is_pending: bool = ..., f_cnt_down: _Optional[int] = ..., is_encrypted: bool = ...) -> None: ...
 
 class EnqueueDeviceQueueItemRequest(_message.Message):
     __slots__ = ["queue_item"]
@@ -373,3 +375,15 @@ class FlushDevNoncesRequest(_message.Message):
     DEV_EUI_FIELD_NUMBER: _ClassVar[int]
     dev_eui: str
     def __init__(self, dev_eui: _Optional[str] = ...) -> None: ...
+
+class GetDeviceNextFCntDownRequest(_message.Message):
+    __slots__ = ["dev_eui"]
+    DEV_EUI_FIELD_NUMBER: _ClassVar[int]
+    dev_eui: str
+    def __init__(self, dev_eui: _Optional[str] = ...) -> None: ...
+
+class GetDeviceNextFCntDownResponse(_message.Message):
+    __slots__ = ["f_cnt_down"]
+    F_CNT_DOWN_FIELD_NUMBER: _ClassVar[int]
+    f_cnt_down: int
+    def __init__(self, f_cnt_down: _Optional[int] = ...) -> None: ...

api/python/src/chirpstack_api/api/device_pb2_grpc.py

@@ -111,6 +111,11 @@ class DeviceServiceStub(object):
                 request_serializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceQueueItemsRequest.SerializeToString,
                 response_deserializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceQueueItemsResponse.FromString,
                 )
+        self.GetNextFCntDown = channel.unary_unary(
+                '/api.DeviceService/GetNextFCntDown',
+                request_serializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownRequest.SerializeToString,
+                response_deserializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownResponse.FromString,
+                )
 
 
 class DeviceServiceServicer(object):
@@ -256,6 +261,15 @@ class DeviceServiceServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def GetNextFCntDown(self, request, context):
+        """GetNextFCntDown returns the next FCntDown to use for enqueing encrypted
+        downlinks. The difference with the DeviceActivation f_cont_down is that
+        this method takes potential existing queue-items into account.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_DeviceServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -354,6 +368,11 @@ def add_DeviceServiceServicer_to_server(servicer, server):
                     request_deserializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceQueueItemsRequest.FromString,
                     response_serializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceQueueItemsResponse.SerializeToString,
             ),
+            'GetNextFCntDown': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetNextFCntDown,
+                    request_deserializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownRequest.FromString,
+                    response_serializer=chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'api.DeviceService', rpc_method_handlers)
@@ -687,3 +706,20 @@ class DeviceService(object):
             chirpstack__api_dot_api_dot_device__pb2.GetDeviceQueueItemsResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def GetNextFCntDown(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/api.DeviceService/GetNextFCntDown',
+            chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownRequest.SerializeToString,
+            chirpstack__api_dot_api_dot_device__pb2.GetDeviceNextFCntDownResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

api/python/src/chirpstack_api/integration/integration_pb2.pyi

@@ -28,6 +28,7 @@ class LogCode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     UPLINK_F_CNT_RETRANSMISSION: _ClassVar[LogCode]
     DOWNLINK_GATEWAY: _ClassVar[LogCode]
     RELAY_NEW_END_DEVICE: _ClassVar[LogCode]
+    F_CNT_DOWN: _ClassVar[LogCode]
 INFO: LogLevel
 WARNING: LogLevel
 ERROR: LogLevel
@@ -41,6 +42,7 @@ UPLINK_MIC: LogCode
 UPLINK_F_CNT_RETRANSMISSION: LogCode
 DOWNLINK_GATEWAY: LogCode
 RELAY_NEW_END_DEVICE: LogCode
+F_CNT_DOWN: LogCode
 
 class DeviceInfo(_message.Message):
     __slots__ = ["tenant_id", "tenant_name", "application_id", "application_name", "device_profile_id", "device_profile_name", "device_name", "dev_eui", "device_class_enabled", "tags"]
@@ -89,8 +91,16 @@ class UplinkRelayRxInfo(_message.Message):
     wor_channel: int
     def __init__(self, dev_eui: _Optional[str] = ..., frequency: _Optional[int] = ..., dr: _Optional[int] = ..., snr: _Optional[int] = ..., rssi: _Optional[int] = ..., wor_channel: _Optional[int] = ...) -> None: ...
 
+class JoinServerContext(_message.Message):
+    __slots__ = ["session_key_id", "app_s_key"]
+    SESSION_KEY_ID_FIELD_NUMBER: _ClassVar[int]
+    APP_S_KEY_FIELD_NUMBER: _ClassVar[int]
+    session_key_id: str
+    app_s_key: _common_pb2.KeyEnvelope
+    def __init__(self, session_key_id: _Optional[str] = ..., app_s_key: _Optional[_Union[_common_pb2.KeyEnvelope, _Mapping]] = ...) -> None: ...
+
 class UplinkEvent(_message.Message):
-    __slots__ = ["deduplication_id", "time", "device_info", "dev_addr", "adr", "dr", "f_cnt", "f_port", "confirmed", "data", "object", "rx_info", "tx_info", "relay_rx_info"]
+    __slots__ = ["deduplication_id", "time", "device_info", "dev_addr", "adr", "dr", "f_cnt", "f_port", "confirmed", "data", "object", "rx_info", "tx_info", "relay_rx_info", "join_server_context"]
     DEDUPLICATION_ID_FIELD_NUMBER: _ClassVar[int]
     TIME_FIELD_NUMBER: _ClassVar[int]
     DEVICE_INFO_FIELD_NUMBER: _ClassVar[int]
@@ -105,6 +115,7 @@ class UplinkEvent(_message.Message):
     RX_INFO_FIELD_NUMBER: _ClassVar[int]
     TX_INFO_FIELD_NUMBER: _ClassVar[int]
     RELAY_RX_INFO_FIELD_NUMBER: _ClassVar[int]
+    JOIN_SERVER_CONTEXT_FIELD_NUMBER: _ClassVar[int]
     deduplication_id: str
     time: _timestamp_pb2.Timestamp
     device_info: DeviceInfo
@@ -119,21 +130,24 @@ class UplinkEvent(_message.Message):
     rx_info: _containers.RepeatedCompositeFieldContainer[_gw_pb2.UplinkRxInfo]
     tx_info: _gw_pb2.UplinkTxInfo
     relay_rx_info: UplinkRelayRxInfo
-    def __init__(self, deduplication_id: _Optional[str] = ..., time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., device_info: _Optional[_Union[DeviceInfo, _Mapping]] = ..., dev_addr: _Optional[str] = ..., adr: bool = ..., dr: _Optional[int] = ..., f_cnt: _Optional[int] = ..., f_port: _Optional[int] = ..., confirmed: bool = ..., data: _Optional[bytes] = ..., object: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., rx_info: _Optional[_Iterable[_Union[_gw_pb2.UplinkRxInfo, _Mapping]]] = ..., tx_info: _Optional[_Union[_gw_pb2.UplinkTxInfo, _Mapping]] = ..., relay_rx_info: _Optional[_Union[UplinkRelayRxInfo, _Mapping]] = ...) -> None: ...
+    join_server_context: JoinServerContext
+    def __init__(self, deduplication_id: _Optional[str] = ..., time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., device_info: _Optional[_Union[DeviceInfo, _Mapping]] = ..., dev_addr: _Optional[str] = ..., adr: bool = ..., dr: _Optional[int] = ..., f_cnt: _Optional[int] = ..., f_port: _Optional[int] = ..., confirmed: bool = ..., data: _Optional[bytes] = ..., object: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., rx_info: _Optional[_Iterable[_Union[_gw_pb2.UplinkRxInfo, _Mapping]]] = ..., tx_info: _Optional[_Union[_gw_pb2.UplinkTxInfo, _Mapping]] = ..., relay_rx_info: _Optional[_Union[UplinkRelayRxInfo, _Mapping]] = ..., join_server_context: _Optional[_Union[JoinServerContext, _Mapping]] = ...) -> None: ...
 
 class JoinEvent(_message.Message):
-    __slots__ = ["deduplication_id", "time", "device_info", "dev_addr", "relay_rx_info"]
+    __slots__ = ["deduplication_id", "time", "device_info", "dev_addr", "relay_rx_info", "join_server_context"]
     DEDUPLICATION_ID_FIELD_NUMBER: _ClassVar[int]
     TIME_FIELD_NUMBER: _ClassVar[int]
     DEVICE_INFO_FIELD_NUMBER: _ClassVar[int]
     DEV_ADDR_FIELD_NUMBER: _ClassVar[int]
     RELAY_RX_INFO_FIELD_NUMBER: _ClassVar[int]
+    JOIN_SERVER_CONTEXT_FIELD_NUMBER: _ClassVar[int]
     deduplication_id: str
     time: _timestamp_pb2.Timestamp
     device_info: DeviceInfo
     dev_addr: str
     relay_rx_info: UplinkRelayRxInfo
-    def __init__(self, deduplication_id: _Optional[str] = ..., time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., device_info: _Optional[_Union[DeviceInfo, _Mapping]] = ..., dev_addr: _Optional[str] = ..., relay_rx_info: _Optional[_Union[UplinkRelayRxInfo, _Mapping]] = ...) -> None: ...
+    join_server_context: JoinServerContext
+    def __init__(self, deduplication_id: _Optional[str] = ..., time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., device_info: _Optional[_Union[DeviceInfo, _Mapping]] = ..., dev_addr: _Optional[str] = ..., relay_rx_info: _Optional[_Union[UplinkRelayRxInfo, _Mapping]] = ..., join_server_context: _Optional[_Union[JoinServerContext, _Mapping]] = ...) -> None: ...
 
 class AckEvent(_message.Message):
     __slots__ = ["deduplication_id", "time", "device_info", "queue_item_id", "acknowledged", "f_cnt_down"]