Bump version to 4.7.0-test.4

This migrates the device-sessions from Redis into PostgreSQL. This fixes
a performance issue in case the same DevAddr is reused many times
(e.g. devices rejoining very often or a NetID with small DevAddr space).

There were two issues:

The Redis key containing the DevAddr -> DevEUIs mapping could contain
DevEUIs that no longer used the DevAddr. This mapping would only expire
from the Redis database after none of the devices would use the DevAddr
for more than the configured device_session_ttl.

The other issue with the previous approach was that on for example a
Type 7 NetID, a single DevAddr could be re-used multiple times. As each
device-session could be stored on a different Redis Cluster instance,
there was no option to retrieve all device-sessions at once. Thus a high
re-usage of a single DevAddr would cause an increase in Redis queries.

Both issues are solved by moving the device-session into PostgreSQL
as the DevAddr is a column of the device record and thus filtering on
this DevAddr would always result in the devices using that DevAddr, as
well all device-sessions for a DevAddr can be retrieved by a single
query.

Note that to migrate the device-sessions, you must run:

chirpstack -c path/to/config migrate-device-sessions-to-postgres

A nice side-effect is that a PostgreSQL backup / restore will also
restore the device connectivity.

Closes #362 and #74.

Cargo.lock

@@ -570,7 +570,7 @@ dependencies = [
 
 [[package]]
 name = "backend"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
  "aes-kw",
  "anyhow",
@@ -787,7 +787,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chirpstack"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
  "aes",
  "anyhow",
@@ -874,7 +874,7 @@ dependencies = [
 
 [[package]]
 name = "chirpstack_api"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
  "diesel",
  "hex",
@@ -2434,7 +2434,7 @@ dependencies = [
 
 [[package]]
 name = "lrwn"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
  "aes",
  "anyhow",
@@ -2448,7 +2448,7 @@ dependencies = [
 
 [[package]]
 name = "lrwn_filters"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
  "hex",
  "lrwn",

api/grpc-web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chirpstack/chirpstack-api-grpc-web",
-  "version": "4.7.0-test.3",
+  "version": "4.7.0-test.4",
   "description": "Chirpstack gRPC-web API",
   "license": "MIT",
   "devDependencies": {

api/java/build.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = "io.chirpstack"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 
 repositories {
     mavenCentral()

api/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chirpstack/chirpstack-api",
-  "version": "4.7.0-test.3",
+  "version": "4.7.0-test.4",
   "description": "Chirpstack JS and TS API",
   "license": "MIT",
   "devDependencies": {

api/kotlin/build.gradle.kts

@@ -9,7 +9,7 @@ plugins {
 }
 
 group = "io.chirpstack"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 
 repositories {
     mavenCentral()

api/python/proto/chirpstack-api/internal/internal.proto

@@ -7,15 +7,9 @@ import "chirpstack-api/gw/gw.proto";
 import "google/protobuf/timestamp.proto";
 
 message DeviceSession {
-  // Device EUI.
-  bytes dev_eui = 1;
-
   // Device address.
   bytes dev_addr = 2;
 
-  // Join EUI.
-  bytes join_eui = 3;
-
   // LoRaWAN mac-version.
   common.MacVersion mac_version = 4;
 

api/python/src/setup.py

@@ -18,7 +18,7 @@ CLASSIFIERS = [
 
 setup(
     name='chirpstack-api',
-    version = "4.7.0-test.3",
+    version = "4.7.0-test.4",
     url='https://github.com/brocaar/chirpstack-api',
     author='Orne Brocaar',
     author_email='info@brocaar.com',

api/rust/Cargo.lock

@@ -149,6 +149,12 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "bytes"
 version = "1.4.0"
@@ -172,8 +178,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chirpstack_api"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 dependencies = [
+ "diesel",
  "hex",
  "pbjson",
  "pbjson-build",
@@ -196,6 +203,39 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "diesel"
+version = "2.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
+dependencies = [
+ "bitflags 2.4.0",
+ "byteorder",
+ "diesel_derives",
+ "itoa",
+]
+
+[[package]]
+name = "diesel_derives"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44"
+dependencies = [
+ "diesel_table_macro_syntax",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "diesel_table_macro_syntax"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
+dependencies = [
+ "syn",
+]
+
 [[package]]
 name = "dirs"
 version = "5.0.1"
@@ -457,15 +497,6 @@ dependencies = [
  "hashbrown 0.14.0",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.11.0"
@@ -590,7 +621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735"
 dependencies = [
  "heck",
- "itertools 0.11.0",
+ "itertools",
  "prost",
  "prost-types",
 ]
@@ -701,7 +732,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac"
 dependencies = [
  "bytes",
  "heck",
- "itertools 0.10.5",
+ "itertools",
  "log",
  "multimap",
  "once_cell",
@@ -722,7 +753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32"
 dependencies = [
  "anyhow",
- "itertools 0.10.5",
+ "itertools",
  "proc-macro2",
  "quote",
  "syn",

api/rust/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "chirpstack_api"
 description = "ChirpStack Protobuf / gRPC API definitions."
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 license = "MIT"
 homepage = "https://www.chirpstack.io"

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "backend"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2018"
 publish = false

chirpstack/Cargo.toml

@@ -3,7 +3,7 @@ name = "chirpstack"
 description = "ChirpStack is an open-source LoRaWAN(TM) Network Server"
 repository = "https://github.com/chirpstack/chirpstack"
 homepage = "https://www.chirpstack.io/"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2021"
 publish = false

chirpstack/src/api/oidc.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::str::FromStr;
 
 use anyhow::{Context, Result};
@@ -7,12 +8,13 @@ use openidconnect::core::{
     CoreResponseType,
 };
 use openidconnect::reqwest::async_http_client;
+use openidconnect::{AdditionalClaims, UserInfoClaims};
 use openidconnect::{
     AuthenticationFlow, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
     OAuth2TokenResponse, RedirectUrl, Scope,
 };
-use openidconnect::{EmptyAdditionalClaims, UserInfoClaims};
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use tracing::{error, trace};
 use warp::{Rejection, Reply};
 
@@ -20,7 +22,15 @@ use crate::config;
 use crate::helpers::errors::PrintFullError;
 use crate::storage::{get_async_redis_conn, redis_key};
 
-pub type User = UserInfoClaims<EmptyAdditionalClaims, CoreGenderClaim>;
+pub type User = UserInfoClaims<CustomClaims, CoreGenderClaim>;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CustomClaims {
+    #[serde(flatten)]
+    other: HashMap<String, Value>,
+}
+
+impl AdditionalClaims for CustomClaims {}
 
 #[derive(Serialize, Deserialize)]
 pub struct CallbackArgs {

chirpstack/src/cmd/configfile.rs

@@ -185,12 +185,6 @@ pub fn run() {
     {{/each}}
   ]
 
-  # Device session expiration.
-  #
-  # The TTL value defines the time after which a device-session expires
-  # after no activity.
-  device_session_ttl="{{ network.device_session_ttl }}"
-
   # Time to wait for uplink de-duplication.
   #
   # This is the time that ChirpStack will wait for other gateways to receive

chirpstack/src/downlink/data.rs

@@ -363,12 +363,7 @@ impl Data {
 
         let gw_down = helpers::select_downlink_gateway(
             Some(self.tenant.id),
-            &self
-                .device
-                .device_session
-                .as_ref()
-                .unwrap()
-                .region_config_id,
+            &self.device.get_device_session()?.region_config_id,
             self.network_conf.gateway_prefer_min_margin,
             self.device_gateway_rx_info.as_mut().unwrap(),
         )?;

chirpstack/src/uplink/data.rs

@@ -585,13 +585,7 @@ impl Data {
 
     fn decrypt_f_opts_mac_commands(&mut self) -> Result<()> {
         trace!("Decrypting mac-commands");
-        let ds = self
-            .device
-            .as_ref()
-            .unwrap()
-            .device_session
-            .as_ref()
-            .unwrap();
+        let ds = self.device.as_ref().unwrap().get_device_session()?;
         if ds.mac_version().to_string().starts_with("1.0") {
             if let Err(e) = self.phy_payload.decode_f_opts_to_mac_commands() {
                 // This avoids failing in case of a corrupted mac-command in the frm_payload.
@@ -610,13 +604,7 @@ impl Data {
 
     fn decrypt_frm_payload(&mut self) -> Result<()> {
         trace!("Decrypting FRMPayload");
-        let ds = self
-            .device
-            .as_ref()
-            .unwrap()
-            .device_session
-            .as_ref()
-            .unwrap();
+        let ds = self.device.as_ref().unwrap().get_device_session()?;
         let mut f_port = 0;
 
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
@@ -656,13 +644,7 @@ impl Data {
 
     fn set_adr(&mut self) -> Result<()> {
         trace!("Set ADR flag in device-session");
-        let ds = self
-            .device
-            .as_mut()
-            .unwrap()
-            .device_session
-            .as_mut()
-            .unwrap();
+        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
             ds.adr = pl.fhdr.f_ctrl.adr;
         }
@@ -790,7 +772,7 @@ impl Data {
         if let lrwn::Payload::MACPayload(pl) = &self.phy_payload.payload {
             if pl.fhdr.f_ctrl.adr_ack_req {
                 let region_conf = region::get(&self.uplink_frame_set.region_config_id)?;
-                let ds = d.device_session.as_mut().unwrap();
+                let ds = d.get_device_session_mut()?;
 
                 // We reset the device-session enabled_uplink_channel_indices and
                 // extra_uplink_channels. On the downlink path, the mac-command handling will
@@ -861,13 +843,7 @@ impl Data {
     }
 
     fn append_meta_data_to_uplink_history(&mut self) -> Result<()> {
-        let ds = self
-            .device
-            .as_mut()
-            .unwrap()
-            .device_session
-            .as_mut()
-            .unwrap();
+        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
 
         // ignore re-transmissions we don't know the source of the
         // re-transmission (it might be a replay-attack)
@@ -912,13 +888,7 @@ impl Data {
     fn append_meta_data_to_uplink_history_relayed(&mut self) -> Result<()> {
         trace!("Apping meta-data of relayed uplink to upink history");
 
-        let ds = self
-            .device
-            .as_mut()
-            .unwrap()
-            .device_session
-            .as_mut()
-            .unwrap();
+        let ds = self.device.as_mut().unwrap().get_device_session_mut()?;
         let relay_ctx = self.relay_context.as_ref().unwrap();
 
         // ignore re-transmissions we don't know the source of the
@@ -1395,13 +1365,10 @@ impl Data {
     }
 
     fn _is_end_to_end_encrypted(&self) -> bool {
-        let ds = self
-            .device
-            .as_ref()
-            .unwrap()
-            .device_session
-            .as_ref()
-            .unwrap();
+        let ds = match self.device.as_ref().unwrap().get_device_session() {
+            Ok(v) => v,
+            Err(_) => return false,
+        };
 
         if !ds.js_session_key_id.is_empty() {
             return true;

chirpstack/src/uplink/join_sns.rs

@@ -641,13 +641,7 @@ impl JoinRequest {
     async fn update_device(&mut self) -> Result<()> {
         trace!("Updating device");
         let dp = self.device_profile.as_ref().unwrap();
-        let ds = self
-            .device
-            .as_ref()
-            .unwrap()
-            .device_session
-            .as_ref()
-            .unwrap();
+        let ds = self.device.as_ref().unwrap().get_device_session()?;
 
         self.device = Some(
             device::partial_update(

lrwn-filters/Cargo.toml

@@ -3,7 +3,7 @@ name = "lrwn_filters"
 description = "Library for filtering LoRaWAN payloads on DevAddr and JoinEUIs prefixes"
 homepage = "https://www.chirpstack.io/"
 license = "MIT"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2021"
 repository = "https://github.com/chirpstack/chirpstack"

lrwn/Cargo.toml

@@ -3,7 +3,7 @@ name = "lrwn"
 description = "Library for encoding / decoding LoRaWAN frames."
 homepage = "https://www.chirpstack.io"
 license = "MIT"
-version = "4.7.0-test.3"
+version = "4.7.0-test.4"
 authors = ["Orne Brocaar <info@brocaar.com>"]
 edition = "2018"
 repository = "https://github.com/chirpstack/chirpstack"

ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chirpstack-ui",
-  "version": "4.7.0-test.3",
+  "version": "4.7.0-test.4",
   "private": true,
   "dependencies": {
     "@ant-design/colors": "^7.0.0",