Implement support for storing DevNonces per JoinEUI.

chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/down.sql

@@ -0,0 +1,2 @@
+alter table device_keys
+  alter column dev_nonces type int[] using '{}';

chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/up.sql

@@ -0,0 +1,7 @@
+alter table device_keys
+  alter column dev_nonces type jsonb using jsonb_build_object('0000000000000000', dev_nonces);
+
+update device_keys
+  set dev_nonces = jsonb_build_object(encode(device.join_eui, 'hex'), dev_nonces['0000000000000000'])
+from device
+  where device.dev_eui = device_keys.dev_eui;

chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/down.sql

@@ -0,0 +1 @@
+update device_keys set dev_nonces = '[]';

chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/up.sql

@@ -0,0 +1 @@
+update device_keys set dev_nonces = '{}';

chirpstack/src/api/device.rs

@@ -469,7 +469,7 @@ impl DeviceService for Device {
             )
             .await?;
 
-        device_keys::set_dev_nonces(&dev_eui, &Vec::new())
+        device_keys::set_dev_nonces(dev_eui, &fields::DevNonces::default())
             .await
             .map_err(|e| e.status())?;
 
@@ -1429,10 +1429,13 @@ pub mod test {
         );
 
         // flush dev nonces
-        let _ =
+        let _ = device_keys::set_dev_nonces(EUI64::from_str("0102030405060708").unwrap(), &{
-            device_keys::set_dev_nonces(&EUI64::from_str("0102030405060708").unwrap(), &[1, 2, 3])
+            let mut dev_nonces = fields::DevNonces::default();
-                .await
+            dev_nonces.insert(EUI64::from_str("0102030405060708").unwrap(), 123);
-                .unwrap();
+            dev_nonces
+        })
+        .await
+        .unwrap();
         let flush_dev_nonces_req = get_request(
             &u.id,
             api::FlushDevNoncesRequest {
@@ -1446,7 +1449,7 @@ pub mod test {
         let dk = device_keys::get(&EUI64::from_str("0102030405060708").unwrap())
             .await
             .unwrap();
-        assert_eq!(0, dk.dev_nonces.len());
+        assert_eq!(fields::DevNonces::default(), dk.dev_nonces);
 
         // delete keys
         let del_keys_req = get_request(

chirpstack/src/storage/device_keys.rs

@@ -38,7 +38,7 @@ impl Default for DeviceKeys {
                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                 0x00, 0x00,
             ]),
-            dev_nonces: Vec::new().into(),
+            dev_nonces: fields::DevNonces::default(),
             join_nonce: 0,
         }
     }
@@ -93,10 +93,12 @@ pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result<DeviceKeys, Error> {
+pub async fn set_dev_nonces(
-    let nonces: Vec<Option<i32>> = nonces.iter().map(|v| Some(*v)).collect();
+    dev_eui: EUI64,
+    nonces: &fields::DevNonces,
+) -> Result<DeviceKeys, Error> {
     let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(dev_eui))
-        .set(device_keys::dev_nonces.eq(fields::DevNonces::from(nonces)))
+        .set(device_keys::dev_nonces.eq(nonces))
         .get_result(&mut get_async_db_conn().await?)
         .await
         .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
@@ -108,8 +110,9 @@ pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result<DeviceKey
 }
 
 pub async fn validate_incr_join_and_store_dev_nonce(
-    dev_eui: &EUI64,
+    join_eui: EUI64,
-    dev_nonce: i32,
+    dev_eui: EUI64,
+    dev_nonce: u16,
 ) -> Result<DeviceKeys, Error> {
     let mut c = get_async_db_conn().await?;
     let dk: DeviceKeys = db_transaction::<DeviceKeys, Error, _>(&mut c, |c| {
@@ -122,11 +125,11 @@ pub async fn validate_incr_join_and_store_dev_nonce(
                 .await
                 .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
 
-            if dk.dev_nonces.contains(&(Some(dev_nonce))) {
+            if dk.dev_nonces.contains(join_eui, dev_nonce) {
                 return Err(Error::InvalidDevNonce);
             }
 
-            dk.dev_nonces.push(Some(dev_nonce));
+            dk.dev_nonces.insert(join_eui, dev_nonce);
             dk.join_nonce += 1;
 
             diesel::update(device_keys::dsl::device_keys.find(&dev_eui))
@@ -155,7 +158,7 @@ pub mod test {
     pub async fn reset_nonces(dev_eui: &EUI64) -> Result<DeviceKeys, Error> {
         let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(&dev_eui))
             .set((
-                device_keys::dev_nonces.eq(fields::DevNonces::from(Vec::new())),
+                device_keys::dev_nonces.eq(fields::DevNonces::default()),
                 device_keys::join_nonce.eq(0),
             ))
             .get_result(&mut get_async_db_conn().await?)

chirpstack/src/storage/fields/dev_nonces.rs

@@ -1,67 +1,48 @@
+use std::collections::HashMap;
+
 use diesel::backend::Backend;
+
 use diesel::{deserialize, serialize};
 #[cfg(feature = "postgres")]
-use diesel::{
+use diesel::{pg::Pg, sql_types::Jsonb};
-    pg::Pg,
-    sql_types::{Array, Int4, Nullable},
-};
 #[cfg(feature = "sqlite")]
 use diesel::{sql_types::Text, sqlite::Sqlite};
-use serde::{Deserialize, Serialize};
 
-#[cfg(feature = "postgres")]
+use lrwn::EUI64;
-type DevNoncesPgType = Array<Nullable<Int4>>;
 
-// Sqlite has no native array type so use text
+#[derive(Default, Debug, Clone, PartialEq, Eq, AsExpression, FromSqlRow)]
-#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq, AsExpression, FromSqlRow)]
+#[cfg_attr(feature = "postgres", diesel(sql_type = Jsonb))]
-#[serde(transparent)]
-#[cfg_attr(feature = "postgres", diesel(sql_type = DevNoncesPgType))]
 #[cfg_attr(feature = "sqlite", diesel(sql_type = Text))]
-#[derive(Default)]
+pub struct DevNonces(HashMap<EUI64, Vec<u16>>);
-pub struct DevNonces(DevNoncesInner);
 
-pub type DevNoncesInner = Vec<Option<i32>>;
+impl DevNonces {
-
+    pub fn contains(&self, join_eui: EUI64, dev_nonce: u16) -> bool {
-impl std::convert::AsRef<DevNoncesInner> for DevNonces {
+        if let Some(v) = self.0.get(&join_eui) {
-    fn as_ref(&self) -> &DevNoncesInner {
+            v.contains(&dev_nonce)
-        &self.0
+        } else {
+            false
+        }
     }
-}
 
-impl std::convert::From<DevNoncesInner> for DevNonces {
+    pub fn insert(&mut self, join_eui: EUI64, dev_nonce: u16) {
-    fn from(value: DevNoncesInner) -> Self {
+        self.0.entry(join_eui).or_default().push(dev_nonce)
-        Self(value)
-    }
-}
-
-impl std::ops::Deref for DevNonces {
-    type Target = DevNoncesInner;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl std::ops::DerefMut for DevNonces {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
     }
 }
 
 #[cfg(feature = "postgres")]
-impl deserialize::FromSql<DevNoncesPgType, Pg> for DevNonces {
+impl deserialize::FromSql<Jsonb, Pg> for DevNonces {
     fn from_sql(value: <Pg as Backend>::RawValue<'_>) -> deserialize::Result<Self> {
-        let sql_val = <DevNoncesInner>::from_sql(value)?;
+        let value = <serde_json::Value as deserialize::FromSql<Jsonb, Pg>>::from_sql(value)?;
-        Ok(DevNonces(sql_val))
+        let dev_nonces: HashMap<EUI64, Vec<u16>> = serde_json::from_value(value)?;
+        Ok(DevNonces(dev_nonces))
     }
 }
 
 #[cfg(feature = "postgres")]
-impl serialize::ToSql<DevNoncesPgType, Pg> for DevNonces {
+impl serialize::ToSql<Jsonb, Pg> for DevNonces {
-    fn to_sql(&self, out: &mut serialize::Output<'_, '_, Pg>) -> serialize::Result {
+    fn to_sql<'b>(&'b self, out: &mut serialize::Output<'b, '_, Pg>) -> serialize::Result {
-        <DevNoncesInner as serialize::ToSql<DevNoncesPgType, Pg>>::to_sql(
+        let value = serde_json::to_value(&self.0)?;
-            &self.0,
+        <serde_json::Value as serialize::ToSql<Jsonb, Pg>>::to_sql(&value, &mut out.reborrow())
-            &mut out.reborrow(),
-        )
     }
 }
 
@@ -73,15 +54,15 @@ where
     fn from_sql(value: <Sqlite as Backend>::RawValue<'_>) -> deserialize::Result<Self> {
         let s =
             <*const str as deserialize::FromSql<diesel::sql_types::Text, Sqlite>>::from_sql(value)?;
-        let nonces = serde_json::from_str::<DevNonces>(unsafe { &*s })?;
+        let dev_nonces: HashMap<EUI64, Vec<u16>> = serde_json::from_str(unsafe { &*s })?;
-        Ok(nonces)
+        Ok(DevNonces(dev_nonces))
     }
 }
 
 #[cfg(feature = "sqlite")]
 impl serialize::ToSql<Text, Sqlite> for DevNonces {
-    fn to_sql<'b>(&self, out: &mut serialize::Output<'b, '_, Sqlite>) -> serialize::Result {
+    fn to_sql<'b>(&'b self, out: &mut serialize::Output<'b, '_, Sqlite>) -> serialize::Result {
-        out.set_value(serde_json::to_string(self)?);
+        out.set_value(serde_json::to_string(&self.0)?);
         Ok(serialize::IsNull::No)
     }
 }

chirpstack/src/storage/fields/mod.rs

@@ -7,7 +7,7 @@ mod multicast_group_scheduling_type;
 mod uuid;
 
 pub use big_decimal::BigDecimal;
-pub use dev_nonces::*;
+pub use dev_nonces::DevNonces;
 pub use device_session::DeviceSession;
 pub use key_value::KeyValue;
 pub use measurements::*;

chirpstack/src/storage/schema_postgres.rs

@@ -75,7 +75,7 @@ diesel::table! {
         updated_at -> Timestamptz,
         nwk_key -> Bytea,
         app_key -> Bytea,
-        dev_nonces -> Array<Nullable<Int4>>,
+        dev_nonces -> Jsonb,
         join_nonce -> Int4,
     }
 }

chirpstack/src/test/otaa_pr_test.rs

@@ -15,7 +15,7 @@ use crate::storage::{
     device::{self, DeviceClass},
     device_keys, device_profile, gateway, tenant,
 };
-use crate::{config, test, uplink};
+use crate::{config, storage::fields, test, uplink};
 use chirpstack_api::gw;
 use lrwn::{AES128Key, EUI64Prefix, NetID, EUI64};
 
@@ -314,7 +314,7 @@ async fn test_sns() {
     let dk = device_keys::create(device_keys::DeviceKeys {
         dev_eui: dev.dev_eui,
         nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        dev_nonces: vec![].into(),
+        dev_nonces: fields::DevNonces::default(),
         ..Default::default()
     })
     .await
@@ -499,7 +499,7 @@ async fn test_sns_roaming_not_allowed() {
     let dk = device_keys::create(device_keys::DeviceKeys {
         dev_eui: dev.dev_eui,
         nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        dev_nonces: vec![].into(),
+        dev_nonces: fields::DevNonces::default(),
         ..Default::default()
     })
     .await

chirpstack/src/test/otaa_test.rs

@@ -10,7 +10,9 @@ use crate::storage::{
     device::{self, DeviceClass},
     device_keys, device_profile, gateway, tenant,
 };
-use crate::{config, gateway::backend as gateway_backend, integration, region, test, uplink};
+use crate::{
+    config, gateway::backend as gateway_backend, integration, region, storage::fields, test, uplink,
+};
 use chirpstack_api::{common, gw, internal, stream};
 use lrwn::keys::get_js_int_key;
 use lrwn::{AES128Key, EUI64};
@@ -101,7 +103,11 @@ async fn test_gateway_filtering() {
     let dk = device_keys::create(device_keys::DeviceKeys {
         dev_eui: dev.dev_eui,
         nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        dev_nonces: vec![Some(258)].into(),
+        dev_nonces: {
+            let mut dev_nonces = fields::DevNonces::default();
+            dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258);
+            dev_nonces
+        },
         ..Default::default()
     })
     .await
@@ -273,7 +279,11 @@ async fn test_lorawan_10() {
     let dk = device_keys::create(device_keys::DeviceKeys {
         dev_eui: dev.dev_eui,
         nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        dev_nonces: vec![Some(258)].into(),
+        dev_nonces: {
+            let mut dev_nonces = fields::DevNonces::default();
+            dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258);
+            dev_nonces
+        },
         ..Default::default()
     })
     .await
@@ -929,7 +939,11 @@ async fn test_lorawan_11() {
         dev_eui: dev.dev_eui,
         nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
         app_key: AES128Key::from_bytes([16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]),
-        dev_nonces: vec![Some(258)].into(),
+        dev_nonces: {
+            let mut dev_nonces = fields::DevNonces::default();
+            dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258);
+            dev_nonces
+        },
         ..Default::default()
     })
     .await

chirpstack/src/uplink/join.rs

@@ -456,8 +456,9 @@ impl JoinRequest {
 
         self.device_keys = Some(
             match device_keys::validate_incr_join_and_store_dev_nonce(
-                &dev.dev_eui,
+                join_request.join_eui,
-                join_request.dev_nonce as i32,
+                dev.dev_eui,
+                join_request.dev_nonce,
             )
             .await
             {

chirpstack/src/uplink/join_sns.rs

@@ -353,8 +353,9 @@ impl JoinRequest {
 
         self.device_keys = Some(
             match device_keys::validate_incr_join_and_store_dev_nonce(
-                &dev.dev_eui,
+                join_request.join_eui,
-                join_request.dev_nonce as i32,
+                dev.dev_eui,
+                join_request.dev_nonce,
             )
             .await
             {