diff --git a/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/down.sql b/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/down.sql new file mode 100644 index 00000000..397015d7 --- /dev/null +++ b/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/down.sql @@ -0,0 +1,2 @@ +alter table device_keys + alter column dev_nonces type int[] using '{}'; diff --git a/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/up.sql b/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/up.sql new file mode 100644 index 00000000..401cb90c --- /dev/null +++ b/chirpstack/migrations_postgres/2024-11-12-135745_dev_nonces_to_json/up.sql @@ -0,0 +1,7 @@ +alter table device_keys + alter column dev_nonces type jsonb using jsonb_build_object('0000000000000000', dev_nonces); + +update device_keys + set dev_nonces = jsonb_build_object(encode(device.join_eui, 'hex'), dev_nonces['0000000000000000']) +from device + where device.dev_eui = device_keys.dev_eui; diff --git a/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/down.sql b/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/down.sql new file mode 100644 index 00000000..ec4d3717 --- /dev/null +++ b/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/down.sql @@ -0,0 +1 @@ +update device_keys set dev_nonces = '[]'; diff --git a/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/up.sql b/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/up.sql new file mode 100644 index 00000000..63a6403a --- /dev/null +++ b/chirpstack/migrations_sqlite/2024-11-12-161305_dev_nonces_to_json/up.sql @@ -0,0 +1 @@ +update device_keys set dev_nonces = '{}'; diff --git a/chirpstack/src/api/device.rs b/chirpstack/src/api/device.rs index 41128551..b71ab22b 100644 --- a/chirpstack/src/api/device.rs +++ b/chirpstack/src/api/device.rs @@ -469,7 +469,7 @@ impl DeviceService for Device { ) .await?; - device_keys::set_dev_nonces(&dev_eui, &Vec::new()) + device_keys::set_dev_nonces(dev_eui, &fields::DevNonces::default()) .await .map_err(|e| e.status())?; @@ -1429,10 +1429,13 @@ pub mod test { ); // flush dev nonces - let _ = - device_keys::set_dev_nonces(&EUI64::from_str("0102030405060708").unwrap(), &[1, 2, 3]) - .await - .unwrap(); + let _ = device_keys::set_dev_nonces(EUI64::from_str("0102030405060708").unwrap(), &{ + let mut dev_nonces = fields::DevNonces::default(); + dev_nonces.insert(EUI64::from_str("0102030405060708").unwrap(), 123); + dev_nonces + }) + .await + .unwrap(); let flush_dev_nonces_req = get_request( &u.id, api::FlushDevNoncesRequest { @@ -1446,7 +1449,7 @@ pub mod test { let dk = device_keys::get(&EUI64::from_str("0102030405060708").unwrap()) .await .unwrap(); - assert_eq!(0, dk.dev_nonces.len()); + assert_eq!(fields::DevNonces::default(), dk.dev_nonces); // delete keys let del_keys_req = get_request( diff --git a/chirpstack/src/storage/device_keys.rs b/chirpstack/src/storage/device_keys.rs index 9a506fdd..03619cbb 100644 --- a/chirpstack/src/storage/device_keys.rs +++ b/chirpstack/src/storage/device_keys.rs @@ -38,7 +38,7 @@ impl Default for DeviceKeys { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ]), - dev_nonces: Vec::new().into(), + dev_nonces: fields::DevNonces::default(), join_nonce: 0, } } @@ -93,10 +93,12 @@ pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> { Ok(()) } -pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result { - let nonces: Vec> = nonces.iter().map(|v| Some(*v)).collect(); +pub async fn set_dev_nonces( + dev_eui: EUI64, + nonces: &fields::DevNonces, +) -> Result { let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(dev_eui)) - .set(device_keys::dev_nonces.eq(fields::DevNonces::from(nonces))) + .set(device_keys::dev_nonces.eq(nonces)) .get_result(&mut get_async_db_conn().await?) .await .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?; @@ -108,8 +110,9 @@ pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result Result { let mut c = get_async_db_conn().await?; let dk: DeviceKeys = db_transaction::(&mut c, |c| { @@ -122,11 +125,11 @@ pub async fn validate_incr_join_and_store_dev_nonce( .await .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?; - if dk.dev_nonces.contains(&(Some(dev_nonce))) { + if dk.dev_nonces.contains(join_eui, dev_nonce) { return Err(Error::InvalidDevNonce); } - dk.dev_nonces.push(Some(dev_nonce)); + dk.dev_nonces.insert(join_eui, dev_nonce); dk.join_nonce += 1; diesel::update(device_keys::dsl::device_keys.find(&dev_eui)) @@ -155,7 +158,7 @@ pub mod test { pub async fn reset_nonces(dev_eui: &EUI64) -> Result { let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(&dev_eui)) .set(( - device_keys::dev_nonces.eq(fields::DevNonces::from(Vec::new())), + device_keys::dev_nonces.eq(fields::DevNonces::default()), device_keys::join_nonce.eq(0), )) .get_result(&mut get_async_db_conn().await?) diff --git a/chirpstack/src/storage/fields/dev_nonces.rs b/chirpstack/src/storage/fields/dev_nonces.rs index b4a10f4e..b9d2d574 100644 --- a/chirpstack/src/storage/fields/dev_nonces.rs +++ b/chirpstack/src/storage/fields/dev_nonces.rs @@ -1,67 +1,48 @@ +use std::collections::HashMap; + use diesel::backend::Backend; + use diesel::{deserialize, serialize}; #[cfg(feature = "postgres")] -use diesel::{ - pg::Pg, - sql_types::{Array, Int4, Nullable}, -}; +use diesel::{pg::Pg, sql_types::Jsonb}; #[cfg(feature = "sqlite")] use diesel::{sql_types::Text, sqlite::Sqlite}; -use serde::{Deserialize, Serialize}; -#[cfg(feature = "postgres")] -type DevNoncesPgType = Array>; +use lrwn::EUI64; -// Sqlite has no native array type so use text -#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq, AsExpression, FromSqlRow)] -#[serde(transparent)] -#[cfg_attr(feature = "postgres", diesel(sql_type = DevNoncesPgType))] +#[derive(Default, Debug, Clone, PartialEq, Eq, AsExpression, FromSqlRow)] +#[cfg_attr(feature = "postgres", diesel(sql_type = Jsonb))] #[cfg_attr(feature = "sqlite", diesel(sql_type = Text))] -#[derive(Default)] -pub struct DevNonces(DevNoncesInner); +pub struct DevNonces(HashMap>); -pub type DevNoncesInner = Vec>; - -impl std::convert::AsRef for DevNonces { - fn as_ref(&self) -> &DevNoncesInner { - &self.0 +impl DevNonces { + pub fn contains(&self, join_eui: EUI64, dev_nonce: u16) -> bool { + if let Some(v) = self.0.get(&join_eui) { + v.contains(&dev_nonce) + } else { + false + } } -} -impl std::convert::From for DevNonces { - fn from(value: DevNoncesInner) -> Self { - Self(value) - } -} - -impl std::ops::Deref for DevNonces { - type Target = DevNoncesInner; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for DevNonces { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 + pub fn insert(&mut self, join_eui: EUI64, dev_nonce: u16) { + self.0.entry(join_eui).or_default().push(dev_nonce) } } #[cfg(feature = "postgres")] -impl deserialize::FromSql for DevNonces { +impl deserialize::FromSql for DevNonces { fn from_sql(value: ::RawValue<'_>) -> deserialize::Result { - let sql_val = ::from_sql(value)?; - Ok(DevNonces(sql_val)) + let value = >::from_sql(value)?; + let dev_nonces: HashMap> = serde_json::from_value(value)?; + Ok(DevNonces(dev_nonces)) } } #[cfg(feature = "postgres")] -impl serialize::ToSql for DevNonces { - fn to_sql(&self, out: &mut serialize::Output<'_, '_, Pg>) -> serialize::Result { - >::to_sql( - &self.0, - &mut out.reborrow(), - ) +impl serialize::ToSql for DevNonces { + fn to_sql<'b>(&'b self, out: &mut serialize::Output<'b, '_, Pg>) -> serialize::Result { + let value = serde_json::to_value(&self.0)?; + >::to_sql(&value, &mut out.reborrow()) } } @@ -73,15 +54,15 @@ where fn from_sql(value: ::RawValue<'_>) -> deserialize::Result { let s = <*const str as deserialize::FromSql>::from_sql(value)?; - let nonces = serde_json::from_str::(unsafe { &*s })?; - Ok(nonces) + let dev_nonces: HashMap> = serde_json::from_str(unsafe { &*s })?; + Ok(DevNonces(dev_nonces)) } } #[cfg(feature = "sqlite")] impl serialize::ToSql for DevNonces { - fn to_sql<'b>(&self, out: &mut serialize::Output<'b, '_, Sqlite>) -> serialize::Result { - out.set_value(serde_json::to_string(self)?); + fn to_sql<'b>(&'b self, out: &mut serialize::Output<'b, '_, Sqlite>) -> serialize::Result { + out.set_value(serde_json::to_string(&self.0)?); Ok(serialize::IsNull::No) } } diff --git a/chirpstack/src/storage/fields/mod.rs b/chirpstack/src/storage/fields/mod.rs index 57fc53dd..75e80363 100644 --- a/chirpstack/src/storage/fields/mod.rs +++ b/chirpstack/src/storage/fields/mod.rs @@ -7,7 +7,7 @@ mod multicast_group_scheduling_type; mod uuid; pub use big_decimal::BigDecimal; -pub use dev_nonces::*; +pub use dev_nonces::DevNonces; pub use device_session::DeviceSession; pub use key_value::KeyValue; pub use measurements::*; diff --git a/chirpstack/src/storage/schema_postgres.rs b/chirpstack/src/storage/schema_postgres.rs index c4d27e9c..1eeae44a 100644 --- a/chirpstack/src/storage/schema_postgres.rs +++ b/chirpstack/src/storage/schema_postgres.rs @@ -75,7 +75,7 @@ diesel::table! { updated_at -> Timestamptz, nwk_key -> Bytea, app_key -> Bytea, - dev_nonces -> Array>, + dev_nonces -> Jsonb, join_nonce -> Int4, } } diff --git a/chirpstack/src/test/otaa_pr_test.rs b/chirpstack/src/test/otaa_pr_test.rs index 4a79e027..ecad919c 100644 --- a/chirpstack/src/test/otaa_pr_test.rs +++ b/chirpstack/src/test/otaa_pr_test.rs @@ -15,7 +15,7 @@ use crate::storage::{ device::{self, DeviceClass}, device_keys, device_profile, gateway, tenant, }; -use crate::{config, test, uplink}; +use crate::{config, storage::fields, test, uplink}; use chirpstack_api::gw; use lrwn::{AES128Key, EUI64Prefix, NetID, EUI64}; @@ -314,7 +314,7 @@ async fn test_sns() { let dk = device_keys::create(device_keys::DeviceKeys { dev_eui: dev.dev_eui, nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), - dev_nonces: vec![].into(), + dev_nonces: fields::DevNonces::default(), ..Default::default() }) .await @@ -499,7 +499,7 @@ async fn test_sns_roaming_not_allowed() { let dk = device_keys::create(device_keys::DeviceKeys { dev_eui: dev.dev_eui, nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), - dev_nonces: vec![].into(), + dev_nonces: fields::DevNonces::default(), ..Default::default() }) .await diff --git a/chirpstack/src/test/otaa_test.rs b/chirpstack/src/test/otaa_test.rs index 91bed7e1..84622d87 100644 --- a/chirpstack/src/test/otaa_test.rs +++ b/chirpstack/src/test/otaa_test.rs @@ -10,7 +10,9 @@ use crate::storage::{ device::{self, DeviceClass}, device_keys, device_profile, gateway, tenant, }; -use crate::{config, gateway::backend as gateway_backend, integration, region, test, uplink}; +use crate::{ + config, gateway::backend as gateway_backend, integration, region, storage::fields, test, uplink, +}; use chirpstack_api::{common, gw, internal, stream}; use lrwn::keys::get_js_int_key; use lrwn::{AES128Key, EUI64}; @@ -101,7 +103,11 @@ async fn test_gateway_filtering() { let dk = device_keys::create(device_keys::DeviceKeys { dev_eui: dev.dev_eui, nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), - dev_nonces: vec![Some(258)].into(), + dev_nonces: { + let mut dev_nonces = fields::DevNonces::default(); + dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258); + dev_nonces + }, ..Default::default() }) .await @@ -273,7 +279,11 @@ async fn test_lorawan_10() { let dk = device_keys::create(device_keys::DeviceKeys { dev_eui: dev.dev_eui, nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), - dev_nonces: vec![Some(258)].into(), + dev_nonces: { + let mut dev_nonces = fields::DevNonces::default(); + dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258); + dev_nonces + }, ..Default::default() }) .await @@ -929,7 +939,11 @@ async fn test_lorawan_11() { dev_eui: dev.dev_eui, nwk_key: AES128Key::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), app_key: AES128Key::from_bytes([16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]), - dev_nonces: vec![Some(258)].into(), + dev_nonces: { + let mut dev_nonces = fields::DevNonces::default(); + dev_nonces.insert(EUI64::from_be_bytes([1, 2, 3, 4, 5, 6, 7, 8]), 258); + dev_nonces + }, ..Default::default() }) .await diff --git a/chirpstack/src/uplink/join.rs b/chirpstack/src/uplink/join.rs index 29b111dd..0323e3ae 100644 --- a/chirpstack/src/uplink/join.rs +++ b/chirpstack/src/uplink/join.rs @@ -456,8 +456,9 @@ impl JoinRequest { self.device_keys = Some( match device_keys::validate_incr_join_and_store_dev_nonce( - &dev.dev_eui, - join_request.dev_nonce as i32, + join_request.join_eui, + dev.dev_eui, + join_request.dev_nonce, ) .await { diff --git a/chirpstack/src/uplink/join_sns.rs b/chirpstack/src/uplink/join_sns.rs index 6736c533..74ff3bd1 100644 --- a/chirpstack/src/uplink/join_sns.rs +++ b/chirpstack/src/uplink/join_sns.rs @@ -353,8 +353,9 @@ impl JoinRequest { self.device_keys = Some( match device_keys::validate_incr_join_and_store_dev_nonce( - &dev.dev_eui, - join_request.dev_nonce as i32, + join_request.join_eui, + dev.dev_eui, + join_request.dev_nonce, ) .await {