ExternalVendorCode/chirpstack
1
0
Fork 0
mirror of https://github.com/chirpstack/chirpstack.git synced 2025-03-15 00:36:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update fuota API. Add options for auto-calculation of params.

Browse Source 
This adds options to auto-calculate the fragment size (based on max.
payload size available for the given data-rate) and multicast
timeout (based on server settings).
This commit is contained in:
Orne Brocaar 2025-02-10 15:20:21 +00:00
parent 1a51aa836d
commit 41d25ae2fb
8 changed files with 324 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
api/proto/api/fuota.proto vendored
View File

@ -112,39 +112,47 @@ message FuotaDeployment {
// has a different meaning for Class-B and Class-C groups.
uint32 multicast_timeout = 10;
// Unicast attempt count.
// The number of attempts before considering an unicast command
// to be failed.
uint32 unicast_attempt_count = 11;
// Calculate multicast timeout.
// If set to true, ChirpStack will calculate the multicast-timeout.
bool calculate_multicast_timeout = 11;
// The number of times ChirpStack will retry an unicast command
// before it considers it to be failed.
uint32 unicast_max_retry_count = 12;
// Fragmentation size.
// This defines the size of each payload fragment. Please refer to the
// Regional Parameters specification for the maximum payload sizes
// per data-rate and region.
uint32 fragmentation_fragment_size = 12;
uint32 fragmentation_fragment_size = 13;
// Fragmentation redundancy.
// The number represents the additional redundant frames to send.
uint32 fragmentation_redundancy = 13;
// Calculate fragmentation size.
// If set to true, ChirpStack will calculate the fragmentation size.
bool calculate_fragmentation_fragment_size = 14;
// Fragmentation redundancy percentage.
// The number represents the percentage (0 - 100) of redundant messages
// to send.
uint32 fragmentation_redundancy_percentage = 15;
// Fragmentation session index.
uint32 fragmentation_session_index = 14;
uint32 fragmentation_session_index = 16;
// Fragmentation matrix.
uint32 fragmentation_matrix = 15;
uint32 fragmentation_matrix = 17;
// Block ack delay.
uint32 fragmentation_block_ack_delay = 16;
uint32 fragmentation_block_ack_delay = 18;
// Descriptor (4 bytes).
bytes fragmentation_descriptor = 17;
bytes fragmentation_descriptor = 19;
// Request fragmentation session status.
RequestFragmentationSessionStatus request_fragmentation_session_status = 18;
RequestFragmentationSessionStatus request_fragmentation_session_status = 20;
// Payload.
// The FUOTA payload to send.
bytes payload = 19;
bytes payload = 21;
}
message FuotaDeploymentListItem {
@ -228,6 +236,12 @@ message GetFuotaDeploymentResponse {
// Updated at timestamp.
google.protobuf.Timestamp updated_at = 3;
// Started at timestamp.
google.protobuf.Timestamp started_at = 4;
// Completed at timestamp.
google.protobuf.Timestamp completed_at = 5;
}
message UpdateFuotaDeploymentRequest {

42
api/rust/proto/chirpstack/api/fuota.proto vendored
View File

@ -112,39 +112,47 @@ message FuotaDeployment {
// has a different meaning for Class-B and Class-C groups.
uint32 multicast_timeout = 10;
// Unicast attempt count.
// The number of attempts before considering an unicast command
// to be failed.
uint32 unicast_attempt_count = 11;
// Calculate multicast timeout.
// If set to true, ChirpStack will calculate the multicast-timeout.
bool calculate_multicast_timeout = 11;
// The number of times ChirpStack will retry an unicast command
// before it considers it to be failed.
uint32 unicast_max_retry_count = 12;
// Fragmentation size.
// This defines the size of each payload fragment. Please refer to the
// Regional Parameters specification for the maximum payload sizes
// per data-rate and region.
uint32 fragmentation_fragment_size = 12;
uint32 fragmentation_fragment_size = 13;
// Fragmentation redundancy.
// The number represents the additional redundant frames to send.
uint32 fragmentation_redundancy = 13;
// Calculate fragmentation size.
// If set to true, ChirpStack will calculate the fragmentation size.
bool calculate_fragmentation_fragment_size = 14;
// Fragmentation redundancy percentage.
// The number represents the percentage (0 - 100) of redundant messages
// to send.
uint32 fragmentation_redundancy_percentage = 15;
// Fragmentation session index.
uint32 fragmentation_session_index = 14;
uint32 fragmentation_session_index = 16;
// Fragmentation matrix.
uint32 fragmentation_matrix = 15;
uint32 fragmentation_matrix = 17;
// Block ack delay.
uint32 fragmentation_block_ack_delay = 16;
uint32 fragmentation_block_ack_delay = 18;
// Descriptor (4 bytes).
bytes fragmentation_descriptor = 17;
bytes fragmentation_descriptor = 19;
// Request fragmentation session status.
RequestFragmentationSessionStatus request_fragmentation_session_status = 18;
RequestFragmentationSessionStatus request_fragmentation_session_status = 20;
// Payload.
// The FUOTA payload to send.
bytes payload = 19;
bytes payload = 21;
}
message FuotaDeploymentListItem {
@ -228,6 +236,12 @@ message GetFuotaDeploymentResponse {
// Updated at timestamp.
google.protobuf.Timestamp updated_at = 3;
// Started at timestamp.
google.protobuf.Timestamp started_at = 4;
// Completed at timestamp.
google.protobuf.Timestamp completed_at = 5;
}
message UpdateFuotaDeploymentRequest {

6
chirpstack/migrations_postgres/2025-01-21-093745_add_fuota_support/up.sql
View File

@ -13,9 +13,9 @@ create table fuota_deployment (
multicast_class_b_ping_slot_nb_k smallint not null,
multicast_frequency bigint not null,
multicast_timeout smallint not null,
unicast_attempt_count smallint not null,
unicast_max_retry_count smallint not null,
fragmentation_fragment_size smallint not null,
fragmentation_redundancy smallint not null,
fragmentation_redundancy_percentage smallint not null,
fragmentation_session_index smallint not null,
fragmentation_matrix smallint not null,
fragmentation_block_ack_delay smallint not null,
@ -51,7 +51,7 @@ create table fuota_deployment_job (
job varchar(20) not null,
created_at timestamp with time zone not null,
completed_at timestamp with time zone null,
max_attempt_count smallint not null,
max_retry_count smallint not null,
attempt_count smallint not null,
scheduler_run_after timestamp with time zone not null,

6
chirpstack/migrations_sqlite/2025-01-27-100007_add_fuota_support/up.sql
View File

@ -13,9 +13,9 @@ create table fuota_deployment (
multicast_class_b_ping_slot_nb_k smallint not null,
multicast_frequency bigint not null,
multicast_timeout smallint not null,
unicast_attempt_count smallint not null,
unicast_max_retry_count smallint not null,
fragmentation_fragment_size smallint not null,
fragmentation_redundancy smallint not null,
fragmentation_redundancy_percentage smallint not null,
fragmentation_session_index smallint not null,
fragmentation_matrix smallint not null,
fragmentation_block_ack_delay smallint not null,
@ -51,7 +51,7 @@ create table fuota_deployment_job (
job varchar(20) not null,
created_at datetime not null,
completed_at datetime null,
max_attempt_count smallint not null,
max_retry_count smallint not null,
attempt_count smallint not null,
scheduler_run_after datetime not null,

64
chirpstack/src/api/fuota.rs
View File

@ -1,5 +1,6 @@
use std::str::FromStr;
use chrono::Utc;
use tonic::{Request, Response, Status};
use uuid::Uuid;
@ -45,7 +46,7 @@ impl FuotaService for Fuota {
)
.await?;
let dp = fuota::FuotaDeployment {
let mut dp = fuota::FuotaDeployment {
name: req_dp.name.clone(),
application_id: app_id.into(),
device_profile_id: dp_id.into(),
@ -61,9 +62,9 @@ impl FuotaService for Fuota {
multicast_class_b_ping_slot_nb_k: req_dp.multicast_class_b_ping_slot_nb_k as i16,
multicast_frequency: req_dp.multicast_frequency as i64,
multicast_timeout: req_dp.multicast_timeout as i16,
unicast_attempt_count: req_dp.unicast_attempt_count as i16,
unicast_max_retry_count: req_dp.unicast_max_retry_count as i16,
fragmentation_fragment_size: req_dp.fragmentation_fragment_size as i16,
fragmentation_redundancy: req_dp.fragmentation_redundancy as i16,
fragmentation_redundancy_percentage: req_dp.fragmentation_redundancy_percentage as i16,
fragmentation_session_index: req_dp.fragmentation_session_index as i16,
fragmentation_matrix: req_dp.fragmentation_matrix as i16,
fragmentation_block_ack_delay: req_dp.fragmentation_block_ack_delay as i16,
@ -74,6 +75,16 @@ impl FuotaService for Fuota {
payload: req_dp.payload.clone(),
..Default::default()
};
if req_dp.calculate_fragmentation_fragment_size {
dp.fragmentation_fragment_size = fuota::get_max_fragment_size(&dp)
.await
.map_err(|e| e.status())? as i16;
}
if req_dp.calculate_multicast_timeout {
dp.multicast_timeout =
fuota::get_multicast_timeout(&dp).map_err(|e| e.status())? as i16;
}
let dp = fuota::create_deployment(dp).await.map_err(|e| e.status())?;
let mut resp = Response::new(api::CreateFuotaDeploymentResponse {
@ -123,9 +134,9 @@ impl FuotaService for Fuota {
multicast_class_b_ping_slot_nb_k: dp.multicast_class_b_ping_slot_nb_k as u32,
multicast_frequency: dp.multicast_frequency as u32,
multicast_timeout: dp.multicast_timeout as u32,
unicast_attempt_count: dp.unicast_attempt_count as u32,
unicast_max_retry_count: dp.unicast_max_retry_count as u32,
fragmentation_fragment_size: dp.fragmentation_fragment_size as u32,
fragmentation_redundancy: dp.fragmentation_redundancy as u32,
fragmentation_redundancy_percentage: dp.fragmentation_redundancy_percentage as u32,
fragmentation_session_index: dp.fragmentation_session_index as u32,
fragmentation_matrix: dp.fragmentation_matrix as u32,
fragmentation_block_ack_delay: dp.fragmentation_block_ack_delay as u32,
@ -135,9 +146,19 @@ impl FuotaService for Fuota {
.to_proto()
.into(),
payload: dp.payload.clone(),
calculate_multicast_timeout: false,
calculate_fragmentation_fragment_size: false,
}),
created_at: Some(helpers::datetime_to_prost_timestamp(&dp.created_at)),
updated_at: Some(helpers::datetime_to_prost_timestamp(&dp.updated_at)),
started_at: dp
.started_at
.as_ref()
.map(helpers::datetime_to_prost_timestamp),
completed_at: dp
.completed_at
.as_ref()
.map(helpers::datetime_to_prost_timestamp),
});
resp.metadata_mut()
.insert("x-log-fuota_deployment_id", req.id.parse().unwrap());
@ -167,7 +188,7 @@ impl FuotaService for Fuota {
)
.await?;
let _ = fuota::update_deployment(fuota::FuotaDeployment {
let mut dp = fuota::FuotaDeployment {
id: id.into(),
name: req_dp.name.clone(),
application_id: app_id.into(),
@ -184,9 +205,9 @@ impl FuotaService for Fuota {
multicast_class_b_ping_slot_nb_k: req_dp.multicast_class_b_ping_slot_nb_k as i16,
multicast_frequency: req_dp.multicast_frequency as i64,
multicast_timeout: req_dp.multicast_timeout as i16,
unicast_attempt_count: req_dp.unicast_attempt_count as i16,
unicast_max_retry_count: req_dp.unicast_max_retry_count as i16,
fragmentation_fragment_size: req_dp.fragmentation_fragment_size as i16,
fragmentation_redundancy: req_dp.fragmentation_redundancy as i16,
fragmentation_redundancy_percentage: req_dp.fragmentation_redundancy_percentage as i16,
fragmentation_session_index: req_dp.fragmentation_session_index as i16,
fragmentation_matrix: req_dp.fragmentation_matrix as i16,
fragmentation_block_ack_delay: req_dp.fragmentation_block_ack_delay as i16,
@ -196,9 +217,18 @@ impl FuotaService for Fuota {
.from_proto(),
payload: req_dp.payload.clone(),
..Default::default()
})
.await
.map_err(|e| e.status())?;
};
if req_dp.calculate_fragmentation_fragment_size {
dp.fragmentation_fragment_size = fuota::get_max_fragment_size(&dp)
.await
.map_err(|e| e.status())? as i16;
}
if req_dp.calculate_multicast_timeout {
dp.multicast_timeout =
fuota::get_multicast_timeout(&dp).map_err(|e| e.status())? as i16;
}
let _ = fuota::update_deployment(dp).await.map_err(|e| e.status())?;
let mut resp = Response::new(());
resp.metadata_mut()
@ -242,12 +272,20 @@ impl FuotaService for Fuota {
)
.await?;
let d = fuota::get_deployment(id).await.map_err(|e| e.status())?;
let mut d = fuota::get_deployment(id).await.map_err(|e| e.status())?;
if d.started_at.is_some() {
return Err(Status::failed_precondition(
"FUOTA deployment has already started",
));
}
d.started_at = Some(Utc::now());
let d = fuota::update_deployment(d).await.map_err(|e| e.status())?;
fuota::create_job(fuota::FuotaDeploymentJob {
fuota_deployment_id: d.id,
job: fields::FuotaJob::McGroupSetup,
max_attempt_count: d.unicast_attempt_count,
max_retry_count: d.unicast_max_retry_count,
..Default::default()
})
.await

215
chirpstack/src/storage/fuota.rs
View File

@ -32,9 +32,9 @@ pub struct FuotaDeployment {
pub multicast_class_b_ping_slot_nb_k: i16,
pub multicast_frequency: i64,
pub multicast_timeout: i16,
pub unicast_attempt_count: i16,
pub unicast_max_retry_count: i16,
pub fragmentation_fragment_size: i16,
pub fragmentation_redundancy: i16,
pub fragmentation_redundancy_percentage: i16,
pub fragmentation_session_index: i16,
pub fragmentation_matrix: i16,
pub fragmentation_block_ack_delay: i16,
@ -62,9 +62,9 @@ impl Default for FuotaDeployment {
multicast_class_b_ping_slot_nb_k: 0,
multicast_frequency: 0,
multicast_timeout: 0,
unicast_attempt_count: 0,
unicast_max_retry_count: 0,
fragmentation_fragment_size: 0,
fragmentation_redundancy: 0,
fragmentation_redundancy_percentage: 0,
fragmentation_session_index: 0,
fragmentation_matrix: 0,
fragmentation_block_ack_delay: 0,
@ -141,7 +141,7 @@ pub struct FuotaDeploymentJob {
pub job: fields::FuotaJob,
pub created_at: DateTime<Utc>,
pub completed_at: Option<DateTime<Utc>>,
pub max_attempt_count: i16,
pub max_retry_count: i16,
pub attempt_count: i16,
pub scheduler_run_after: DateTime<Utc>,
}
@ -155,7 +155,7 @@ impl Default for FuotaDeploymentJob {
job: fields::FuotaJob::McGroupSetup,
created_at: now,
completed_at: None,
max_attempt_count: 0,
max_retry_count: 0,
attempt_count: 0,
scheduler_run_after: now,
}
@ -208,9 +208,10 @@ pub async fn update_deployment(d: FuotaDeployment) -> Result<FuotaDeployment, Er
.eq(&d.multicast_class_b_ping_slot_nb_k),
fuota_deployment::multicast_frequency.eq(&d.multicast_frequency),
fuota_deployment::multicast_timeout.eq(&d.multicast_timeout),
fuota_deployment::unicast_attempt_count.eq(&d.unicast_attempt_count),
fuota_deployment::unicast_max_retry_count.eq(&d.unicast_max_retry_count),
fuota_deployment::fragmentation_fragment_size.eq(&d.fragmentation_fragment_size),
fuota_deployment::fragmentation_redundancy.eq(&d.fragmentation_redundancy),
fuota_deployment::fragmentation_redundancy_percentage
.eq(&d.fragmentation_redundancy_percentage),
fuota_deployment::fragmentation_session_index.eq(&d.fragmentation_session_index),
fuota_deployment::fragmentation_matrix.eq(&d.fragmentation_matrix),
fuota_deployment::fragmentation_block_ack_delay.eq(&d.fragmentation_block_ack_delay),
@ -587,6 +588,69 @@ pub async fn get_schedulable_jobs(limit: usize) -> Result<Vec<FuotaDeploymentJob
.context("Get FUOTA jobs")
}
pub async fn get_max_fragment_size(d: &FuotaDeployment) -> Result<usize> {
let dp = device_profile::get(&d.device_profile_id).await?;
let region_conf = lrwn::region::get(dp.region, false, false);
let max_pl_size = region_conf
.get_max_payload_size(dp.mac_version, dp.reg_params_revision, d.multicast_dr as u8)?
.n
- 3;
Ok(max_pl_size)
}
pub fn get_multicast_timeout(d: &FuotaDeployment) -> Result<usize> {
let conf = config::get();
let fragments = (d.payload.len() as f32 / d.fragmentation_fragment_size as f32).ceil() as usize;
let redundancy =
(fragments as f32 * d.fragmentation_redundancy_percentage as f32 / 100.0).ceil() as usize;
let total_fragments = fragments + redundancy;
match d.multicast_group_type.as_ref() {
"B" => {
// Calculate number of ping-slots per beacon period.
let nb_ping_slots = 1 << (d.multicast_class_b_ping_slot_nb_k as usize);
// Calculate number of beacon-periods needed.
// One beacon period is added as the first ping-slot might be in the next beacon-period.
let beacon_periods =
(total_fragments as f32 / nb_ping_slots as f32).ceil() as usize + 1;
// Calculate the timeout value. In case of Class-B, timeout represents the number
// of beacon periods (beacon periods = 2^timeout).
for i in 0..16 {
// i is 0-15
if (1 << i) >= beacon_periods {
return Ok(i);
}
}
Err(anyhow!("Max. number of beacon period exceeded"))
}
"C" => {
// Get the margin between each multicast Class-C downlink.
let mc_class_c_margin_secs =
conf.network.scheduler.multicast_class_c_margin.as_secs() as usize;
// Multiply by the number of fragments (+1 for additional margin).
let mc_class_c_duration_secs = mc_class_c_margin_secs * (total_fragments + 1 as usize);
// Calculate the timeout value. In case of Class-B, timeout is defined as seconds,
// where the number of seconds is 2^timeout.
for i in 0..16 {
// i = 0-15
if (1 << i) >= mc_class_c_duration_secs {
return Ok(i);
}
}
Err(anyhow!("Max timeout exceeded"))
}
_ => Ok(0),
}
}
#[cfg(test)]
mod test {
use super::*;
@ -893,7 +957,7 @@ mod test {
let mut job = create_job(FuotaDeploymentJob {
fuota_deployment_id: d.id,
job: fields::FuotaJob::McGroupSetup,
max_attempt_count: 3,
max_retry_count: 3,
attempt_count: 1,
..Default::default()
})
@ -915,7 +979,7 @@ mod test {
let job2 = create_job(FuotaDeploymentJob {
fuota_deployment_id: d.id,
job: fields::FuotaJob::FragStatus,
max_attempt_count: 3,
max_retry_count: 3,
attempt_count: 1,
..Default::default()
})
@ -937,4 +1001,135 @@ mod test {
let jobs = get_schedulable_jobs(10).await.unwrap();
assert_eq!(0, jobs.len());
}
#[tokio::test]
async fn test_get_max_fragment_size() {
let _guard = test::prepare().await;
let t = tenant::create(tenant::Tenant {
name: "test-tenant".into(),
..Default::default()
})
.await
.unwrap();
let app = application::create(application::Application {
name: "test-app".into(),
tenant_id: t.id,
..Default::default()
})
.await
.unwrap();
let dp = device_profile::create(device_profile::DeviceProfile {
tenant_id: t.id,
name: "test-dp".into(),
..Default::default()
})
.await
.unwrap();
// create
let d = create_deployment(FuotaDeployment {
application_id: app.id,
device_profile_id: dp.id,
name: "test-fuota-deployment".into(),
multicast_dr: 5,
..Default::default()
})
.await
.unwrap();
assert_eq!(239, get_max_fragment_size(&d).await.unwrap());
}
#[tokio::test]
async fn test_get_multicast_timeout() {
let _guard = test::prepare().await;
struct Test {
name: String,
deployment: FuotaDeployment,
expected_timeout: usize,
expected_error: Option<String>,
}
let tests = [
Test {
name: "Class-B - 1 / beacon period - 15 fragments".into(),
deployment: FuotaDeployment {
multicast_group_type: "B".into(),
multicast_class_b_ping_slot_nb_k: 0,
fragmentation_fragment_size: 10,
fragmentation_redundancy_percentage: 50,
payload: vec![0; 100],
..Default::default()
},
expected_timeout: 4,
expected_error: None,
},
Test {
name: "Class-B - 1 / beacon period - 16 fragments".into(),
deployment: FuotaDeployment {
multicast_group_type: "B".into(),
multicast_class_b_ping_slot_nb_k: 0,
fragmentation_fragment_size: 10,
fragmentation_redundancy_percentage: 60,
payload: vec![0; 100],
..Default::default()
},
expected_timeout: 5,
expected_error: None,
},
Test {
name: "Class-B - 16 / beacon period - 16 fragments".into(),
deployment: FuotaDeployment {
multicast_group_type: "B".into(),
multicast_class_b_ping_slot_nb_k: 4,
fragmentation_fragment_size: 10,
fragmentation_redundancy_percentage: 60,
payload: vec![0; 100],
..Default::default()
},
expected_timeout: 1,
expected_error: None,
},
Test {
name: "Class-B - 16 / beacon period - 17 fragments".into(),
deployment: FuotaDeployment {
multicast_group_type: "B".into(),
multicast_class_b_ping_slot_nb_k: 4,
fragmentation_fragment_size: 10,
fragmentation_redundancy_percentage: 70,
payload: vec![0; 100],
..Default::default()
},
expected_timeout: 2,
expected_error: None,
},
Test {
name: "Class-C - 1 fragment".into(),
deployment: FuotaDeployment {
multicast_group_type: "C".into(),
fragmentation_fragment_size: 10,
payload: vec![0; 10],
..Default::default()
},
expected_timeout: 3,
expected_error: None,
},
];
for t in &tests {
println!("> {}", t.name);
let res = get_multicast_timeout(&t.deployment);
if let Some(err_str) = &t.expected_error {
assert!(res.is_err());
assert_eq!(err_str, &res.err().unwrap().to_string());
} else {
assert!(res.is_ok());
assert_eq!(t.expected_timeout, res.unwrap());
}
}
}
}

6
chirpstack/src/storage/schema_postgres.rs
View File

@ -200,9 +200,9 @@ diesel::table! {
multicast_class_b_ping_slot_nb_k -> Int2,
multicast_frequency -> Int8,
multicast_timeout -> Int2,
unicast_attempt_count -> Int2,
unicast_max_retry_count -> Int2,
fragmentation_fragment_size -> Int2,
fragmentation_redundancy -> Int2,
fragmentation_redundancy_percentage -> Int2,
fragmentation_session_index -> Int2,
fragmentation_matrix -> Int2,
fragmentation_block_ack_delay -> Int2,
@ -241,7 +241,7 @@ diesel::table! {
job -> Varchar,
created_at -> Timestamptz,
completed_at -> Nullable<Timestamptz>,
max_attempt_count -> Int2,
max_retry_count -> Int2,
attempt_count -> Int2,
scheduler_run_after -> Timestamptz,
}

6
chirpstack/src/storage/schema_sqlite.rs
View File

@ -177,9 +177,9 @@ diesel::table! {
multicast_class_b_ping_slot_nb_k -> SmallInt,
multicast_frequency -> BigInt,
multicast_timeout -> SmallInt,
unicast_attempt_count -> SmallInt,
unicast_max_retry_count -> SmallInt,
fragmentation_fragment_size -> SmallInt,
fragmentation_redundancy -> SmallInt,
fragmentation_redundancy_percentage -> SmallInt,
fragmentation_session_index -> SmallInt,
fragmentation_matrix -> SmallInt,
fragmentation_block_ack_delay -> SmallInt,
@ -216,7 +216,7 @@ diesel::table! {
job -> Text,
created_at -> TimestamptzSqlite,
completed_at -> Nullable<TimestamptzSqlite>,
max_attempt_count -> SmallInt,
max_retry_count -> SmallInt,
attempt_count -> SmallInt,
scheduler_run_after -> TimestamptzSqlite,
}