ExternalVendorCode/chirpstack
1
0
Fork 0
mirror of https://github.com/chirpstack/chirpstack.git synced 2025-02-06 18:59:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor code to use diesel-async.

Browse Source 
This still depends on unreleased diesel and diesel-async code. As soon
as new diesel and diesel-async code has been released, we can remove
the [patch.crates-io] from Cargo.toml.
This commit is contained in:
Orne Brocaar 2023-11-28 13:12:18 +00:00
parent 3f57609981
commit 8e2eda3d5b
30 changed files with 3384 additions and 4122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
Cargo.lock generated
View File

@ -759,9 +759,11 @@ dependencies = [
"chrono", "chrono",
"clap", "clap",
"diesel", "diesel",
"diesel-async",
"diesel_migrations", "diesel_migrations",
"dotenv", "dotenv",
"futures", "futures",
"futures-util",
"gcp_auth", "gcp_auth",
"geohash", "geohash",
"handlebars", "handlebars",
@ -796,6 +798,9 @@ dependencies = [
"reqwest", "reqwest",
"rquickjs", "rquickjs",
"rust-embed", "rust-embed",
"rustls",
"rustls-native-certs",
"rustls-pemfile",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
@ -804,6 +809,8 @@ dependencies = [
"thiserror", "thiserror",
"tokio", "tokio",
"tokio-executor-trait", "tokio-executor-trait",
"tokio-postgres",
"tokio-postgres-rustls",
"tokio-reactor-trait", "tokio-reactor-trait",
"tokio-stream", "tokio-stream",
"toml", "toml",
@ -1172,6 +1179,24 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "deadpool"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb84100978c1c7b37f09ed3ce3e5f843af02c2a2c431bae5b19230dad2c1b490"
dependencies = [
"async-trait",
"deadpool-runtime",
"num_cpus",
"tokio",
]
[[package]]
name = "deadpool-runtime"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49"
[[package]] [[package]]
name = "der" name = "der"
version = "0.7.8" version = "0.7.8"
@ -1204,9 +1229,8 @@ dependencies = [
[[package]] [[package]]
name = "diesel" name = "diesel"
version = "2.1.4" version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/diesel-rs/diesel.git?rev=566dcccc6df6adb6ceddef8df5e1806e2a065c40#566dcccc6df6adb6ceddef8df5e1806e2a065c40"
checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
dependencies = [ dependencies = [
"bigdecimal", "bigdecimal",
"bitflags 2.4.1", "bitflags 2.4.1",
@ -1217,19 +1241,31 @@ dependencies = [
"num-bigint", "num-bigint",
"num-integer", "num-integer",
"num-traits", "num-traits",
"pq-sys",
"r2d2",
"serde_json", "serde_json",
"uuid", "uuid",
] ]
[[package]]
name = "diesel-async"
version = "0.4.1"
source = "git+https://github.com/weiznich/diesel_async.git?rev=017ebe2fb7a2709ab5db92148dea5ce812a35e09#017ebe2fb7a2709ab5db92148dea5ce812a35e09"
dependencies = [
"async-trait",
"deadpool",
"diesel",
"futures-util",
"scoped-futures",
"tokio",
"tokio-postgres",
]
[[package]] [[package]]
name = "diesel_derives" name = "diesel_derives"
version = "2.1.2" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/diesel-rs/diesel.git?rev=566dcccc6df6adb6ceddef8df5e1806e2a065c40#566dcccc6df6adb6ceddef8df5e1806e2a065c40"
checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44"
dependencies = [ dependencies = [
"diesel_table_macro_syntax", "diesel_table_macro_syntax",
"dsl_auto_type",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.39",
@ -1249,8 +1285,7 @@ dependencies = [
[[package]] [[package]]
name = "diesel_table_macro_syntax" name = "diesel_table_macro_syntax"
version = "0.1.0" version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/diesel-rs/diesel.git?rev=566dcccc6df6adb6ceddef8df5e1806e2a065c40#566dcccc6df6adb6ceddef8df5e1806e2a065c40"
checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
dependencies = [ dependencies = [
"syn 2.0.39", "syn 2.0.39",
] ]
@ -1306,6 +1341,19 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dsl_auto_type"
version = "0.1.0"
source = "git+https://github.com/diesel-rs/diesel.git?rev=566dcccc6df6adb6ceddef8df5e1806e2a065c40#566dcccc6df6adb6ceddef8df5e1806e2a065c40"
dependencies = [
"darling",
"either",
"heck",
"proc-macro2",
"quote",
"syn 2.0.39",
]
[[package]] [[package]]
name = "dtoa" name = "dtoa"
version = "1.0.9" version = "1.0.9"
@ -1453,6 +1501,12 @@ dependencies = [
"async-trait", "async-trait",
] ]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "1.9.0" version = "1.9.0"
@ -1484,6 +1538,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7"
[[package]]
name = "finl_unicode"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6"
[[package]] [[package]]
name = "fixedbitset" name = "fixedbitset"
version = "0.4.2" version = "0.4.2"
@ -2410,6 +2470,16 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.6.4" version = "2.6.4"
@ -2968,6 +3038,15 @@ dependencies = [
"indexmap 2.1.0", "indexmap 2.1.0",
] ]
[[package]]
name = "phf"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
dependencies = [
"phf_shared 0.11.2",
]
[[package]] [[package]]
name = "phf_shared" name = "phf_shared"
version = "0.10.0" version = "0.10.0"
@ -2977,6 +3056,15 @@ dependencies = [
"siphasher", "siphasher",
] ]
[[package]]
name = "phf_shared"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
dependencies = [
"siphasher",
]
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.1.3" version = "1.1.3"
@ -3095,6 +3183,35 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "postgres-protocol"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520"
dependencies = [
"base64 0.21.5",
"byteorder",
"bytes",
"fallible-iterator",
"hmac",
"md-5",
"memchr",
"rand",
"sha2",
"stringprep",
]
[[package]]
name = "postgres-types"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c"
dependencies = [
"bytes",
"fallible-iterator",
"postgres-protocol",
]
[[package]] [[package]]
name = "powerfmt" name = "powerfmt"
version = "0.2.0" version = "0.2.0"
@ -3107,15 +3224,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "pq-sys"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd"
dependencies = [
"vcpkg",
]
[[package]] [[package]]
name = "precomputed-hash" name = "precomputed-hash"
version = "0.1.1" version = "0.1.1"
@ -3733,6 +3841,16 @@ dependencies = [
"parking_lot", "parking_lot",
] ]
[[package]]
name = "scoped-futures"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1473e24c637950c9bd38763220bea91ec3e095a89f672bbd7a10d03e77ba467"
dependencies = [
"cfg-if",
"pin-utils",
]
[[package]] [[package]]
name = "scoped-tls" name = "scoped-tls"
version = "1.0.1" version = "1.0.1"
@ -4110,10 +4228,21 @@ dependencies = [
"new_debug_unreachable", "new_debug_unreachable",
"once_cell", "once_cell",
"parking_lot", "parking_lot",
"phf_shared", "phf_shared 0.10.0",
"precomputed-hash", "precomputed-hash",
] ]
[[package]]
name = "stringprep"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6"
dependencies = [
"finl_unicode",
"unicode-bidi",
"unicode-normalization",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.10.0" version = "0.10.0"
@ -4344,6 +4473,46 @@ dependencies = [
"syn 2.0.39", "syn 2.0.39",
] ]
[[package]]
name = "tokio-postgres"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8"
dependencies = [
"async-trait",
"byteorder",
"bytes",
"fallible-iterator",
"futures-channel",
"futures-util",
"log",
"parking_lot",
"percent-encoding",
"phf",
"pin-project-lite",
"postgres-protocol",
"postgres-types",
"rand",
"socket2 0.5.5",
"tokio",
"tokio-util",
"whoami",
]
[[package]]
name = "tokio-postgres-rustls"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f"
dependencies = [
"futures",
"ring 0.16.20",
"rustls",
"tokio",
"tokio-postgres",
"tokio-rustls",
]
[[package]] [[package]]
name = "tokio-reactor-trait" name = "tokio-reactor-trait"
version = "1.1.0" version = "1.1.0"
@ -4934,6 +5103,16 @@ dependencies = [
"rustix 0.38.25", "rustix 0.38.25",
] ]
[[package]]
name = "whoami"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
dependencies = [
"wasm-bindgen",
"web-sys",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

4
Cargo.toml
View File

@ -13,3 +13,7 @@ members = [
opt-level = 'z' opt-level = 'z'
lto = true lto = true
codegen-units = 1 codegen-units = 1
[patch.crates-io]
diesel = { git = "https://github.com/diesel-rs/diesel.git", rev = "566dcccc6df6adb6ceddef8df5e1806e2a065c40" }
diesel-async = { git = "https://github.com/weiznich/diesel_async.git", rev = "017ebe2fb7a2709ab5db92148dea5ce812a35e09" }

10
chirpstack/Cargo.toml
View File

@ -26,14 +26,16 @@ handlebars = "4.4"
validator = "0.16" validator = "0.16"
diesel = { version = "2.1", features = [ diesel = { version = "2.1", features = [
"chrono", "chrono",
"postgres",
"r2d2",
"uuid", "uuid",
"serde_json", "serde_json",
"numeric", "numeric",
"64-column-tables", "64-column-tables",
"postgres_backend",
] } ] }
diesel_migrations = { version = "2.1" } diesel_migrations = { version = "2.1" }
diesel-async = { version = "0.4", features = ["deadpool", "postgres", "async-connection-wrapper"] }
tokio-postgres = "0.7"
tokio-postgres-rustls = "0.10.0"
r2d2 = "0.8" r2d2 = "0.8"
bigdecimal = "0.4" bigdecimal = "0.4"
redis = { version = "0.23", features = ["r2d2", "cluster", "tls-rustls"] } redis = { version = "0.23", features = ["r2d2", "cluster", "tls-rustls"] }
@ -84,6 +86,7 @@ warp = { version = "0.3", features = ["tls"], default-features = false }
hyper = "0.14" hyper = "0.14"
tower = "0.4" tower = "0.4"
futures = "0.3" futures = "0.3"
futures-util = "0.3"
http = "0.2" http = "0.2"
http-body = "0.4" http-body = "0.4"
rust-embed = "8.0" rust-embed = "8.0"
@ -98,6 +101,9 @@ anyhow = "1.0"
pbkdf2 = { version = "0.12", features = ["simple"] } pbkdf2 = { version = "0.12", features = ["simple"] }
rand_core = { version = "0.6", features = ["std"] } rand_core = { version = "0.6", features = ["std"] }
jsonwebtoken = "8.3" jsonwebtoken = "8.3"
rustls = "0.21"
rustls-native-certs = "0.6"
rustls-pemfile = "1.0"
openssl = { version = "0.10" } openssl = { version = "0.10" }
openidconnect = { version = "3.3", features = ["accept-rfc3339-timestamps"] } openidconnect = { version = "3.3", features = ["accept-rfc3339-timestamps"] }

2965
chirpstack/src/api/auth/validator.rs
View File

File diff suppressed because it is too large Load Diff

16
chirpstack/src/api/monitoring/mod.rs
View File

@ -2,14 +2,14 @@ use std::convert::Infallible;
use std::net::SocketAddr; use std::net::SocketAddr;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use diesel::RunQueryDsl; use diesel_async::RunQueryDsl;
use tokio::task; use tokio::task;
use tracing::info; use tracing::info;
use warp::{http::Response, http::StatusCode, Filter}; use warp::{http::Response, http::StatusCode, Filter};
use crate::config; use crate::config;
use crate::monitoring::prometheus; use crate::monitoring::prometheus;
use crate::storage::{get_db_conn, get_redis_conn}; use crate::storage::{get_async_db_conn, get_redis_conn};
pub async fn setup() { pub async fn setup() {
let conf = config::get(); let conf = config::get();
@ -50,17 +50,17 @@ async fn health_handler() -> Result<impl warp::Reply, Infallible> {
} }
async fn _health_handler() -> Result<()> { async fn _health_handler() -> Result<()> {
let mut c = get_async_db_conn().await?;
diesel::sql_query("select 1")
.execute(&mut c)
.await
.context("PostgreSQL connection error")?;
task::spawn_blocking(move || -> Result<()> { task::spawn_blocking(move || -> Result<()> {
let mut r = get_redis_conn()?; let mut r = get_redis_conn()?;
if !r.check_connection() { if !r.check_connection() {
return Err(anyhow!("Redis connection error")); return Err(anyhow!("Redis connection error"));
} }
let mut c = get_db_conn()?;
diesel::sql_query("select 1")
.execute(&mut c)
.context("PostgreSQL connection error")?;
Ok(()) Ok(())
}) })
.await? .await?

18
chirpstack/src/cmd/configfile.rs
View File

@ -41,11 +41,12 @@ pub fn run() {
# PostgreSQL connection pool. # PostgreSQL connection pool.
max_open_connections={{ postgresql.max_open_connections }} max_open_connections={{ postgresql.max_open_connections }}
# Min idle connections. # CA certificate (optional).
# #
# This sets the min. number of idle connections in the PostgreSQL connection # Set this to the path of the CA certificate in case you are using TLS and
# pool (0 = equal to max_open_connections). # the server-certificate is not signed by a CA in the platform certificate
min_idle_connections={{ postgresql.min_idle_connections }} # store.
ca_cert="{{ postgresql.ca_cert }}"
# Redis configuration. # Redis configuration.
@ -459,11 +460,12 @@ pub fn run() {
# PostgreSQL connection pool. # PostgreSQL connection pool.
max_open_connections={{ integration.postgresql.max_open_connections }} max_open_connections={{ integration.postgresql.max_open_connections }}
# Min idle connections. # CA certificate (optional).
# #
# This sets the min. number of idle connections in the PostgreSQL connection # Set this to the path of the CA certificate in case you are using TLS and
# pool (0 = equal to max_open_connections). # the server-certificate is not signed by a CA in the platform certificate
min_idle_connections={{ integration.postgresql.min_idle_connections }} # store.
ca_cert="{{ integration.postgresql.ca_cert }}"
# AMQP / RabbitMQ integration configuration. # AMQP / RabbitMQ integration configuration.

8
chirpstack/src/config.rs
View File

@ -54,7 +54,7 @@ impl Default for Logging {
pub struct Postgresql { pub struct Postgresql {
pub dsn: String, pub dsn: String,
pub max_open_connections: u32, pub max_open_connections: u32,
pub min_idle_connections: u32, pub ca_cert: String,
} }
impl Default for Postgresql { impl Default for Postgresql {
@ -62,7 +62,7 @@ impl Default for Postgresql {
Postgresql { Postgresql {
dsn: "postgresql://chirpstack:chirpstack@localhost/chirpstack?sslmode=disable".into(), dsn: "postgresql://chirpstack:chirpstack@localhost/chirpstack?sslmode=disable".into(),
max_open_connections: 10, max_open_connections: 10,
min_idle_connections: 0, ca_cert: "".into(),
} }
} }
} }
@ -307,7 +307,7 @@ impl Default for MqttIntegrationClient {
pub struct PostgresqlIntegration { pub struct PostgresqlIntegration {
pub dsn: String, pub dsn: String,
pub max_open_connections: u32, pub max_open_connections: u32,
pub min_idle_connections: u32, pub ca_cert: String,
} }
impl Default for PostgresqlIntegration { impl Default for PostgresqlIntegration {
@ -315,7 +315,7 @@ impl Default for PostgresqlIntegration {
PostgresqlIntegration { PostgresqlIntegration {
dsn: "postgresql://chirpstack_integration:chirpstack_integration@localhost/chirpstack_integration?sslmode=disable".into(), dsn: "postgresql://chirpstack_integration:chirpstack_integration@localhost/chirpstack_integration?sslmode=disable".into(),
max_open_connections: 10, max_open_connections: 10,
min_idle_connections: 0, ca_cert: "".into(),
} }
} }
} }

1
chirpstack/src/integration/mod.rs
View File

@ -56,6 +56,7 @@ pub async fn setup() -> Result<()> {
} }
"postgresql" => integrations.push(Box::new( "postgresql" => integrations.push(Box::new(
postgresql::Integration::new(&conf.integration.postgresql) postgresql::Integration::new(&conf.integration.postgresql)
.await
.context("Setup PostgreSQL integration")?, .context("Setup PostgreSQL integration")?,
)), )),
"amqp" => integrations.push(Box::new( "amqp" => integrations.push(Box::new(

214
chirpstack/src/integration/postgresql/mod.rs
View File

@ -1,19 +1,24 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::File;
use std::io::BufReader;
use std::str::FromStr; use std::str::FromStr;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use async_trait::async_trait; use async_trait::async_trait;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::pg::PgConnection; use diesel::{ConnectionError, ConnectionResult};
use diesel::prelude::*; use diesel_async::async_connection_wrapper::AsyncConnectionWrapper;
use diesel::r2d2::{ConnectionManager, Pool}; use diesel_async::pooled_connection::deadpool::{Object as DeadpoolObject, Pool as DeadpoolPool};
use diesel_async::pooled_connection::{AsyncDieselConnectionManager, ManagerConfig};
use diesel_async::{AsyncPgConnection, RunQueryDsl};
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
use tokio::task; use futures_util::future::BoxFuture;
use tracing::info; use futures_util::FutureExt;
use tracing::{error, info};
use uuid::Uuid; use uuid::Uuid;
use super::Integration as IntegrationTrait; use super::Integration as IntegrationTrait;
use crate::config::PostgresqlIntegration as Config; use crate::config::{self, PostgresqlIntegration as Config};
use chirpstack_api::integration; use chirpstack_api::integration;
use schema::{ use schema::{
event_ack, event_integration, event_join, event_location, event_log, event_status, event_ack, event_integration, event_join, event_location, event_log, event_status,
@ -25,7 +30,8 @@ mod schema;
pub const MIGRATIONS: EmbeddedMigrations = pub const MIGRATIONS: EmbeddedMigrations =
embed_migrations!("./src/integration/postgresql/migrations"); embed_migrations!("./src/integration/postgresql/migrations");
type PgPool = Pool<ConnectionManager<PgConnection>>; pub type AsyncPgPool = DeadpoolPool<AsyncPgConnection>;
pub type AsyncPgPoolConnection = DeadpoolObject<AsyncPgConnection>;
#[derive(Insertable)] #[derive(Insertable)]
#[diesel(table_name = event_up)] #[diesel(table_name = event_up)]
@ -189,32 +195,87 @@ struct EventIntegration {
} }
pub struct Integration { pub struct Integration {
pg_pool: PgPool, pg_pool: AsyncPgPool,
} }
impl Integration { impl Integration {
pub fn new(conf: &Config) -> Result<Integration> { pub async fn new(conf: &Config) -> Result<Integration> {
info!("Initializing PostgreSQL integration"); info!("Initializing PostgreSQL integration");
let pg_pool = PgPool::builder() let mut config = ManagerConfig::default();
.max_size(conf.max_open_connections) config.custom_setup = Box::new(pg_establish_connection);
.min_idle(match conf.min_idle_connections {
0 => None, let mgr =
_ => Some(conf.min_idle_connections), AsyncDieselConnectionManager::<AsyncPgConnection>::new_with_config(&conf.dsn, config);
}) let pg_pool = DeadpoolPool::builder(mgr)
.build(ConnectionManager::new(&conf.dsn)) .max_size(conf.max_open_connections as usize)
.context("Setup PostgreSQL connection pool error")?; .build()?;
let mut db_conn = pg_pool.get()?;
let c = pg_pool.get().await?;
let mut c_wrapped: AsyncConnectionWrapper<AsyncPgPoolConnection> =
AsyncConnectionWrapper::from(c);
info!("Applying schema migrations"); info!("Applying schema migrations");
db_conn tokio::task::spawn_blocking(move || -> Result<()> {
.run_pending_migrations(MIGRATIONS) c_wrapped
.map_err(|e| anyhow!("{}", e))?; .run_pending_migrations(MIGRATIONS)
.map_err(|e| anyhow!("{}", e))?;
Ok(())
})
.await??;
Ok(Integration { pg_pool }) Ok(Integration { pg_pool })
} }
} }
// Source:
// https://github.com/weiznich/diesel_async/blob/main/examples/postgres/pooled-with-rustls/src/main.rs
fn pg_establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConnection>> {
let fut = async {
let root_certs =
pg_root_certs().map_err(|e| ConnectionError::BadConnection(e.to_string()))?;
let rustls_config = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_certs)
.with_no_client_auth();
let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config);
let (client, conn) = tokio_postgres::connect(config, tls)
.await
.map_err(|e| ConnectionError::BadConnection(e.to_string()))?;
tokio::spawn(async move {
if let Err(e) = conn.await {
error!(error = %e, "PostgreSQL connection error");
}
});
AsyncPgConnection::try_from(client).await
};
fut.boxed()
}
fn pg_root_certs() -> Result<rustls::RootCertStore> {
let conf = config::get();
let mut roots = rustls::RootCertStore::empty();
let certs = rustls_native_certs::load_native_certs()?;
let certs: Vec<_> = certs.into_iter().map(|cert| cert.0).collect();
roots.add_parsable_certificates(&certs);
if !conf.postgresql.ca_cert.is_empty() {
let f = File::open(&conf.integration.postgresql.ca_cert).context("Open ca certificate")?;
let mut reader = BufReader::new(f);
let certs = rustls_pemfile::certs(&mut reader)?;
for cert in certs
.into_iter()
.map(rustls::Certificate)
.collect::<Vec<_>>()
{
roots.add(&cert)?;
}
}
Ok(roots)
}
#[async_trait] #[async_trait]
impl IntegrationTrait for Integration { impl IntegrationTrait for Integration {
async fn uplink_event( async fn uplink_event(
@ -254,16 +315,12 @@ impl IntegrationTrait for Integration {
rx_info: serde_json::to_value(&pl.rx_info)?, rx_info: serde_json::to_value(&pl.rx_info)?,
tx_info: serde_json::to_value(&pl.tx_info)?, tx_info: serde_json::to_value(&pl.tx_info)?,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_up::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_up::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -295,16 +352,12 @@ impl IntegrationTrait for Integration {
tags: serde_json::to_value(&di.tags)?, tags: serde_json::to_value(&di.tags)?,
dev_addr: pl.dev_addr.clone(), dev_addr: pl.dev_addr.clone(),
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_join::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_join::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -338,16 +391,12 @@ impl IntegrationTrait for Integration {
acknowledged: pl.acknowledged, acknowledged: pl.acknowledged,
f_cnt_down: pl.f_cnt_down as i64, f_cnt_down: pl.f_cnt_down as i64,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_ack::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_ack::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -382,16 +431,12 @@ impl IntegrationTrait for Integration {
gateway_id: pl.gateway_id.clone(), gateway_id: pl.gateway_id.clone(),
tx_info: serde_json::to_value(&pl.tx_info)?, tx_info: serde_json::to_value(&pl.tx_info)?,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_tx_ack::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_tx_ack::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -425,16 +470,12 @@ impl IntegrationTrait for Integration {
description: pl.description.clone(), description: pl.description.clone(),
context: serde_json::to_value(&pl.context)?, context: serde_json::to_value(&pl.context)?,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_log::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_log::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -469,15 +510,12 @@ impl IntegrationTrait for Integration {
battery_level_unavailable: pl.battery_level_unavailable, battery_level_unavailable: pl.battery_level_unavailable,
battery_level: pl.battery_level, battery_level: pl.battery_level,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> { diesel::insert_into(event_status::table)
diesel::insert_into(event_status::table) .values(&e)
.values(&e) .execute(&mut c)
.execute(&mut c)?; .await?;
Ok(())
})
.await??;
Ok(()) Ok(())
} }
@ -514,16 +552,12 @@ impl IntegrationTrait for Integration {
source: loc.source.to_string(), source: loc.source.to_string(),
accuracy: loc.accuracy, accuracy: loc.accuracy,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_location::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_location::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
@ -557,16 +591,12 @@ impl IntegrationTrait for Integration {
event_type: pl.event_type.clone(), event_type: pl.event_type.clone(),
object: serde_json::to_value(&pl.object)?, object: serde_json::to_value(&pl.object)?,
}; };
let mut c = self.pg_pool.get()?; let mut c = self.pg_pool.get().await?;
task::spawn_blocking(move || -> Result<()> {
diesel::insert_into(event_integration::table)
.values(&e)
.execute(&mut c)?;
Ok(())
})
.await??;
diesel::insert_into(event_integration::table)
.values(&e)
.execute(&mut c)
.await?;
Ok(()) Ok(())
} }
} }

118
chirpstack/src/storage/api_key.rs
View File

@ -2,13 +2,13 @@ use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::dsl;
use diesel::prelude::*; use diesel::prelude::*;
use tokio::task; use diesel_async::RunQueryDsl;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use super::error::Error; use super::error::Error;
use super::schema::api_key; use super::schema::api_key;
use super::{error, get_db_conn}; use super::{error, get_async_db_conn};
#[derive(Queryable, Insertable, PartialEq, Eq, Debug)] #[derive(Queryable, Insertable, PartialEq, Eq, Debug)]
#[diesel(table_name = api_key)] #[diesel(table_name = api_key)]
@ -51,81 +51,61 @@ pub struct Filters {
pub async fn create(ak: ApiKey) -> Result<ApiKey, Error> { pub async fn create(ak: ApiKey) -> Result<ApiKey, Error> {
ak.validate()?; ak.validate()?;
let ak = task::spawn_blocking(move || -> Result<ApiKey, Error> { let mut c = get_async_db_conn().await?;
let mut c = get_db_conn()?; let ak: ApiKey = diesel::insert_into(api_key::table)
diesel::insert_into(api_key::table) .values(&ak)
.values(&ak) .get_result(&mut c)
.get_result(&mut c) .await
.map_err(|e| error::Error::from_diesel(e, ak.id.to_string())) .map_err(|e| error::Error::from_diesel(e, ak.id.to_string()))?;
})
.await??;
info!(id = %ak.id, "Api-key created"); info!(id = %ak.id, "Api-key created");
Ok(ak) Ok(ak)
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(api_key::dsl::api_key.find(&id))
.execute(&mut c)
move || -> Result<(), Error> { .await?;
let mut c = get_db_conn()?; if ra == 0 {
let ra = diesel::delete(api_key::dsl::api_key.find(&id)).execute(&mut c)?; return Err(Error::NotFound(id.to_string()));
if ra == 0 { }
return Err(Error::NotFound(id.to_string())); info!(id = %id, "Api-key deleted");
} Ok(())
info!(id = %id, "Api-key deleted");
Ok(())
}
})
.await?
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone();
move || -> Result<i64, Error> { let mut q = api_key::dsl::api_key
let mut c = get_db_conn()?; .select(dsl::count_star())
.filter(api_key::dsl::is_admin.eq(filters.is_admin))
.into_boxed();
let mut q = api_key::dsl::api_key if let Some(tenant_id) = &filters.tenant_id {
.select(dsl::count_star()) q = q.filter(api_key::dsl::tenant_id.eq(tenant_id));
.filter(api_key::dsl::is_admin.eq(filters.is_admin)) }
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { Ok(q.first(&mut c).await?)
q = q.filter(api_key::dsl::tenant_id.eq(tenant_id));
}
Ok(q.first(&mut c)?)
}
})
.await?
} }
pub async fn list(limit: i64, offset: i64, filters: &Filters) -> Result<Vec<ApiKey>, Error> { pub async fn list(limit: i64, offset: i64, filters: &Filters) -> Result<Vec<ApiKey>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone();
move || -> Result<Vec<ApiKey>, Error> { let mut q = api_key::dsl::api_key
let mut c = get_db_conn()?; .filter(api_key::dsl::is_admin.eq(filters.is_admin))
.into_boxed();
let mut q = api_key::dsl::api_key if let Some(tenant_id) = &filters.tenant_id {
.filter(api_key::dsl::is_admin.eq(filters.is_admin)) q = q.filter(api_key::dsl::tenant_id.eq(tenant_id));
.into_boxed(); }
if let Some(tenant_id) = &filters.tenant_id { let items = q
q = q.filter(api_key::dsl::tenant_id.eq(tenant_id)); .order_by(api_key::dsl::name)
} .limit(limit)
.offset(offset)
let items = q .load(&mut c)
.order_by(api_key::dsl::name) .await?;
.limit(limit) Ok(items)
.offset(offset)
.load(&mut c)?;
Ok(items)
}
})
.await?
} }
#[cfg(test)] #[cfg(test)]
@ -143,18 +123,12 @@ pub mod test {
} }
pub async fn get(id: &Uuid) -> Result<ApiKey, Error> { pub async fn get(id: &Uuid) -> Result<ApiKey, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; api_key::dsl::api_key
.find(&id)
move || -> Result<ApiKey, Error> { .first(&mut c)
let mut c = get_db_conn()?; .await
api_key::dsl::api_key .map_err(|e| error::Error::from_diesel(e, id.to_string()))
.find(&id)
.first(&mut c)
.map_err(|e| error::Error::from_diesel(e, id.to_string()))
}
})
.await?
} }
pub async fn create_api_key(is_admin: bool, is_tenant: bool) -> ApiKey { pub async fn create_api_key(is_admin: bool, is_tenant: bool) -> ApiKey {

404
chirpstack/src/storage/application.rs
View File

@ -4,20 +4,22 @@ use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::backend::Backend; use diesel::{
use diesel::dsl; backend::Backend,
use diesel::pg::Pg; deserialize, dsl,
use diesel::prelude::*; pg::Pg,
use diesel::sql_types::{Jsonb, Text}; prelude::*,
use diesel::{deserialize, serialize}; serialize,
sql_types::{Jsonb, Text},
};
use diesel_async::RunQueryDsl;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use super::error::Error; use super::error::Error;
use super::schema::{application, application_integration}; use super::schema::{application, application_integration};
use super::{fields, get_db_conn}; use super::{fields, get_async_db_conn};
#[derive(Clone, Queryable, Insertable, PartialEq, Eq, Debug)] #[derive(Clone, Queryable, Insertable, PartialEq, Eq, Debug)]
#[diesel(table_name = application)] #[diesel(table_name = application)]
@ -289,77 +291,58 @@ impl Default for Integration {
pub async fn create(a: Application) -> Result<Application, Error> { pub async fn create(a: Application) -> Result<Application, Error> {
a.validate()?; a.validate()?;
task::spawn_blocking({
move || -> Result<Application, Error> {
let mut c = get_db_conn()?;
let a: Application = diesel::insert_into(application::table)
.values(&a)
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, a.id.to_string()))?;
info!(id = %a.id, "Application created"); let mut c = get_async_db_conn().await?;
let a: Application = diesel::insert_into(application::table)
.values(&a)
.get_result(&mut c)
.await
.map_err(|e| Error::from_diesel(e, a.id.to_string()))?;
Ok(a) info!(id = %a.id, "Application created");
}
}) Ok(a)
.await?
} }
pub async fn get(id: &Uuid) -> Result<Application, Error> { pub async fn get(id: &Uuid) -> Result<Application, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let a = application::dsl::application
move || -> Result<Application, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
let a = application::dsl::application .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.first(&mut c) Ok(a)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(a)
}
})
.await?
} }
pub async fn update(a: Application) -> Result<Application, Error> { pub async fn update(a: Application) -> Result<Application, Error> {
a.validate()?; a.validate()?;
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Application, Error> { let a: Application = diesel::update(application::dsl::application.find(&a.id))
let mut c = get_db_conn()?; .set((
let a: Application = diesel::update(application::dsl::application.find(&a.id)) application::updated_at.eq(Utc::now()),
.set(( application::name.eq(&a.name),
application::updated_at.eq(Utc::now()), application::description.eq(&a.description),
application::name.eq(&a.name), application::tags.eq(&a.tags),
application::description.eq(&a.description), ))
application::tags.eq(&a.tags), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, a.id.to_string()))?;
.map_err(|e| Error::from_diesel(e, a.id.to_string()))?;
info!( info!(
application_id = %a.id, application_id = %a.id,
"Application updated" "Application updated"
); );
Ok(a) Ok(a)
}
})
.await?
} }
pub async fn update_mqtt_cls_cert(id: &Uuid, cert: &[u8]) -> Result<Application, Error> { pub async fn update_mqtt_cls_cert(id: &Uuid, cert: &[u8]) -> Result<Application, Error> {
let app = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let app: Application = diesel::update(application::dsl::application.find(&id))
let cert = cert.to_vec(); .set(application::mqtt_tls_cert.eq(cert))
move || -> Result<Application, Error> { .get_result(&mut c)
let mut c = get_db_conn()?; .await
let app: Application = diesel::update(application::dsl::application.find(&id)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.set(application::mqtt_tls_cert.eq(cert))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(app)
}
})
.await??;
info!( info!(
application_id = %id, application_id = %id,
@ -370,47 +353,37 @@ pub async fn update_mqtt_cls_cert(id: &Uuid, cert: &[u8]) -> Result<Application,
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(application::dsl::application.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = diesel::delete(application::dsl::application.find(&id)).execute(&mut c)?; if ra == 0 {
if ra == 0 { return Err(Error::NotFound(id.to_string()));
return Err(Error::NotFound(id.to_string())); }
}
info!( info!(
application_id = %id, application_id = %id,
"Application deleted" "Application deleted"
); );
Ok(()) Ok(())
}
})
.await?
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = application::dsl::application
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .into_boxed();
let mut q = application::dsl::application
.select(dsl::count_star())
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(application::dsl::tenant_id.eq(tenant_id)); q = q.filter(application::dsl::tenant_id.eq(tenant_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(application::dsl::name.ilike(format!("%{}%", search))); q = q.filter(application::dsl::name.ilike(format!("%{}%", search)));
} }
Ok(q.first(&mut c)?) Ok(q.first(&mut c).await?)
}
})
.await?
} }
pub async fn list( pub async fn list(
@ -418,156 +391,128 @@ pub async fn list(
offset: i64, offset: i64,
filters: &Filters, filters: &Filters,
) -> Result<Vec<ApplicationListItem>, Error> { ) -> Result<Vec<ApplicationListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = application::dsl::application
move || -> Result<Vec<ApplicationListItem>, Error> { .select((
let mut c = get_db_conn()?; application::id,
let mut q = application::dsl::application application::created_at,
.select(( application::updated_at,
application::id, application::name,
application::created_at, application::description,
application::updated_at, ))
application::name, .into_boxed();
application::description,
))
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(application::dsl::tenant_id.eq(tenant_id)); q = q.filter(application::dsl::tenant_id.eq(tenant_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(application::dsl::name.ilike(format!("%{}%", search))); q = q.filter(application::dsl::name.ilike(format!("%{}%", search)));
} }
let items = q let items = q
.order_by(application::dsl::name) .order_by(application::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c)?; .load(&mut c)
Ok(items) .await?;
} Ok(items)
})
.await?
} }
pub async fn create_integration(i: Integration) -> Result<Integration, Error> { pub async fn create_integration(i: Integration) -> Result<Integration, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Integration, Error> { let i: Integration = diesel::insert_into(application_integration::table)
let mut c = get_db_conn()?; .values(&i)
let i: Integration = diesel::insert_into(application_integration::table) .get_result(&mut c)
.values(&i) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, i.kind.to_string()))?;
.map_err(|e| Error::from_diesel(e, i.kind.to_string()))?;
info!(application_id = %i.application_id, kind = %i.kind, "Integration created"); info!(application_id = %i.application_id, kind = %i.kind, "Integration created");
Ok(i) Ok(i)
}
})
.await?
} }
pub async fn get_integration( pub async fn get_integration(
application_id: &Uuid, application_id: &Uuid,
kind: IntegrationKind, kind: IntegrationKind,
) -> Result<Integration, Error> { ) -> Result<Integration, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let application_id = *application_id; let mut i: Integration = application_integration::dsl::application_integration
move || -> Result<Integration, Error> { .filter(
let mut c = get_db_conn()?; application_integration::dsl::application_id
let mut i: Integration = application_integration::dsl::application_integration .eq(application_id)
.filter( .and(application_integration::dsl::kind.eq(kind)),
application_integration::dsl::application_id )
.eq(application_id) .first(&mut c)
.and(application_integration::dsl::kind.eq(kind)), .await
) .map_err(|e| Error::from_diesel(e, application_id.to_string()))?;
.first(&mut c)
.map_err(|e| Error::from_diesel(e, application_id.to_string()))?;
// For backwards compatibiliy // For backwards compatibiliy
if let IntegrationConfiguration::LoraCloud(conf) = &mut i.configuration { if let IntegrationConfiguration::LoraCloud(conf) = &mut i.configuration {
if conf.modem_geolocation_services.forward_f_ports.is_empty() { if conf.modem_geolocation_services.forward_f_ports.is_empty() {
conf.modem_geolocation_services.forward_f_ports = vec![ conf.modem_geolocation_services.forward_f_ports = vec![
conf.modem_geolocation_services.modem_port, conf.modem_geolocation_services.modem_port,
conf.modem_geolocation_services.gnss_port, conf.modem_geolocation_services.gnss_port,
197, 197,
192, 192,
]; ];
}
}
Ok(i)
} }
}) }
.await?
Ok(i)
} }
pub async fn update_integration(i: Integration) -> Result<Integration, Error> { pub async fn update_integration(i: Integration) -> Result<Integration, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Integration, Error> { let i: Integration = diesel::update(
let mut c = get_db_conn()?; application_integration::dsl::application_integration.filter(
let i: Integration = diesel::update( application_integration::dsl::application_id
application_integration::dsl::application_integration.filter( .eq(&i.application_id)
application_integration::dsl::application_id .and(application_integration::dsl::kind.eq(&i.kind)),
.eq(&i.application_id) ),
.and(application_integration::dsl::kind.eq(&i.kind)), )
), .set((
) application_integration::updated_at.eq(Utc::now()),
.set(( application_integration::configuration.eq(&i.configuration),
application_integration::updated_at.eq(Utc::now()), ))
application_integration::configuration.eq(&i.configuration), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, i.application_id.to_string()))?;
.map_err(|e| Error::from_diesel(e, i.application_id.to_string()))?;
info!(application_id = %i.application_id, kind = %i.kind, "Integration updated"); info!(application_id = %i.application_id, kind = %i.kind, "Integration updated");
Ok(i) Ok(i)
}
})
.await?
} }
pub async fn delete_integration(application_id: &Uuid, kind: IntegrationKind) -> Result<(), Error> { pub async fn delete_integration(application_id: &Uuid, kind: IntegrationKind) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let application_id = *application_id; let ra = diesel::delete(
move || -> Result<(), Error> { application_integration::dsl::application_integration.filter(
let mut c = get_db_conn()?; application_integration::dsl::application_id
let ra = diesel::delete( .eq(&application_id)
application_integration::dsl::application_integration.filter( .and(application_integration::dsl::kind.eq(&kind)),
application_integration::dsl::application_id ),
.eq(&application_id) )
.and(application_integration::dsl::kind.eq(&kind)), .execute(&mut c)
), .await?;
)
.execute(&mut c)?;
if ra == 0 { if ra == 0 {
return Err(Error::NotFound(application_id.to_string())); return Err(Error::NotFound(application_id.to_string()));
} }
info!(application_id = %application_id, kind = %kind, "Integration deleted"); info!(application_id = %application_id, kind = %kind, "Integration deleted");
Ok(()) Ok(())
}
})
.await?
} }
pub async fn get_integrations_for_application( pub async fn get_integrations_for_application(
application_id: &Uuid, application_id: &Uuid,
) -> Result<Vec<Integration>, Error> { ) -> Result<Vec<Integration>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let application_id = *application_id; let items: Vec<Integration> = application_integration::dsl::application_integration
move || -> Result<Vec<Integration>, Error> { .filter(application_integration::dsl::application_id.eq(&application_id))
let mut c = get_db_conn()?; .order_by(application_integration::dsl::kind)
let items: Vec<Integration> = application_integration::dsl::application_integration .load(&mut c)
.filter(application_integration::dsl::application_id.eq(&application_id)) .await?;
.order_by(application_integration::dsl::kind) Ok(items)
.load(&mut c)?;
Ok(items)
}
})
.await?
} }
pub async fn get_measurement_keys(application_id: &Uuid) -> Result<Vec<String>, Error> { pub async fn get_measurement_keys(application_id: &Uuid) -> Result<Vec<String>, Error> {
@ -577,12 +522,9 @@ pub async fn get_measurement_keys(application_id: &Uuid) -> Result<Vec<String>,
pub key: String, pub key: String,
} }
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let application_id = *application_id; let keys: Vec<Measurement> = diesel::sql_query(
move || -> Result<Vec<String>, Error> { r#"
let mut c = get_db_conn()?;
let keys: Vec<Measurement> = diesel::sql_query(
r#"
select select
distinct jsonb_object_keys(dp.measurements) as key distinct jsonb_object_keys(dp.measurements) as key
from from
@ -594,14 +536,12 @@ pub async fn get_measurement_keys(application_id: &Uuid) -> Result<Vec<String>,
order by order by
key key
"#, "#,
) )
.bind::<diesel::sql_types::Uuid, _>(application_id) .bind::<diesel::sql_types::Uuid, _>(application_id)
.load(&mut c) .load(&mut c)
.map_err(|e| Error::from_diesel(e, application_id.to_string()))?; .await
Ok(keys.iter().map(|k| k.key.clone()).collect()) .map_err(|e| Error::from_diesel(e, application_id.to_string()))?;
} Ok(keys.iter().map(|k| k.key.clone()).collect())
})
.await?
} }
#[cfg(test)] #[cfg(test)]

484
chirpstack/src/storage/device.rs
View File

@ -6,14 +6,14 @@ use anyhow::{Context, Result};
use bigdecimal::BigDecimal; use bigdecimal::BigDecimal;
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use diesel::{backend::Backend, deserialize, dsl, prelude::*, serialize, sql_types::Text}; use diesel::{backend::Backend, deserialize, dsl, prelude::*, serialize, sql_types::Text};
use tokio::task; use diesel_async::RunQueryDsl;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use lrwn::{DevAddr, EUI64}; use lrwn::{DevAddr, EUI64};
use super::schema::{application, device, device_profile, multicast_group_device, tenant}; use super::schema::{application, device, device_profile, multicast_group_device, tenant};
use super::{error::Error, fields, get_db_conn}; use super::{error::Error, fields, get_async_db_conn};
use crate::config; use crate::config;
#[derive(Debug, Clone, Copy, Eq, PartialEq, AsExpression, FromSqlRow)] #[derive(Debug, Clone, Copy, Eq, PartialEq, AsExpression, FromSqlRow)]
@ -177,11 +177,11 @@ pub struct DevicesDataRate {
} }
pub async fn create(d: Device) -> Result<Device, Error> { pub async fn create(d: Device) -> Result<Device, Error> {
d.validate()?; let mut c = get_async_db_conn().await?;
let d = task::spawn_blocking({ let d: Device = c
move || -> Result<Device, Error> { .build_transaction()
let mut c = get_db_conn()?; .run::<Device, Error, _>(|c| {
c.transaction::<Device, Error, _>(|c| { Box::pin(async move {
// use for update to lock the tenant // use for update to lock the tenant
let t: super::tenant::Tenant = tenant::dsl::tenant let t: super::tenant::Tenant = tenant::dsl::tenant
.select(( .select((
@ -200,13 +200,15 @@ pub async fn create(d: Device) -> Result<Device, Error> {
.inner_join(application::table) .inner_join(application::table)
.filter(application::dsl::id.eq(&d.application_id)) .filter(application::dsl::id.eq(&d.application_id))
.for_update() .for_update()
.first(c)?; .first(c)
.await?;
let dev_count: i64 = device::dsl::device let dev_count: i64 = device::dsl::device
.select(dsl::count_star()) .select(dsl::count_star())
.inner_join(application::table) .inner_join(application::table)
.filter(application::dsl::tenant_id.eq(&t.id)) .filter(application::dsl::tenant_id.eq(&t.id))
.first(c)?; .first(c)
.await?;
if t.max_device_count != 0 && dev_count as i32 >= t.max_device_count { if t.max_device_count != 0 && dev_count as i32 >= t.max_device_count {
return Err(Error::NotAllowed( return Err(Error::NotAllowed(
@ -217,100 +219,77 @@ pub async fn create(d: Device) -> Result<Device, Error> {
diesel::insert_into(device::table) diesel::insert_into(device::table)
.values(&d) .values(&d)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, d.dev_eui.to_string())) .map_err(|e| Error::from_diesel(e, d.dev_eui.to_string()))
}) })
} })
}) .await?;
.await??;
info!(dev_eui = %d.dev_eui, "Device created"); info!(dev_eui = %d.dev_eui, "Device created");
Ok(d) Ok(d)
} }
pub async fn get(dev_eui: &EUI64) -> Result<Device, Error> { pub async fn get(dev_eui: &EUI64) -> Result<Device, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let d = device::dsl::device
move || -> Result<Device, Error> { .find(&dev_eui)
let mut c = get_db_conn()?; .first(&mut c)
let d = device::dsl::device .await
.find(&dev_eui) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.first(&mut c) Ok(d)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
Ok(d)
}
})
.await?
} }
pub async fn update(d: Device) -> Result<Device, Error> { pub async fn update(d: Device) -> Result<Device, Error> {
d.validate()?; d.validate()?;
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Device, Error> { let d: Device = diesel::update(device::dsl::device.find(&d.dev_eui))
let mut c = get_db_conn()?; .set((
diesel::update(device::dsl::device.find(&d.dev_eui)) device::updated_at.eq(Utc::now()),
.set(( device::application_id.eq(&d.application_id),
device::updated_at.eq(Utc::now()), device::device_profile_id.eq(&d.device_profile_id),
device::application_id.eq(&d.application_id), device::name.eq(&d.name),
device::device_profile_id.eq(&d.device_profile_id), device::description.eq(&d.description),
device::name.eq(&d.name), device::skip_fcnt_check.eq(&d.skip_fcnt_check),
device::description.eq(&d.description), device::is_disabled.eq(&d.is_disabled),
device::skip_fcnt_check.eq(&d.skip_fcnt_check), device::tags.eq(&d.tags),
device::is_disabled.eq(&d.is_disabled), device::variables.eq(&d.variables),
device::tags.eq(&d.tags), device::join_eui.eq(&d.join_eui),
device::variables.eq(&d.variables), ))
device::join_eui.eq(&d.join_eui), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, d.dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, d.dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %d.dev_eui, "Device updated"); info!(dev_eui = %d.dev_eui, "Device updated");
Ok(d) Ok(d)
} }
pub async fn set_enabled_class(dev_eui: &EUI64, mode: DeviceClass) -> Result<Device, Error> { pub async fn set_enabled_class(dev_eui: &EUI64, mode: DeviceClass) -> Result<Device, Error> {
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let d: Device = diesel::update(device::dsl::device.find(&dev_eui))
.set(device::enabled_class.eq(&mode))
move || -> Result<Device, Error> { .get_result(&mut c)
let mut c = get_db_conn()?; .await
diesel::update(device::dsl::device.find(&dev_eui)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.set(device::enabled_class.eq(&mode))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, enabled_class = %mode, "Enabled class updated"); info!(dev_eui = %dev_eui, enabled_class = %mode, "Enabled class updated");
Ok(d) Ok(d)
} }
pub async fn set_join_eui(dev_eui: EUI64, join_eui: EUI64) -> Result<Device, Error> { pub async fn set_join_eui(dev_eui: EUI64, join_eui: EUI64) -> Result<Device, Error> {
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Device, Error> { let d: Device = diesel::update(device::dsl::device.find(&dev_eui))
let mut c = get_db_conn()?; .set(device::join_eui.eq(&join_eui))
diesel::update(device::dsl::device.find(&dev_eui)) .get_result(&mut c)
.set(device::join_eui.eq(&join_eui)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, join_eui = %join_eui, "Updated JoinEUI"); info!(dev_eui = %dev_eui, join_eui = %join_eui, "Updated JoinEUI");
Ok(d) Ok(d)
} }
pub async fn set_dev_addr(dev_eui: EUI64, dev_addr: DevAddr) -> Result<Device, Error> { pub async fn set_dev_addr(dev_eui: EUI64, dev_addr: DevAddr) -> Result<Device, Error> {
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Device, Error> { let d: Device = diesel::update(device::dsl::device.find(&dev_eui))
let mut c = get_db_conn()?; .set(device::dev_addr.eq(&dev_addr))
diesel::update(device::dsl::device.find(&dev_eui)) .get_result(&mut c)
.set(device::dev_addr.eq(&dev_addr)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, dev_addr = %dev_addr, "Updated DevAddr"); info!(dev_eui = %dev_eui, dev_addr = %dev_addr, "Updated DevAddr");
Ok(d) Ok(d)
} }
@ -323,34 +302,24 @@ pub async fn set_scheduler_run_after(
dev_eui: &EUI64, dev_eui: &EUI64,
new_ts: Option<DateTime<Utc>>, new_ts: Option<DateTime<Utc>>,
) -> Result<Device, Error> { ) -> Result<Device, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; diesel::update(device::dsl::device.find(&dev_eui))
move || -> Result<Device, Error> { .set(device::scheduler_run_after.eq(&new_ts))
let mut c = get_db_conn()?; .get_result(&mut c)
diesel::update(device::dsl::device.find(&dev_eui)) .await
.set(device::scheduler_run_after.eq(&new_ts)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await?
} }
pub async fn set_last_seen_dr(dev_eui: &EUI64, dr: u8) -> Result<Device, Error> { pub async fn set_last_seen_dr(dev_eui: &EUI64, dr: u8) -> Result<Device, Error> {
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let d: Device = diesel::update(device::dsl::device.find(&dev_eui))
move || -> Result<Device, Error> { .set((
let mut c = get_db_conn()?; device::last_seen_at.eq(Utc::now()),
diesel::update(device::dsl::device.find(&dev_eui)) device::dr.eq(dr as i16),
.set(( ))
device::last_seen_at.eq(Utc::now()), .get_result(&mut c)
device::dr.eq(dr as i16), .await
)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, dr = dr, "Data-rate updated"); info!(dev_eui = %dev_eui, dr = dr, "Data-rate updated");
Ok(d) Ok(d)
} }
@ -361,70 +330,53 @@ pub async fn set_status(
external_power_source: bool, external_power_source: bool,
battery_level: Option<BigDecimal>, battery_level: Option<BigDecimal>,
) -> Result<Device, Error> { ) -> Result<Device, Error> {
let d = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let d: Device = diesel::update(device::dsl::device.find(&dev_eui))
move || -> Result<Device, Error> { .set((
let mut c = get_db_conn()?; device::margin.eq(Some(margin)),
diesel::update(device::dsl::device.find(&dev_eui)) device::external_power_source.eq(external_power_source),
.set(( device::battery_level.eq(battery_level),
device::margin.eq(Some(margin)), ))
device::external_power_source.eq(external_power_source), .get_result(&mut c)
device::battery_level.eq(battery_level), .await
)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, "Device status updated"); info!(dev_eui = %dev_eui, "Device status updated");
Ok(d) Ok(d)
} }
pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> { pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let ra = diesel::delete(device::dsl::device.find(&dev_eui))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = diesel::delete(device::dsl::device.find(&dev_eui)).execute(&mut c)?; if ra == 0 {
if ra == 0 { return Err(Error::NotFound(dev_eui.to_string()));
return Err(Error::NotFound(dev_eui.to_string())); }
}
Ok(())
}
})
.await??;
info!(dev_eui = %dev_eui, "Device deleted"); info!(dev_eui = %dev_eui, "Device deleted");
Ok(()) Ok(())
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device::dsl::device
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .distinct()
let mut q = device::dsl::device .left_join(multicast_group_device::table)
.select(dsl::count_star()) .into_boxed();
.distinct()
.left_join(multicast_group_device::table)
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(device::dsl::application_id.eq(application_id)); q = q.filter(device::dsl::application_id.eq(application_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(device::dsl::name.ilike(format!("%{}%", search))); q = q.filter(device::dsl::name.ilike(format!("%{}%", search)));
} }
if let Some(multicast_group_id) = &filters.multicast_group_id { if let Some(multicast_group_id) = &filters.multicast_group_id {
q = q q = q.filter(multicast_group_device::dsl::multicast_group_id.eq(multicast_group_id));
.filter(multicast_group_device::dsl::multicast_group_id.eq(multicast_group_id)); }
}
Ok(q.first(&mut c)?) Ok(q.first(&mut c).await?)
}
})
.await?
} }
pub async fn list( pub async fn list(
@ -432,133 +384,118 @@ pub async fn list(
offset: i64, offset: i64,
filters: &Filters, filters: &Filters,
) -> Result<Vec<DeviceListItem>, Error> { ) -> Result<Vec<DeviceListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device::dsl::device
move || -> Result<Vec<DeviceListItem>, Error> { .inner_join(device_profile::table)
let mut c = get_db_conn()?; .left_join(multicast_group_device::table)
let mut q = device::dsl::device .select((
.inner_join(device_profile::table) device::dev_eui,
.left_join(multicast_group_device::table) device::name,
.select(( device::description,
device::dev_eui, device_profile::id,
device::name, device_profile::name,
device::description, device::created_at,
device_profile::id, device::updated_at,
device_profile::name, device::last_seen_at,
device::created_at, device::margin,
device::updated_at, device::external_power_source,
device::last_seen_at, device::battery_level,
device::margin, ))
device::external_power_source, .distinct()
device::battery_level, .into_boxed();
))
.distinct()
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(device::dsl::application_id.eq(application_id)); q = q.filter(device::dsl::application_id.eq(application_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(device::dsl::name.ilike(format!("%{}%", search))); q = q.filter(device::dsl::name.ilike(format!("%{}%", search)));
} }
if let Some(multicast_group_id) = &filters.multicast_group_id { if let Some(multicast_group_id) = &filters.multicast_group_id {
q = q q = q.filter(multicast_group_device::dsl::multicast_group_id.eq(multicast_group_id));
.filter(multicast_group_device::dsl::multicast_group_id.eq(multicast_group_id)); }
}
q.order_by(device::dsl::name) q.order_by(device::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c) .load(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn get_active_inactive(tenant_id: &Option<Uuid>) -> Result<DevicesActiveInactive, Error> { pub async fn get_active_inactive(tenant_id: &Option<Uuid>) -> Result<DevicesActiveInactive, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; diesel::sql_query(r#"
move || -> Result<DevicesActiveInactive, Error> { with device_active_inactive as (
let mut c = get_db_conn()?; select
diesel::sql_query(r#" make_interval(secs => dp.uplink_interval) * 1.5 as uplink_interval,
with device_active_inactive as ( d.last_seen_at as last_seen_at
select from
make_interval(secs => dp.uplink_interval) * 1.5 as uplink_interval, device d
d.last_seen_at as last_seen_at inner join device_profile dp
from on d.device_profile_id = dp.id
device d where
inner join device_profile dp $1 is null or dp.tenant_id = $1
on d.device_profile_id = dp.id )
where select
$1 is null or dp.tenant_id = $1 coalesce(sum(case when last_seen_at is null then 1 end), 0) as never_seen_count,
) coalesce(sum(case when (now() - uplink_interval) > last_seen_at then 1 end), 0) as inactive_count,
select coalesce(sum(case when (now() - uplink_interval) <= last_seen_at then 1 end), 0) as active_count
coalesce(sum(case when last_seen_at is null then 1 end), 0) as never_seen_count, from
coalesce(sum(case when (now() - uplink_interval) > last_seen_at then 1 end), 0) as inactive_count, device_active_inactive
coalesce(sum(case when (now() - uplink_interval) <= last_seen_at then 1 end), 0) as active_count "#)
from .bind::<diesel::sql_types::Nullable<diesel::sql_types::Uuid>, _>(tenant_id)
device_active_inactive .get_result(&mut c).await
"#) .map_err(|e| Error::from_diesel(e, "".into()))
.bind::<diesel::sql_types::Nullable<diesel::sql_types::Uuid>, _>(tenant_id)
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, "".into()))
}
})
.await?
} }
pub async fn get_data_rates(tenant_id: &Option<Uuid>) -> Result<Vec<DevicesDataRate>, Error> { pub async fn get_data_rates(tenant_id: &Option<Uuid>) -> Result<Vec<DevicesDataRate>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let mut q = device::dsl::device
move || -> Result<Vec<DevicesDataRate>, Error> { .inner_join(device_profile::table)
let mut c = get_db_conn()?; //.select((device::dr, dsl::count_star()))
let mut q = device::dsl::device .select((
.inner_join(device_profile::table) device::dr,
//.select((device::dr, dsl::count_star())) diesel::dsl::sql::<diesel::sql_types::BigInt>("count(1)"),
.select(( ))
device::dr, .group_by(device::dr)
diesel::dsl::sql::<diesel::sql_types::BigInt>("count(1)"), .filter(device::dsl::dr.is_not_null())
)) .into_boxed();
.group_by(device::dr)
.filter(device::dsl::dr.is_not_null())
.into_boxed();
if let Some(id) = &tenant_id { if let Some(id) = &tenant_id {
q = q.filter(device_profile::dsl::tenant_id.eq(id)); q = q.filter(device_profile::dsl::tenant_id.eq(id));
} }
q.load(&mut c).map_err(|e| Error::from_diesel(e, "".into())) q.load(&mut c)
} .await
}) .map_err(|e| Error::from_diesel(e, "".into()))
.await?
} }
pub async fn get_with_class_b_c_queue_items(limit: usize) -> Result<Vec<Device>> { pub async fn get_with_class_b_c_queue_items(limit: usize) -> Result<Vec<Device>> {
task::spawn_blocking(move || -> Result<Vec<Device>> { let mut c = get_async_db_conn().await?;
let mut c = get_db_conn()?; c.build_transaction()
c.transaction::<Vec<Device>, Error, _>(|c| { .run::<Vec<Device>, Error, _>(|c| {
let conf = config::get(); Box::pin(async {
let conf = config::get();
// This query will: // This query will:
// * Select the devices for which a Class-B or Class-C downlink can be scheduled. // * Select the devices for which a Class-B or Class-C downlink can be scheduled.
// * Lock the device records for update with skip locked such that other // * Lock the device records for update with skip locked such that other
// ChirpStack instances are able to do the same for the remaining devices. // ChirpStack instances are able to do the same for the remaining devices.
// * Update the scheduler_run_after for these devices to now() + 2 * scheduler // * Update the scheduler_run_after for these devices to now() + 2 * scheduler
// interval to avoid concurrency issues (other ChirpStack instance scheduling // interval to avoid concurrency issues (other ChirpStack instance scheduling
// the same queue items). // the same queue items).
// //
// This way, we do not have to keep the device records locked until the scheduler // This way, we do not have to keep the device records locked until the scheduler
// finishes its batch as the same set of devices will not be returned until after // finishes its batch as the same set of devices will not be returned until after
// the updated scheduler_run_after. Only if the scheduler takes more time than 2x the // the updated scheduler_run_after. Only if the scheduler takes more time than 2x the
// interval (the scheduler is still working on processing the batch after 2 x interval) // interval (the scheduler is still working on processing the batch after 2 x interval)
// this might cause issues. // this might cause issues.
// The alternative would be to keep the transaction open for a long time + keep // The alternative would be to keep the transaction open for a long time + keep
// the device records locked during this time which could case issues as well. // the device records locked during this time which could case issues as well.
diesel::sql_query( diesel::sql_query(
r#" r#"
update update
device device
set set
@ -590,18 +527,19 @@ pub async fn get_with_class_b_c_queue_items(limit: usize) -> Result<Vec<Device>>
) )
returning * returning *
"#, "#,
) )
.bind::<diesel::sql_types::Integer, _>(limit as i32) .bind::<diesel::sql_types::Integer, _>(limit as i32)
.bind::<diesel::sql_types::Timestamptz, _>(Utc::now()) .bind::<diesel::sql_types::Timestamptz, _>(Utc::now())
.bind::<diesel::sql_types::Timestamptz, _>( .bind::<diesel::sql_types::Timestamptz, _>(
Utc::now() + Duration::from_std(2 * conf.network.scheduler.interval).unwrap(), Utc::now() + Duration::from_std(2 * conf.network.scheduler.interval).unwrap(),
) )
.load(c) .load(c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
.map_err(|e| Error::from_diesel(e, "".into()))
})
}) })
.await
.context("Get with Class B/C queue-items transaction") .context("Get with Class B/C queue-items transaction")
})
.await?
} }
#[cfg(test)] #[cfg(test)]

134
chirpstack/src/storage/device_keys.rs
View File

@ -1,13 +1,13 @@
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::prelude::*; use diesel::prelude::*;
use tokio::task; use diesel_async::RunQueryDsl;
use tracing::info; use tracing::info;
use lrwn::{AES128Key, EUI64}; use lrwn::{AES128Key, EUI64};
use super::error::Error; use super::error::Error;
use super::get_db_conn; use super::get_async_db_conn;
use super::schema::device_keys; use super::schema::device_keys;
#[derive(Queryable, Insertable, AsChangeset, PartialEq, Eq, Debug, Clone)] #[derive(Queryable, Insertable, AsChangeset, PartialEq, Eq, Debug, Clone)]
@ -45,16 +45,12 @@ impl Default for DeviceKeys {
} }
pub async fn create(dk: DeviceKeys) -> Result<DeviceKeys, Error> { pub async fn create(dk: DeviceKeys) -> Result<DeviceKeys, Error> {
let dk = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceKeys, Error> { let dk: DeviceKeys = diesel::insert_into(device_keys::table)
let mut c = get_db_conn()?; .values(&dk)
diesel::insert_into(device_keys::table) .get_result(&mut c)
.values(&dk) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, dk.dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, dk.dev_eui.to_string()))
}
})
.await??;
info!( info!(
dev_eui = %dk.dev_eui, dev_eui = %dk.dev_eui,
"Device-keys created" "Device-keys created"
@ -63,31 +59,22 @@ pub async fn create(dk: DeviceKeys) -> Result<DeviceKeys, Error> {
} }
pub async fn get(dev_eui: &EUI64) -> Result<DeviceKeys, Error> { pub async fn get(dev_eui: &EUI64) -> Result<DeviceKeys, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let dk = device_keys::dsl::device_keys
move || -> Result<DeviceKeys, Error> { .find(&dev_eui)
let mut c = get_db_conn()?; .first(&mut c)
let dk = device_keys::dsl::device_keys .await
.find(&dev_eui) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.first(&mut c) Ok(dk)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
Ok(dk)
}
})
.await?
} }
pub async fn update(dk: DeviceKeys) -> Result<DeviceKeys, Error> { pub async fn update(dk: DeviceKeys) -> Result<DeviceKeys, Error> {
let dk = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceKeys, Error> { let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(&dk.dev_eui))
let mut c = get_db_conn()?; .set(&dk)
diesel::update(device_keys::dsl::device_keys.find(&dk.dev_eui)) .get_result(&mut c)
.set(&dk) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, dk.dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, dk.dev_eui.to_string()))
}
})
.await??;
info!( info!(
dev_eui = %dk.dev_eui, dev_eui = %dk.dev_eui,
"Device-keys updated" "Device-keys updated"
@ -96,19 +83,13 @@ pub async fn update(dk: DeviceKeys) -> Result<DeviceKeys, Error> {
} }
pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> { pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let ra = diesel::delete(device_keys::dsl::device_keys.find(&dev_eui))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = if ra == 0 {
diesel::delete(device_keys::dsl::device_keys.find(&dev_eui)).execute(&mut c)?; return Err(Error::NotFound(dev_eui.to_string()));
if ra == 0 { }
return Err(Error::NotFound(dev_eui.to_string()));
}
Ok(())
}
})
.await??;
info!( info!(
dev_eui = %dev_eui, dev_eui = %dev_eui,
"Device-keys deleted" "Device-keys deleted"
@ -117,18 +98,12 @@ pub async fn delete(dev_eui: &EUI64) -> Result<(), Error> {
} }
pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result<DeviceKeys, Error> { pub async fn set_dev_nonces(dev_eui: &EUI64, nonces: &[i32]) -> Result<DeviceKeys, Error> {
let dk = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(dev_eui))
let nonces = nonces.to_vec(); .set(device_keys::dev_nonces.eq(nonces))
move || -> Result<DeviceKeys, Error> { .get_result(&mut c)
let mut c = get_db_conn()?; .await
diesel::update(device_keys::dsl::device_keys.find(&dev_eui)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.set(device_keys::dev_nonces.eq(&nonces))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!( info!(
dev_eui = %dev_eui, dev_eui = %dev_eui,
"Dev-nonces updated" "Dev-nonces updated"
@ -140,15 +115,16 @@ pub async fn validate_incr_join_and_store_dev_nonce(
dev_eui: &EUI64, dev_eui: &EUI64,
dev_nonce: i32, dev_nonce: i32,
) -> Result<DeviceKeys, Error> { ) -> Result<DeviceKeys, Error> {
let dk = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let dk: DeviceKeys = c
move || -> Result<DeviceKeys, Error> { .build_transaction()
let mut c = get_db_conn()?; .run::<DeviceKeys, Error, _>(|c| {
c.transaction::<DeviceKeys, Error, _>(|c| { Box::pin(async move {
let mut dk: DeviceKeys = device_keys::dsl::device_keys let mut dk: DeviceKeys = device_keys::dsl::device_keys
.find(&dev_eui) .find(&dev_eui)
.for_update() .for_update()
.first(c) .first(c)
.await
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?; .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
if dk.dev_nonces.contains(&(Some(dev_nonce))) { if dk.dev_nonces.contains(&(Some(dev_nonce))) {
@ -165,11 +141,11 @@ pub async fn validate_incr_join_and_store_dev_nonce(
device_keys::join_nonce.eq(&dk.join_nonce), device_keys::join_nonce.eq(&dk.join_nonce),
)) ))
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, dev_eui.to_string())) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}) })
} })
}) .await?;
.await??;
info!(dev_eui = %dev_eui, dev_nonce = dev_nonce, "Device-nonce validated, join-nonce incremented and stored"); info!(dev_eui = %dev_eui, dev_nonce = dev_nonce, "Device-nonce validated, join-nonce incremented and stored");
Ok(dk) Ok(dk)
@ -182,20 +158,16 @@ pub mod test {
use crate::test; use crate::test;
pub async fn reset_nonces(dev_eui: &EUI64) -> Result<DeviceKeys, Error> { pub async fn reset_nonces(dev_eui: &EUI64) -> Result<DeviceKeys, Error> {
let dk = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let dk: DeviceKeys = diesel::update(device_keys::dsl::device_keys.find(&dev_eui))
move || -> Result<DeviceKeys, Error> { .set((
let mut c = get_db_conn()?; device_keys::dev_nonces.eq::<Vec<i32>>(Vec::new()),
diesel::update(device_keys::dsl::device_keys.find(&dev_eui)) device_keys::join_nonce.eq(0),
.set(( ))
device_keys::dev_nonces.eq::<Vec<i32>>(Vec::new()), .get_result(&mut c)
device_keys::join_nonce.eq(0), .await
)) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!( info!(
dev_eui = %dev_eui, dev_eui = %dev_eui,
"Nonces reset" "Nonces reset"

306
chirpstack/src/storage/device_profile.rs
View File

@ -2,9 +2,8 @@ use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
@ -12,7 +11,7 @@ use lrwn::region::{CommonName, MacVersion, Revision};
use super::error::Error; use super::error::Error;
use super::schema::device_profile; use super::schema::device_profile;
use super::{error, fields, get_db_conn}; use super::{error, fields, get_async_db_conn};
use crate::api::helpers::ToProto; use crate::api::helpers::ToProto;
use crate::codec::Codec; use crate::codec::Codec;
use chirpstack_api::internal; use chirpstack_api::internal;
@ -199,170 +198,136 @@ pub struct Filters {
pub async fn create(dp: DeviceProfile) -> Result<DeviceProfile, Error> { pub async fn create(dp: DeviceProfile) -> Result<DeviceProfile, Error> {
dp.validate()?; dp.validate()?;
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfile, Error> { let dp: DeviceProfile = diesel::insert_into(device_profile::table)
let mut c = get_db_conn()?; .values(&dp)
diesel::insert_into(device_profile::table) .get_result(&mut c)
.values(&dp) .await
.get_result(&mut c) .map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))?;
.map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))
}
})
.await??;
info!(id = %dp.id, "Device-profile created"); info!(id = %dp.id, "Device-profile created");
Ok(dp) Ok(dp)
} }
pub async fn get(id: &Uuid) -> Result<DeviceProfile, Error> { pub async fn get(id: &Uuid) -> Result<DeviceProfile, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let dp = device_profile::dsl::device_profile
move || -> Result<DeviceProfile, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
let dp = device_profile::dsl::device_profile .await
.find(&id) .map_err(|e| error::Error::from_diesel(e, id.to_string()))?;
.first(&mut c) Ok(dp)
.map_err(|e| error::Error::from_diesel(e, id.to_string()))?;
Ok(dp)
}
})
.await?
} }
pub async fn update(dp: DeviceProfile) -> Result<DeviceProfile, Error> { pub async fn update(dp: DeviceProfile) -> Result<DeviceProfile, Error> {
dp.validate()?; dp.validate()?;
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfile, Error> {
let mut c = get_db_conn()?; let dp: DeviceProfile = diesel::update(device_profile::dsl::device_profile.find(&dp.id))
.set((
device_profile::updated_at.eq(Utc::now()),
device_profile::name.eq(&dp.name),
device_profile::description.eq(&dp.description),
device_profile::region.eq(&dp.region),
device_profile::mac_version.eq(&dp.mac_version),
device_profile::reg_params_revision.eq(&dp.reg_params_revision),
device_profile::adr_algorithm_id.eq(&dp.adr_algorithm_id),
device_profile::payload_codec_runtime.eq(&dp.payload_codec_runtime),
device_profile::payload_codec_script.eq(&dp.payload_codec_script),
device_profile::flush_queue_on_activate.eq(&dp.flush_queue_on_activate),
device_profile::uplink_interval.eq(&dp.uplink_interval),
device_profile::device_status_req_interval.eq(&dp.device_status_req_interval),
device_profile::supports_otaa.eq(&dp.supports_otaa),
device_profile::supports_class_b.eq(&dp.supports_class_b),
device_profile::supports_class_c.eq(&dp.supports_class_c),
device_profile::class_b_timeout.eq(&dp.class_b_timeout),
device_profile::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k),
device_profile::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr),
device_profile::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq),
device_profile::class_c_timeout.eq(&dp.class_c_timeout),
device_profile::abp_rx1_delay.eq(&dp.abp_rx1_delay),
device_profile::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset),
device_profile::abp_rx2_dr.eq(&dp.abp_rx2_dr),
device_profile::abp_rx2_freq.eq(&dp.abp_rx2_freq),
device_profile::tags.eq(&dp.tags),
device_profile::measurements.eq(&dp.measurements),
device_profile::auto_detect_measurements.eq(&dp.auto_detect_measurements),
device_profile::region_config_id.eq(&dp.region_config_id),
device_profile::is_relay.eq(&dp.is_relay),
device_profile::is_relay_ed.eq(&dp.is_relay_ed),
device_profile::relay_ed_relay_only.eq(&dp.relay_ed_relay_only),
device_profile::relay_enabled.eq(&dp.relay_enabled),
device_profile::relay_cad_periodicity.eq(&dp.relay_cad_periodicity),
device_profile::relay_default_channel_index.eq(&dp.relay_default_channel_index),
device_profile::relay_second_channel_freq.eq(&dp.relay_second_channel_freq),
device_profile::relay_second_channel_dr.eq(&dp.relay_second_channel_dr),
device_profile::relay_second_channel_ack_offset.eq(&dp.relay_second_channel_ack_offset),
device_profile::relay_ed_activation_mode.eq(&dp.relay_ed_activation_mode),
device_profile::relay_ed_smart_enable_level.eq(&dp.relay_ed_smart_enable_level),
device_profile::relay_ed_back_off.eq(&dp.relay_ed_back_off),
device_profile::relay_ed_uplink_limit_bucket_size
.eq(&dp.relay_ed_uplink_limit_bucket_size),
device_profile::relay_ed_uplink_limit_reload_rate
.eq(&dp.relay_ed_uplink_limit_reload_rate),
device_profile::relay_join_req_limit_reload_rate
.eq(&dp.relay_join_req_limit_reload_rate),
device_profile::relay_notify_limit_reload_rate.eq(&dp.relay_notify_limit_reload_rate),
device_profile::relay_global_uplink_limit_reload_rate
.eq(&dp.relay_global_uplink_limit_reload_rate),
device_profile::relay_overall_limit_reload_rate.eq(&dp.relay_overall_limit_reload_rate),
device_profile::relay_join_req_limit_bucket_size
.eq(&dp.relay_join_req_limit_bucket_size),
device_profile::relay_notify_limit_bucket_size.eq(&dp.relay_notify_limit_bucket_size),
device_profile::relay_global_uplink_limit_bucket_size
.eq(&dp.relay_global_uplink_limit_bucket_size),
device_profile::relay_overall_limit_bucket_size.eq(&dp.relay_overall_limit_bucket_size),
device_profile::allow_roaming.eq(&dp.allow_roaming),
))
.get_result(&mut c)
.await
.map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))?;
diesel::update(device_profile::dsl::device_profile.find(&dp.id))
.set((
device_profile::updated_at.eq(Utc::now()),
device_profile::name.eq(&dp.name),
device_profile::description.eq(&dp.description),
device_profile::region.eq(&dp.region),
device_profile::mac_version.eq(&dp.mac_version),
device_profile::reg_params_revision.eq(&dp.reg_params_revision),
device_profile::adr_algorithm_id.eq(&dp.adr_algorithm_id),
device_profile::payload_codec_runtime.eq(&dp.payload_codec_runtime),
device_profile::payload_codec_script.eq(&dp.payload_codec_script),
device_profile::flush_queue_on_activate.eq(&dp.flush_queue_on_activate),
device_profile::uplink_interval.eq(&dp.uplink_interval),
device_profile::device_status_req_interval.eq(&dp.device_status_req_interval),
device_profile::supports_otaa.eq(&dp.supports_otaa),
device_profile::supports_class_b.eq(&dp.supports_class_b),
device_profile::supports_class_c.eq(&dp.supports_class_c),
device_profile::class_b_timeout.eq(&dp.class_b_timeout),
device_profile::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k),
device_profile::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr),
device_profile::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq),
device_profile::class_c_timeout.eq(&dp.class_c_timeout),
device_profile::abp_rx1_delay.eq(&dp.abp_rx1_delay),
device_profile::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset),
device_profile::abp_rx2_dr.eq(&dp.abp_rx2_dr),
device_profile::abp_rx2_freq.eq(&dp.abp_rx2_freq),
device_profile::tags.eq(&dp.tags),
device_profile::measurements.eq(&dp.measurements),
device_profile::auto_detect_measurements.eq(&dp.auto_detect_measurements),
device_profile::region_config_id.eq(&dp.region_config_id),
device_profile::is_relay.eq(&dp.is_relay),
device_profile::is_relay_ed.eq(&dp.is_relay_ed),
device_profile::relay_ed_relay_only.eq(&dp.relay_ed_relay_only),
device_profile::relay_enabled.eq(&dp.relay_enabled),
device_profile::relay_cad_periodicity.eq(&dp.relay_cad_periodicity),
device_profile::relay_default_channel_index.eq(&dp.relay_default_channel_index),
device_profile::relay_second_channel_freq.eq(&dp.relay_second_channel_freq),
device_profile::relay_second_channel_dr.eq(&dp.relay_second_channel_dr),
device_profile::relay_second_channel_ack_offset
.eq(&dp.relay_second_channel_ack_offset),
device_profile::relay_ed_activation_mode.eq(&dp.relay_ed_activation_mode),
device_profile::relay_ed_smart_enable_level.eq(&dp.relay_ed_smart_enable_level),
device_profile::relay_ed_back_off.eq(&dp.relay_ed_back_off),
device_profile::relay_ed_uplink_limit_bucket_size
.eq(&dp.relay_ed_uplink_limit_bucket_size),
device_profile::relay_ed_uplink_limit_reload_rate
.eq(&dp.relay_ed_uplink_limit_reload_rate),
device_profile::relay_join_req_limit_reload_rate
.eq(&dp.relay_join_req_limit_reload_rate),
device_profile::relay_notify_limit_reload_rate
.eq(&dp.relay_notify_limit_reload_rate),
device_profile::relay_global_uplink_limit_reload_rate
.eq(&dp.relay_global_uplink_limit_reload_rate),
device_profile::relay_overall_limit_reload_rate
.eq(&dp.relay_overall_limit_reload_rate),
device_profile::relay_join_req_limit_bucket_size
.eq(&dp.relay_join_req_limit_bucket_size),
device_profile::relay_notify_limit_bucket_size
.eq(&dp.relay_notify_limit_bucket_size),
device_profile::relay_global_uplink_limit_bucket_size
.eq(&dp.relay_global_uplink_limit_bucket_size),
device_profile::relay_overall_limit_bucket_size
.eq(&dp.relay_overall_limit_bucket_size),
device_profile::allow_roaming.eq(&dp.allow_roaming),
))
.get_result(&mut c)
.map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))
}
})
.await??;
info!(id = %dp.id, "Device-profile updated"); info!(id = %dp.id, "Device-profile updated");
Ok(dp) Ok(dp)
} }
pub async fn set_measurements(id: Uuid, m: &fields::Measurements) -> Result<DeviceProfile, Error> { pub async fn set_measurements(id: Uuid, m: &fields::Measurements) -> Result<DeviceProfile, Error> {
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let m = m.clone(); let dp: DeviceProfile = diesel::update(device_profile::dsl::device_profile.find(&id))
move || -> Result<DeviceProfile, Error> { .set(device_profile::measurements.eq(m))
let mut c = get_db_conn()?; .get_result(&mut c)
diesel::update(device_profile::dsl::device_profile.find(&id)) .await
.set(device_profile::measurements.eq(m)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))
}
})
.await??;
info!(id = %id, "Device-profile measurements updated"); info!(id = %id, "Device-profile measurements updated");
Ok(dp) Ok(dp)
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(device_profile::dsl::device_profile.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = if ra == 0 {
diesel::delete(device_profile::dsl::device_profile.find(&id)).execute(&mut c)?; return Err(error::Error::NotFound(id.to_string()));
if ra == 0 { }
return Err(error::Error::NotFound(id.to_string()));
}
Ok(())
}
})
.await??;
info!(id = %id, "Device-profile deleted"); info!(id = %id, "Device-profile deleted");
Ok(()) Ok(())
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device_profile::dsl::device_profile
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .into_boxed();
let mut q = device_profile::dsl::device_profile
.select(dsl::count_star())
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(device_profile::dsl::tenant_id.eq(tenant_id)); q = q.filter(device_profile::dsl::tenant_id.eq(tenant_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(device_profile::dsl::name.ilike(format!("%{}%", search))); q = q.filter(device_profile::dsl::name.ilike(format!("%{}%", search)));
} }
Ok(q.first(&mut c)?) Ok(q.first(&mut c).await?)
}
})
.await?
} }
pub async fn list( pub async fn list(
@ -370,42 +335,37 @@ pub async fn list(
offset: i64, offset: i64,
filters: &Filters, filters: &Filters,
) -> Result<Vec<DeviceProfileListItem>, Error> { ) -> Result<Vec<DeviceProfileListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device_profile::dsl::device_profile
move || -> Result<Vec<DeviceProfileListItem>, Error> { .select((
let mut c = get_db_conn()?; device_profile::id,
let mut q = device_profile::dsl::device_profile device_profile::created_at,
.select(( device_profile::updated_at,
device_profile::id, device_profile::name,
device_profile::created_at, device_profile::region,
device_profile::updated_at, device_profile::mac_version,
device_profile::name, device_profile::reg_params_revision,
device_profile::region, device_profile::supports_otaa,
device_profile::mac_version, device_profile::supports_class_b,
device_profile::reg_params_revision, device_profile::supports_class_c,
device_profile::supports_otaa, ))
device_profile::supports_class_b, .into_boxed();
device_profile::supports_class_c,
))
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(device_profile::dsl::tenant_id.eq(tenant_id)); q = q.filter(device_profile::dsl::tenant_id.eq(tenant_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(device_profile::dsl::name.ilike(format!("%{}%", search))); q = q.filter(device_profile::dsl::name.ilike(format!("%{}%", search)));
} }
let items = q let items = q
.order_by(device_profile::dsl::name) .order_by(device_profile::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c)?; .load(&mut c)
Ok(items) .await?;
} Ok(items)
})
.await?
} }
#[cfg(test)] #[cfg(test)]

298
chirpstack/src/storage/device_profile_template.rs
View File

@ -2,17 +2,16 @@ use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use regex::Regex; use regex::Regex;
use tokio::task;
use tracing::info; use tracing::info;
use lrwn::region::{CommonName, MacVersion, Revision}; use lrwn::region::{CommonName, MacVersion, Revision};
use super::error::Error; use super::error::Error;
use super::schema::device_profile_template; use super::schema::device_profile_template;
use super::{error, fields, get_db_conn}; use super::{error, fields, get_async_db_conn};
use crate::codec::Codec; use crate::codec::Codec;
#[derive(Clone, Queryable, Insertable, Debug, PartialEq, Eq)] #[derive(Clone, Queryable, Insertable, Debug, PartialEq, Eq)]
@ -134,197 +133,164 @@ pub struct DeviceProfileTemplateListItem {
pub async fn create(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> { pub async fn create(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> {
dp.validate()?; dp.validate()?;
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfileTemplate, Error> { let dp: DeviceProfileTemplate = diesel::insert_into(device_profile_template::table)
let mut c = get_db_conn()?; .values(&dp)
diesel::insert_into(device_profile_template::table) .get_result(&mut c)
.values(&dp) .await
.get_result(&mut c) .map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))?;
.map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))
}
})
.await??;
info!(id = %dp.id, "Device-profile template created"); info!(id = %dp.id, "Device-profile template created");
Ok(dp) Ok(dp)
} }
pub async fn upsert(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> { pub async fn upsert(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> {
dp.validate()?; dp.validate()?;
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfileTemplate, Error> { let dp: DeviceProfileTemplate = diesel::insert_into(device_profile_template::table)
let mut c = get_db_conn()?; .values(&dp)
diesel::insert_into(device_profile_template::table) .on_conflict(device_profile_template::id)
.values(&dp) .do_update()
.on_conflict(device_profile_template::id) .set((
.do_update() device_profile_template::updated_at.eq(Utc::now()),
.set(( device_profile_template::name.eq(&dp.name),
device_profile_template::updated_at.eq(Utc::now()), device_profile_template::description.eq(&dp.description),
device_profile_template::name.eq(&dp.name), device_profile_template::vendor.eq(&dp.vendor),
device_profile_template::description.eq(&dp.description), device_profile_template::firmware.eq(&dp.firmware),
device_profile_template::vendor.eq(&dp.vendor), device_profile_template::region.eq(&dp.region),
device_profile_template::firmware.eq(&dp.firmware), device_profile_template::mac_version.eq(&dp.mac_version),
device_profile_template::region.eq(&dp.region), device_profile_template::reg_params_revision.eq(&dp.reg_params_revision),
device_profile_template::mac_version.eq(&dp.mac_version), device_profile_template::adr_algorithm_id.eq(&dp.adr_algorithm_id),
device_profile_template::reg_params_revision.eq(&dp.reg_params_revision), device_profile_template::payload_codec_runtime.eq(&dp.payload_codec_runtime),
device_profile_template::adr_algorithm_id.eq(&dp.adr_algorithm_id), device_profile_template::payload_codec_script.eq(&dp.payload_codec_script),
device_profile_template::payload_codec_runtime.eq(&dp.payload_codec_runtime), device_profile_template::uplink_interval.eq(&dp.uplink_interval),
device_profile_template::payload_codec_script.eq(&dp.payload_codec_script), device_profile_template::device_status_req_interval.eq(&dp.device_status_req_interval),
device_profile_template::uplink_interval.eq(&dp.uplink_interval), device_profile_template::flush_queue_on_activate.eq(&dp.flush_queue_on_activate),
device_profile_template::device_status_req_interval device_profile_template::supports_otaa.eq(&dp.supports_otaa),
.eq(&dp.device_status_req_interval), device_profile_template::supports_class_b.eq(&dp.supports_class_b),
device_profile_template::flush_queue_on_activate device_profile_template::supports_class_c.eq(&dp.supports_class_c),
.eq(&dp.flush_queue_on_activate), device_profile_template::class_b_timeout.eq(&dp.class_b_timeout),
device_profile_template::supports_otaa.eq(&dp.supports_otaa), device_profile_template::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k),
device_profile_template::supports_class_b.eq(&dp.supports_class_b), device_profile_template::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr),
device_profile_template::supports_class_c.eq(&dp.supports_class_c), device_profile_template::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq),
device_profile_template::class_b_timeout.eq(&dp.class_b_timeout), device_profile_template::class_c_timeout.eq(&dp.class_c_timeout),
device_profile_template::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k), device_profile_template::abp_rx1_delay.eq(&dp.abp_rx1_delay),
device_profile_template::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr), device_profile_template::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset),
device_profile_template::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq), device_profile_template::abp_rx2_dr.eq(&dp.abp_rx2_dr),
device_profile_template::class_c_timeout.eq(&dp.class_c_timeout), device_profile_template::abp_rx2_freq.eq(&dp.abp_rx2_freq),
device_profile_template::abp_rx1_delay.eq(&dp.abp_rx1_delay), device_profile_template::tags.eq(&dp.tags),
device_profile_template::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset), device_profile_template::measurements.eq(&dp.measurements),
device_profile_template::abp_rx2_dr.eq(&dp.abp_rx2_dr), device_profile_template::auto_detect_measurements.eq(&dp.auto_detect_measurements),
device_profile_template::abp_rx2_freq.eq(&dp.abp_rx2_freq), ))
device_profile_template::tags.eq(&dp.tags), .get_result(&mut c)
device_profile_template::measurements.eq(&dp.measurements), .await
device_profile_template::auto_detect_measurements .map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))?;
.eq(&dp.auto_detect_measurements),
))
.get_result(&mut c)
.map_err(|e| error::Error::from_diesel(e, dp.id.to_string()))
}
})
.await??;
info!(id = %dp.id, "Device-profile template upserted"); info!(id = %dp.id, "Device-profile template upserted");
Ok(dp) Ok(dp)
} }
pub async fn get(id: &str) -> Result<DeviceProfileTemplate, Error> { pub async fn get(id: &str) -> Result<DeviceProfileTemplate, Error> {
task::spawn_blocking({ let id = id.to_string();
let id = id.to_string(); let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfileTemplate, Error> { let dp = device_profile_template::dsl::device_profile_template
let mut c = get_db_conn()?; .find(&id)
let dp = device_profile_template::dsl::device_profile_template .first(&mut c)
.find(&id) .await
.first(&mut c) .map_err(|e| error::Error::from_diesel(e, id.clone()))?;
.map_err(|e| error::Error::from_diesel(e, id.clone()))?; Ok(dp)
Ok(dp)
}
})
.await?
} }
pub async fn update(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> { pub async fn update(dp: DeviceProfileTemplate) -> Result<DeviceProfileTemplate, Error> {
dp.validate()?; dp.validate()?;
let dp = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceProfileTemplate, Error> {
let mut c = get_db_conn()?;
diesel::update(device_profile_template::dsl::device_profile_template.find(&dp.id)) let dp: DeviceProfileTemplate =
.set(( diesel::update(device_profile_template::dsl::device_profile_template.find(&dp.id))
device_profile_template::updated_at.eq(Utc::now()), .set((
device_profile_template::name.eq(&dp.name), device_profile_template::updated_at.eq(Utc::now()),
device_profile_template::description.eq(&dp.description), device_profile_template::name.eq(&dp.name),
device_profile_template::vendor.eq(&dp.vendor), device_profile_template::description.eq(&dp.description),
device_profile_template::firmware.eq(&dp.firmware), device_profile_template::vendor.eq(&dp.vendor),
device_profile_template::region.eq(&dp.region), device_profile_template::firmware.eq(&dp.firmware),
device_profile_template::mac_version.eq(&dp.mac_version), device_profile_template::region.eq(&dp.region),
device_profile_template::reg_params_revision.eq(&dp.reg_params_revision), device_profile_template::mac_version.eq(&dp.mac_version),
device_profile_template::adr_algorithm_id.eq(&dp.adr_algorithm_id), device_profile_template::reg_params_revision.eq(&dp.reg_params_revision),
device_profile_template::payload_codec_runtime.eq(&dp.payload_codec_runtime), device_profile_template::adr_algorithm_id.eq(&dp.adr_algorithm_id),
device_profile_template::payload_codec_script.eq(&dp.payload_codec_script), device_profile_template::payload_codec_runtime.eq(&dp.payload_codec_runtime),
device_profile_template::uplink_interval.eq(&dp.uplink_interval), device_profile_template::payload_codec_script.eq(&dp.payload_codec_script),
device_profile_template::device_status_req_interval device_profile_template::uplink_interval.eq(&dp.uplink_interval),
.eq(&dp.device_status_req_interval), device_profile_template::device_status_req_interval
device_profile_template::flush_queue_on_activate .eq(&dp.device_status_req_interval),
.eq(&dp.flush_queue_on_activate), device_profile_template::flush_queue_on_activate.eq(&dp.flush_queue_on_activate),
device_profile_template::supports_otaa.eq(&dp.supports_otaa), device_profile_template::supports_otaa.eq(&dp.supports_otaa),
device_profile_template::supports_class_b.eq(&dp.supports_class_b), device_profile_template::supports_class_b.eq(&dp.supports_class_b),
device_profile_template::supports_class_c.eq(&dp.supports_class_c), device_profile_template::supports_class_c.eq(&dp.supports_class_c),
device_profile_template::class_b_timeout.eq(&dp.class_b_timeout), device_profile_template::class_b_timeout.eq(&dp.class_b_timeout),
device_profile_template::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k), device_profile_template::class_b_ping_slot_nb_k.eq(&dp.class_b_ping_slot_nb_k),
device_profile_template::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr), device_profile_template::class_b_ping_slot_dr.eq(&dp.class_b_ping_slot_dr),
device_profile_template::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq), device_profile_template::class_b_ping_slot_freq.eq(&dp.class_b_ping_slot_freq),
device_profile_template::class_c_timeout.eq(&dp.class_c_timeout), device_profile_template::class_c_timeout.eq(&dp.class_c_timeout),
device_profile_template::abp_rx1_delay.eq(&dp.abp_rx1_delay), device_profile_template::abp_rx1_delay.eq(&dp.abp_rx1_delay),
device_profile_template::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset), device_profile_template::abp_rx1_dr_offset.eq(&dp.abp_rx1_dr_offset),
device_profile_template::abp_rx2_dr.eq(&dp.abp_rx2_dr), device_profile_template::abp_rx2_dr.eq(&dp.abp_rx2_dr),
device_profile_template::abp_rx2_freq.eq(&dp.abp_rx2_freq), device_profile_template::abp_rx2_freq.eq(&dp.abp_rx2_freq),
device_profile_template::tags.eq(&dp.tags), device_profile_template::tags.eq(&dp.tags),
)) ))
.get_result(&mut c) .get_result(&mut c)
.map_err(|e| error::Error::from_diesel(e, dp.id.clone())) .await
} .map_err(|e| error::Error::from_diesel(e, dp.id.clone()))?;
})
.await??;
info!(id = %dp.id, "Device-profile template updated"); info!(id = %dp.id, "Device-profile template updated");
Ok(dp) Ok(dp)
} }
pub async fn delete(id: &str) -> Result<(), Error> { pub async fn delete(id: &str) -> Result<(), Error> {
task::spawn_blocking({ let id = id.to_string();
let id = id.to_string(); let mut c = get_async_db_conn().await?;
move || -> Result<(), Error> { let ra = diesel::delete(device_profile_template::dsl::device_profile_template.find(&id))
let mut c = get_db_conn()?; .execute(&mut c)
let ra = .await?;
diesel::delete(device_profile_template::dsl::device_profile_template.find(&id)) if ra == 0 {
.execute(&mut c)?; return Err(error::Error::NotFound(id));
if ra == 0 { }
return Err(error::Error::NotFound(id));
}
Ok(())
}
})
.await??;
info!(id = %id, "Device-profile template deleted"); info!(id = %id, "Device-profile template deleted");
Ok(()) Ok(())
} }
pub async fn get_count() -> Result<i64, Error> { pub async fn get_count() -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<i64, Error> { Ok(device_profile_template::dsl::device_profile_template
let mut c = get_db_conn()?; .select(dsl::count_star())
Ok(device_profile_template::dsl::device_profile_template .first(&mut c)
.select(dsl::count_star()) .await?)
.first(&mut c)?)
}
})
.await?
} }
pub async fn list(limit: i64, offset: i64) -> Result<Vec<DeviceProfileTemplateListItem>, Error> { pub async fn list(limit: i64, offset: i64) -> Result<Vec<DeviceProfileTemplateListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Vec<DeviceProfileTemplateListItem>, Error> { let items = device_profile_template::dsl::device_profile_template
let mut c = get_db_conn()?; .select((
let items = device_profile_template::dsl::device_profile_template device_profile_template::id,
.select(( device_profile_template::created_at,
device_profile_template::id, device_profile_template::updated_at,
device_profile_template::created_at, device_profile_template::name,
device_profile_template::updated_at, device_profile_template::vendor,
device_profile_template::name, device_profile_template::firmware,
device_profile_template::vendor, device_profile_template::region,
device_profile_template::firmware, device_profile_template::mac_version,
device_profile_template::region, device_profile_template::reg_params_revision,
device_profile_template::mac_version, device_profile_template::supports_otaa,
device_profile_template::reg_params_revision, device_profile_template::supports_class_b,
device_profile_template::supports_otaa, device_profile_template::supports_class_c,
device_profile_template::supports_class_b, ))
device_profile_template::supports_class_c, .order_by((
)) device_profile_template::dsl::vendor,
.order_by(( device_profile_template::dsl::name,
device_profile_template::dsl::vendor, device_profile_template::dsl::firmware,
device_profile_template::dsl::name, device_profile_template::dsl::region,
device_profile_template::dsl::firmware, ))
device_profile_template::dsl::region, .limit(limit)
)) .offset(offset)
.limit(limit) .load(&mut c)
.offset(offset) .await?;
.load(&mut c)?; Ok(items)
Ok(items)
}
})
.await?
} }
#[cfg(test)] #[cfg(test)]

220
chirpstack/src/storage/device_queue.rs
View File

@ -1,12 +1,12 @@
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::{dsl, prelude::*}; use diesel::{dsl, prelude::*};
use tokio::task; use diesel_async::RunQueryDsl;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use super::error::Error; use super::error::Error;
use super::get_db_conn; use super::get_async_db_conn;
use super::schema::device_queue_item; use super::schema::device_queue_item;
use lrwn::EUI64; use lrwn::EUI64;
@ -64,172 +64,128 @@ impl Default for DeviceQueueItem {
pub async fn enqueue_item(qi: DeviceQueueItem) -> Result<DeviceQueueItem, Error> { pub async fn enqueue_item(qi: DeviceQueueItem) -> Result<DeviceQueueItem, Error> {
qi.validate()?; qi.validate()?;
let mut c = get_async_db_conn().await?;
let qi = task::spawn_blocking({ let qi: DeviceQueueItem = diesel::insert_into(device_queue_item::table)
move || -> Result<DeviceQueueItem, Error> { .values(&qi)
let mut c = get_db_conn()?; .get_result(&mut c)
diesel::insert_into(device_queue_item::table) .await
.values(&qi) .map_err(|e| Error::from_diesel(e, qi.id.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, qi.id.to_string()))
}
})
.await??;
info!(id = %qi.id, dev_eui = %qi.dev_eui, "Device queue-item enqueued"); info!(id = %qi.id, dev_eui = %qi.dev_eui, "Device queue-item enqueued");
Ok(qi) Ok(qi)
} }
pub async fn get_item(id: &Uuid) -> Result<DeviceQueueItem, Error> { pub async fn get_item(id: &Uuid) -> Result<DeviceQueueItem, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let qi = device_queue_item::dsl::device_queue_item
move || -> Result<DeviceQueueItem, Error> { .find(id)
let mut c = get_db_conn()?; .first(&mut c)
let qi = device_queue_item::dsl::device_queue_item .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.first(&mut c) Ok(qi)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(qi)
}
})
.await?
} }
pub async fn update_item(qi: DeviceQueueItem) -> Result<DeviceQueueItem, Error> { pub async fn update_item(qi: DeviceQueueItem) -> Result<DeviceQueueItem, Error> {
let qi = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<DeviceQueueItem, Error> { let qi: DeviceQueueItem =
let mut c = get_db_conn()?; diesel::update(device_queue_item::dsl::device_queue_item.find(&qi.id))
diesel::update(device_queue_item::dsl::device_queue_item.find(&qi.id)) .set((
.set(( device_queue_item::is_pending.eq(&qi.is_pending),
device_queue_item::is_pending.eq(&qi.is_pending), device_queue_item::f_cnt_down.eq(&qi.f_cnt_down),
device_queue_item::f_cnt_down.eq(&qi.f_cnt_down), device_queue_item::timeout_after.eq(&qi.timeout_after),
device_queue_item::timeout_after.eq(&qi.timeout_after), ))
)) .get_result(&mut c)
.get_result(&mut c) .await
.map_err(|e| Error::from_diesel(e, qi.id.to_string())) .map_err(|e| Error::from_diesel(e, qi.id.to_string()))?;
}
})
.await??;
info!(id = %qi.id, dev_eui = %qi.dev_eui, "Device queue-item updated"); info!(id = %qi.id, dev_eui = %qi.dev_eui, "Device queue-item updated");
Ok(qi) Ok(qi)
} }
pub async fn delete_item(id: &Uuid) -> Result<(), Error> { pub async fn delete_item(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(device_queue_item::dsl::device_queue_item.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = diesel::delete(device_queue_item::dsl::device_queue_item.find(&id)) if ra == 0 {
.execute(&mut c)?; return Err(Error::NotFound(id.to_string()));
if ra == 0 { }
return Err(Error::NotFound(id.to_string()));
}
Ok(())
}
})
.await??;
info!(id = %id, "Device queue-item deleted"); info!(id = %id, "Device queue-item deleted");
Ok(()) Ok(())
} }
/// It returns the device queue-item and a bool indicating if there are more items in the queue. /// It returns the device queue-item and a bool indicating if there are more items in the queue.
pub async fn get_next_for_dev_eui(dev_eui: &EUI64) -> Result<(DeviceQueueItem, bool), Error> { pub async fn get_next_for_dev_eui(dev_eui: &EUI64) -> Result<(DeviceQueueItem, bool), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let items: Vec<DeviceQueueItem> = device_queue_item::dsl::device_queue_item
move || -> Result<(DeviceQueueItem, bool), Error> { .filter(device_queue_item::dev_eui.eq(&dev_eui))
let mut c = get_db_conn()?; .order_by(device_queue_item::created_at)
let items: Vec<DeviceQueueItem> = device_queue_item::dsl::device_queue_item .limit(2)
.filter(device_queue_item::dev_eui.eq(&dev_eui)) .load(&mut c)
.order_by(device_queue_item::created_at) .await
.limit(2) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.load(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
// Return NotFound on empty Vec. // Return NotFound on empty Vec.
if items.is_empty() { if items.is_empty() {
return Err(Error::NotFound(dev_eui.to_string()));
}
// In case the transmission is pending and hasn't timed-out yet, do not
// return it.
if items[0].is_pending {
if let Some(timeout_after) = &items[0].timeout_after {
if timeout_after > &Utc::now() {
return Err(Error::NotFound(dev_eui.to_string())); return Err(Error::NotFound(dev_eui.to_string()));
} }
// In case the transmission is pending and hasn't timed-out yet, do not
// return it.
if items[0].is_pending {
if let Some(timeout_after) = &items[0].timeout_after {
if timeout_after > &Utc::now() {
return Err(Error::NotFound(dev_eui.to_string()));
}
}
}
// Return first item and bool indicating if there are more items in the queue.
Ok((items[0].clone(), items.len() > 1))
} }
}) }
.await?
// Return first item and bool indicating if there are more items in the queue.
Ok((items[0].clone(), items.len() > 1))
} }
pub async fn get_for_dev_eui(dev_eui: &EUI64) -> Result<Vec<DeviceQueueItem>, Error> { pub async fn get_for_dev_eui(dev_eui: &EUI64) -> Result<Vec<DeviceQueueItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let items = device_queue_item::dsl::device_queue_item
move || -> Result<Vec<DeviceQueueItem>, Error> { .filter(device_queue_item::dev_eui.eq(&dev_eui))
let mut c = get_db_conn()?; .order_by(device_queue_item::created_at)
let items = device_queue_item::dsl::device_queue_item .load(&mut c)
.filter(device_queue_item::dev_eui.eq(&dev_eui)) .await
.order_by(device_queue_item::created_at) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.load(&mut c) Ok(items)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
Ok(items)
}
})
.await?
} }
pub async fn flush_for_dev_eui(dev_eui: &EUI64) -> Result<(), Error> { pub async fn flush_for_dev_eui(dev_eui: &EUI64) -> Result<(), Error> {
let count = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let count: usize = diesel::delete(
move || -> Result<usize, Error> { device_queue_item::dsl::device_queue_item.filter(device_queue_item::dev_eui.eq(&dev_eui)),
let mut c = get_db_conn()?; )
diesel::delete( .execute(&mut c)
device_queue_item::dsl::device_queue_item .await
.filter(device_queue_item::dev_eui.eq(&dev_eui)), .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
)
.execute(&mut c)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))
}
})
.await??;
info!(dev_eui = %dev_eui, count = count, "Device queue flushed"); info!(dev_eui = %dev_eui, count = count, "Device queue flushed");
Ok(()) Ok(())
} }
pub async fn get_pending_for_dev_eui(dev_eui: &EUI64) -> Result<DeviceQueueItem, Error> { pub async fn get_pending_for_dev_eui(dev_eui: &EUI64) -> Result<DeviceQueueItem, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let dev_eui = *dev_eui; let qi = device_queue_item::dsl::device_queue_item
move || -> Result<DeviceQueueItem, Error> { .filter(
let mut c = get_db_conn()?; device_queue_item::dev_eui
let qi = device_queue_item::dsl::device_queue_item .eq(&dev_eui)
.filter( .and(device_queue_item::is_pending.eq(true)),
device_queue_item::dev_eui )
.eq(&dev_eui) .first(&mut c)
.and(device_queue_item::is_pending.eq(true)), .await
) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.first(&mut c) Ok(qi)
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
Ok(qi)
}
})
.await?
} }
pub async fn get_max_f_cnt_down(dev_eui: EUI64) -> Result<Option<i64>, Error> { pub async fn get_max_f_cnt_down(dev_eui: EUI64) -> Result<Option<i64>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Option<i64>, Error> { Ok(device_queue_item::dsl::device_queue_item
let mut c = get_db_conn()?; .select(dsl::max(device_queue_item::f_cnt_down))
Ok(device_queue_item::dsl::device_queue_item .filter(device_queue_item::dsl::dev_eui.eq(dev_eui))
.select(dsl::max(device_queue_item::f_cnt_down)) .first(&mut c)
.filter(device_queue_item::dsl::dev_eui.eq(dev_eui)) .await?)
.first(&mut c)?)
}
})
.await?
} }
#[cfg(test)] #[cfg(test)]
@ -281,7 +237,7 @@ pub mod test {
// get for dev eui // get for dev eui
let queue = get_for_dev_eui(&d.dev_eui).await.unwrap(); let queue = get_for_dev_eui(&d.dev_eui).await.unwrap();
assert_eq!(&qi, queue.first().unwrap()); assert_eq!(qi, queue[0]);
// next next queue item for dev eui // next next queue item for dev eui
let resp = get_next_for_dev_eui(&d.dev_eui).await.unwrap(); let resp = get_next_for_dev_eui(&d.dev_eui).await.unwrap();

364
chirpstack/src/storage/gateway.rs
View File

@ -2,16 +2,15 @@ use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use lrwn::EUI64; use lrwn::EUI64;
use super::schema::{gateway, multicast_group_gateway, tenant}; use super::schema::{gateway, multicast_group_gateway, tenant};
use super::{error::Error, fields, get_db_conn}; use super::{error::Error, fields, get_async_db_conn};
#[derive(Queryable, Insertable, PartialEq, Debug)] #[derive(Queryable, Insertable, PartialEq, Debug)]
#[diesel(table_name = gateway)] #[diesel(table_name = gateway)]
@ -110,15 +109,17 @@ impl Default for Gateway {
pub async fn create(gw: Gateway) -> Result<Gateway, Error> { pub async fn create(gw: Gateway) -> Result<Gateway, Error> {
gw.validate()?; gw.validate()?;
let gw = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Gateway, Error> { let gw: Gateway = c
let mut c = get_db_conn()?; .build_transaction()
c.transaction::<Gateway, Error, _>(|c| { .run::<Gateway, Error, _>(|c| {
Box::pin(async move {
// use for_update to lock the tenant. // use for_update to lock the tenant.
let t: super::tenant::Tenant = tenant::dsl::tenant let t: super::tenant::Tenant = tenant::dsl::tenant
.find(&gw.tenant_id) .find(&gw.tenant_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, gw.tenant_id.to_string()))?; .map_err(|e| Error::from_diesel(e, gw.tenant_id.to_string()))?;
if !t.can_have_gateways { if !t.can_have_gateways {
@ -128,7 +129,8 @@ pub async fn create(gw: Gateway) -> Result<Gateway, Error> {
let gw_count: i64 = gateway::dsl::gateway let gw_count: i64 = gateway::dsl::gateway
.select(dsl::count_star()) .select(dsl::count_star())
.filter(gateway::dsl::tenant_id.eq(&gw.tenant_id)) .filter(gateway::dsl::tenant_id.eq(&gw.tenant_id))
.first(c)?; .first(c)
.await?;
if t.max_gateway_count != 0 && gw_count as i32 >= t.max_gateway_count { if t.max_gateway_count != 0 && gw_count as i32 >= t.max_gateway_count {
return Err(Error::NotAllowed( return Err(Error::NotAllowed(
@ -139,11 +141,11 @@ pub async fn create(gw: Gateway) -> Result<Gateway, Error> {
diesel::insert_into(gateway::table) diesel::insert_into(gateway::table)
.values(&gw) .values(&gw)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, gw.gateway_id.to_string())) .map_err(|e| Error::from_diesel(e, gw.gateway_id.to_string()))
}) })
} })
}) .await?;
.await??;
info!( info!(
gateway_id = %gw.gateway_id, gateway_id = %gw.gateway_id,
"Gateway created" "Gateway created"
@ -152,41 +154,32 @@ pub async fn create(gw: Gateway) -> Result<Gateway, Error> {
} }
pub async fn get(gateway_id: &EUI64) -> Result<Gateway, Error> { pub async fn get(gateway_id: &EUI64) -> Result<Gateway, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let gateway_id = *gateway_id; let gw = gateway::dsl::gateway
move || -> Result<Gateway, Error> { .find(&gateway_id)
let mut c = get_db_conn()?; .first(&mut c)
let gw = gateway::dsl::gateway .await
.find(&gateway_id) .map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?;
.first(&mut c) Ok(gw)
.map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?;
Ok(gw)
}
})
.await?
} }
pub async fn update(gw: Gateway) -> Result<Gateway, Error> { pub async fn update(gw: Gateway) -> Result<Gateway, Error> {
gw.validate()?; gw.validate()?;
let gw = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Gateway, Error> { let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&gw.gateway_id))
let mut c = get_db_conn()?; .set((
diesel::update(gateway::dsl::gateway.find(&gw.gateway_id)) gateway::updated_at.eq(Utc::now()),
.set(( gateway::name.eq(&gw.name),
gateway::updated_at.eq(Utc::now()), gateway::description.eq(&gw.description),
gateway::name.eq(&gw.name), gateway::latitude.eq(&gw.latitude),
gateway::description.eq(&gw.description), gateway::longitude.eq(&gw.longitude),
gateway::latitude.eq(&gw.latitude), gateway::altitude.eq(&gw.altitude),
gateway::longitude.eq(&gw.longitude), gateway::stats_interval_secs.eq(&gw.stats_interval_secs),
gateway::altitude.eq(&gw.altitude), gateway::tags.eq(&gw.tags),
gateway::stats_interval_secs.eq(&gw.stats_interval_secs), ))
gateway::tags.eq(&gw.tags), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, gw.gateway_id.to_string()))?;
.map_err(|e| Error::from_diesel(e, gw.gateway_id.to_string()))
}
})
.await??;
info!( info!(
gateway_id = %gw.gateway_id, gateway_id = %gw.gateway_id,
"Gateway updated" "Gateway updated"
@ -195,23 +188,16 @@ pub async fn update(gw: Gateway) -> Result<Gateway, Error> {
} }
pub async fn update_state(id: &EUI64, props: &HashMap<String, String>) -> Result<Gateway, Error> { pub async fn update_state(id: &EUI64, props: &HashMap<String, String>) -> Result<Gateway, Error> {
let gw = task::spawn_blocking({ let props = fields::KeyValue::new(props.clone());
let id = *id; let mut c = get_async_db_conn().await?;
let props = fields::KeyValue::new(props.clone()); let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id))
move || -> Result<Gateway, Error> { .set((
let mut c = get_db_conn()?; gateway::last_seen_at.eq(Some(Utc::now())),
let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id)) gateway::properties.eq(props),
.set(( ))
gateway::last_seen_at.eq(Some(Utc::now())), .get_result(&mut c)
gateway::properties.eq(props), .await
)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(gw)
}
})
.await??;
info!( info!(
gateway_id = %id, gateway_id = %id,
@ -228,26 +214,19 @@ pub async fn update_state_and_loc(
alt: f32, alt: f32,
props: &HashMap<String, String>, props: &HashMap<String, String>,
) -> Result<Gateway, Error> { ) -> Result<Gateway, Error> {
let gw = task::spawn_blocking({ let props = fields::KeyValue::new(props.clone());
let id = *id; let mut c = get_async_db_conn().await?;
let props = fields::KeyValue::new(props.clone()); let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id))
move || -> Result<Gateway, Error> { .set((
let mut c = get_db_conn()?; gateway::last_seen_at.eq(Some(Utc::now())),
let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id)) gateway::latitude.eq(lat),
.set(( gateway::longitude.eq(lon),
gateway::last_seen_at.eq(Some(Utc::now())), gateway::altitude.eq(alt),
gateway::latitude.eq(lat), gateway::properties.eq(props),
gateway::longitude.eq(lon), ))
gateway::altitude.eq(alt), .get_result(&mut c)
gateway::properties.eq(props), .await
)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(gw)
}
})
.await??;
info!( info!(
gateway_id = %id, gateway_id = %id,
@ -258,20 +237,12 @@ pub async fn update_state_and_loc(
} }
pub async fn update_tls_cert(id: &EUI64, cert: &[u8]) -> Result<Gateway, Error> { pub async fn update_tls_cert(id: &EUI64, cert: &[u8]) -> Result<Gateway, Error> {
let gw = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id))
let cert = cert.to_vec(); .set(gateway::tls_certificate.eq(cert))
move || -> Result<Gateway, Error> { .get_result(&mut c)
let mut c = get_db_conn()?; .await
let gw: Gateway = diesel::update(gateway::dsl::gateway.find(&id)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.set(gateway::tls_certificate.eq(cert))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(gw)
}
})
.await??;
info!( info!(
gateway_id = %id, gateway_id = %id,
"Gateway tls certificate updated" "Gateway tls certificate updated"
@ -281,18 +252,13 @@ pub async fn update_tls_cert(id: &EUI64, cert: &[u8]) -> Result<Gateway, Error>
} }
pub async fn delete(gateway_id: &EUI64) -> Result<(), Error> { pub async fn delete(gateway_id: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let gateway_id = *gateway_id; let ra = diesel::delete(gateway::dsl::gateway.find(&gateway_id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = diesel::delete(gateway::dsl::gateway.find(&gateway_id)).execute(&mut c)?; if ra == 0 {
if ra == 0 { return Err(Error::NotFound(gateway_id.to_string()));
return Err(Error::NotFound(gateway_id.to_string())); }
}
Ok(())
}
})
.await??;
info!( info!(
gateway_id = %gateway_id, gateway_id = %gateway_id,
"Gateway deleted" "Gateway deleted"
@ -301,34 +267,26 @@ pub async fn delete(gateway_id: &EUI64) -> Result<(), Error> {
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = gateway::dsl::gateway
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .distinct()
let mut q = gateway::dsl::gateway .left_join(multicast_group_gateway::table)
.select(dsl::count_star()) .into_boxed();
.distinct()
.left_join(multicast_group_gateway::table)
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(gateway::dsl::tenant_id.eq(tenant_id)); q = q.filter(gateway::dsl::tenant_id.eq(tenant_id));
} }
if let Some(multicast_group_id) = &filters.multicast_group_id { if let Some(multicast_group_id) = &filters.multicast_group_id {
q = q.filter( q = q.filter(multicast_group_gateway::dsl::multicast_group_id.eq(multicast_group_id));
multicast_group_gateway::dsl::multicast_group_id.eq(multicast_group_id), }
);
}
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(gateway::dsl::name.ilike(format!("%{}%", search))); q = q.filter(gateway::dsl::name.ilike(format!("%{}%", search)));
} }
Ok(q.first(&mut c)?) Ok(q.first(&mut c).await?)
}
})
.await?
} }
pub async fn list( pub async fn list(
@ -336,98 +294,80 @@ pub async fn list(
offset: i64, offset: i64,
filters: &Filters, filters: &Filters,
) -> Result<Vec<GatewayListItem>, Error> { ) -> Result<Vec<GatewayListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = gateway::dsl::gateway
move || -> Result<Vec<GatewayListItem>, Error> { .left_join(multicast_group_gateway::table)
let mut c = get_db_conn()?; .select((
let mut q = gateway::dsl::gateway gateway::tenant_id,
.left_join(multicast_group_gateway::table) gateway::gateway_id,
.select(( gateway::name,
gateway::tenant_id, gateway::description,
gateway::gateway_id, gateway::created_at,
gateway::name, gateway::updated_at,
gateway::description, gateway::last_seen_at,
gateway::created_at, gateway::latitude,
gateway::updated_at, gateway::longitude,
gateway::last_seen_at, gateway::altitude,
gateway::latitude, gateway::properties,
gateway::longitude, gateway::stats_interval_secs,
gateway::altitude, ))
gateway::properties, .distinct()
gateway::stats_interval_secs, .into_boxed();
))
.distinct()
.into_boxed();
if let Some(tenant_id) = &filters.tenant_id { if let Some(tenant_id) = &filters.tenant_id {
q = q.filter(gateway::dsl::tenant_id.eq(tenant_id)); q = q.filter(gateway::dsl::tenant_id.eq(tenant_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(gateway::dsl::name.ilike(format!("%{}%", search))); q = q.filter(gateway::dsl::name.ilike(format!("%{}%", search)));
} }
if let Some(multicast_group_id) = &filters.multicast_group_id { if let Some(multicast_group_id) = &filters.multicast_group_id {
q = q.filter( q = q.filter(multicast_group_gateway::dsl::multicast_group_id.eq(multicast_group_id));
multicast_group_gateway::dsl::multicast_group_id.eq(multicast_group_id), }
);
}
let items = q let items = q
.order_by(gateway::dsl::name) .order_by(gateway::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c)?; .load(&mut c)
Ok(items) .await?;
} Ok(items)
})
.await?
} }
pub async fn get_meta(gateway_id: &EUI64) -> Result<GatewayMeta, Error> { pub async fn get_meta(gateway_id: &EUI64) -> Result<GatewayMeta, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let gateway_id = *gateway_id; let meta = gateway::dsl::gateway
move || -> Result<GatewayMeta, Error> { .inner_join(tenant::table)
let mut c = get_db_conn()?; .select((
let meta = gateway::dsl::gateway gateway::gateway_id,
.inner_join(tenant::table) gateway::tenant_id,
.select(( gateway::latitude,
gateway::gateway_id, gateway::longitude,
gateway::tenant_id, gateway::altitude,
gateway::latitude, tenant::private_gateways_up,
gateway::longitude, tenant::private_gateways_down,
gateway::altitude, ))
tenant::private_gateways_up, .filter(gateway::dsl::gateway_id.eq(&gateway_id))
tenant::private_gateways_down, .first(&mut c)
)) .await
.filter(gateway::dsl::gateway_id.eq(&gateway_id)) .map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?;
.first(&mut c) Ok(meta)
.map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?;
Ok(meta)
}
})
.await?
} }
pub async fn get_counts_by_state(tenant_id: &Option<Uuid>) -> Result<GatewayCountsByState, Error> { pub async fn get_counts_by_state(tenant_id: &Option<Uuid>) -> Result<GatewayCountsByState, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let counts: GatewayCountsByState = diesel::sql_query(r#"
move || -> Result<GatewayCountsByState, Error> { select
let mut c = get_db_conn()?; coalesce(sum(case when last_seen_at is null then 1 end), 0) as never_seen_count,
let counts: GatewayCountsByState = diesel::sql_query(r#" coalesce(sum(case when (now() - make_interval(secs => stats_interval_secs * 2)) > last_seen_at then 1 end), 0) as offline_count,
select coalesce(sum(case when (now() - make_interval(secs => stats_interval_secs * 2)) <= last_seen_at then 1 end), 0) as online_count
coalesce(sum(case when last_seen_at is null then 1 end), 0) as never_seen_count, from
coalesce(sum(case when (now() - make_interval(secs => stats_interval_secs * 2)) > last_seen_at then 1 end), 0) as offline_count, gateway
coalesce(sum(case when (now() - make_interval(secs => stats_interval_secs * 2)) <= last_seen_at then 1 end), 0) as online_count where
from $1 is null or tenant_id = $1
gateway "#).bind::<diesel::sql_types::Nullable<diesel::sql_types::Uuid>, _>(tenant_id).get_result(&mut c).await?;
where Ok(counts)
$1 is null or tenant_id = $1
"#).bind::<diesel::sql_types::Nullable<diesel::sql_types::Uuid>, _>(tenant_id).get_result(&mut c)?;
Ok(counts)
}
}).await?
} }
#[cfg(test)] #[cfg(test)]

28
chirpstack/src/storage/helpers.rs
View File

@ -1,28 +1,24 @@
use diesel::prelude::*; use diesel::prelude::*;
use tokio::task; use diesel_async::RunQueryDsl;
use super::schema::{application, device, device_profile, tenant}; use super::schema::{application, device, device_profile, tenant};
use super::{ use super::{
application::Application, device::Device, device_profile::DeviceProfile, tenant::Tenant, application::Application, device::Device, device_profile::DeviceProfile, tenant::Tenant,
}; };
use super::{error::Error, get_db_conn}; use super::{error::Error, get_async_db_conn};
use lrwn::EUI64; use lrwn::EUI64;
pub async fn get_all_device_data( pub async fn get_all_device_data(
dev_eui: EUI64, dev_eui: EUI64,
) -> Result<(Device, Application, Tenant, DeviceProfile), Error> { ) -> Result<(Device, Application, Tenant, DeviceProfile), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<(Device, Application, Tenant, DeviceProfile), Error> { let res = device::table
let mut c = get_db_conn()?; .inner_join(application::table)
let res = device::table .inner_join(tenant::table.on(application::dsl::tenant_id.eq(tenant::dsl::id)))
.inner_join(application::table) .inner_join(device_profile::table)
.inner_join(tenant::table.on(application::dsl::tenant_id.eq(tenant::dsl::id))) .filter(device::dsl::dev_eui.eq(&dev_eui))
.inner_join(device_profile::table) .first::<(Device, Application, Tenant, DeviceProfile)>(&mut c)
.filter(device::dsl::dev_eui.eq(&dev_eui)) .await
.first::<(Device, Application, Tenant, DeviceProfile)>(&mut c) .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?; Ok(res)
Ok(res)
}
})
.await?
} }

151
chirpstack/src/storage/mod.rs
View File

@ -1,12 +1,21 @@
use std::fs::File;
use std::io::BufReader;
use std::ops::{Deref, DerefMut}; use std::ops::{Deref, DerefMut};
use std::sync::RwLock; use std::sync::RwLock;
use anyhow::Context; use anyhow::Context;
use anyhow::Result; use anyhow::Result;
use diesel::pg::PgConnection; use diesel::{ConnectionError, ConnectionResult};
use diesel::r2d2::{ConnectionManager, Pool, PooledConnection}; use diesel_async::async_connection_wrapper::AsyncConnectionWrapper;
use diesel_async::pooled_connection::deadpool::{Object as DeadpoolObject, Pool as DeadpoolPool};
use diesel_async::pooled_connection::{AsyncDieselConnectionManager, ManagerConfig};
use diesel_async::AsyncPgConnection;
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
use tracing::info; use futures_util::future::BoxFuture;
use futures_util::FutureExt;
use r2d2::{Pool, PooledConnection};
use tokio::task;
use tracing::{error, info};
use crate::config; use crate::config;
@ -34,11 +43,11 @@ pub mod search;
pub mod tenant; pub mod tenant;
pub mod user; pub mod user;
pub type PgPool = Pool<ConnectionManager<PgConnection>>; pub type AsyncPgPool = DeadpoolPool<AsyncPgConnection>;
pub type PgPoolConnection = PooledConnection<ConnectionManager<PgConnection>>; pub type AsyncPgPoolConnection = DeadpoolObject<AsyncPgConnection>;
lazy_static! { lazy_static! {
static ref PG_POOL: RwLock<Option<PgPool>> = RwLock::new(None); static ref ASYNC_PG_POOL: RwLock<Option<AsyncPgPool>> = RwLock::new(None);
static ref REDIS_POOL: RwLock<Option<RedisPool>> = RwLock::new(None); static ref REDIS_POOL: RwLock<Option<RedisPool>> = RwLock::new(None);
static ref REDIS_PREFIX: RwLock<String> = RwLock::new("".to_string()); static ref REDIS_PREFIX: RwLock<String> = RwLock::new("".to_string());
} }
@ -170,21 +179,18 @@ pub async fn setup() -> Result<()> {
let conf = config::get(); let conf = config::get();
info!("Setting up PostgreSQL connection pool"); info!("Setting up PostgreSQL connection pool");
let pg_pool = PgPool::builder() let mut config = ManagerConfig::default();
.max_size(conf.postgresql.max_open_connections) config.custom_setup = Box::new(pg_establish_connection);
.min_idle(match conf.postgresql.min_idle_connections {
0 => None,
_ => Some(conf.postgresql.min_idle_connections),
})
.build(ConnectionManager::new(&conf.postgresql.dsn))
.context("Setup PostgreSQL connection pool error")?;
set_db_pool(pg_pool);
let mut pg_conn = get_db_conn()?;
info!("Applying schema migrations"); let mgr = AsyncDieselConnectionManager::<AsyncPgConnection>::new_with_config(
pg_conn &conf.postgresql.dsn,
.run_pending_migrations(MIGRATIONS) config,
.map_err(|e| anyhow!("{}", e))?; );
let pool = DeadpoolPool::builder(mgr)
.max_size(conf.postgresql.max_open_connections as usize)
.build()?;
set_async_db_pool(pool);
run_db_migrations().await?;
info!("Setting up Redis client"); info!("Setting up Redis client");
if conf.redis.cluster { if conf.redis.cluster {
@ -221,18 +227,66 @@ pub async fn setup() -> Result<()> {
Ok(()) Ok(())
} }
pub fn get_db_pool() -> Result<PgPool> { // Source:
let pool_r = PG_POOL.read().unwrap(); // https://github.com/weiznich/diesel_async/blob/main/examples/postgres/pooled-with-rustls/src/main.rs
let pool = pool_r fn pg_establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConnection>> {
let fut = async {
let root_certs =
pg_root_certs().map_err(|e| ConnectionError::BadConnection(e.to_string()))?;
let rustls_config = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_certs)
.with_no_client_auth();
let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config);
let (client, conn) = tokio_postgres::connect(config, tls)
.await
.map_err(|e| ConnectionError::BadConnection(e.to_string()))?;
tokio::spawn(async move {
if let Err(e) = conn.await {
error!(error = %e, "PostgreSQL connection error");
}
});
AsyncPgConnection::try_from(client).await
};
fut.boxed()
}
fn pg_root_certs() -> Result<rustls::RootCertStore> {
let conf = config::get();
let mut roots = rustls::RootCertStore::empty();
let certs = rustls_native_certs::load_native_certs()?;
let certs: Vec<_> = certs.into_iter().map(|cert| cert.0).collect();
roots.add_parsable_certificates(&certs);
if !conf.postgresql.ca_cert.is_empty() {
let f = File::open(&conf.postgresql.ca_cert).context("Open ca certificate")?;
let mut reader = BufReader::new(f);
let certs = rustls_pemfile::certs(&mut reader)?;
for cert in certs
.into_iter()
.map(rustls::Certificate)
.collect::<Vec<_>>()
{
roots.add(&cert)?;
}
}
Ok(roots)
}
pub fn get_async_db_pool() -> Result<AsyncPgPool> {
let pool_r = ASYNC_PG_POOL.read().unwrap();
let pool: AsyncPgPool = pool_r
.as_ref() .as_ref()
.ok_or_else(|| anyhow!("PostgreSQL connection pool is not initialized (yet)"))? .ok_or_else(|| anyhow!("PostgreSQL connection pool is not initialized"))?
.clone(); .clone();
Ok(pool) Ok(pool)
} }
pub fn get_db_conn() -> Result<PgPoolConnection> { pub async fn get_async_db_conn() -> Result<AsyncPgPoolConnection> {
let pool = get_db_pool()?; let pool = get_async_db_pool()?;
Ok(pool.get()?) Ok(pool.get().await?)
} }
pub fn get_redis_conn() -> Result<RedisPoolConnection> { pub fn get_redis_conn() -> Result<RedisPoolConnection> {
@ -246,11 +300,28 @@ pub fn get_redis_conn() -> Result<RedisPoolConnection> {
}) })
} }
pub fn set_db_pool(p: PgPool) { pub fn set_async_db_pool(p: AsyncPgPool) {
let mut pool_w = PG_POOL.write().unwrap(); let mut pool_w = ASYNC_PG_POOL.write().unwrap();
*pool_w = Some(p); *pool_w = Some(p);
} }
pub async fn run_db_migrations() -> Result<()> {
info!("Applying schema migrations");
let c = get_async_db_conn().await?;
let mut c_wrapped: AsyncConnectionWrapper<AsyncPgPoolConnection> =
AsyncConnectionWrapper::from(c);
task::spawn_blocking(move || -> Result<()> {
c_wrapped
.run_pending_migrations(MIGRATIONS)
.map_err(|e| anyhow!("{}", e))?;
Ok(())
})
.await?
}
pub fn set_redis_pool(p: RedisPool) { pub fn set_redis_pool(p: RedisPool) {
let mut pool_w = REDIS_POOL.write().unwrap(); let mut pool_w = REDIS_POOL.write().unwrap();
*pool_w = Some(p); *pool_w = Some(p);
@ -262,14 +333,22 @@ pub fn redis_key(s: String) -> String {
} }
#[cfg(test)] #[cfg(test)]
pub fn reset_db() -> Result<()> { pub async fn reset_db() -> Result<()> {
let mut conn = get_db_conn()?; let c = get_async_db_conn().await?;
conn.revert_all_migrations(MIGRATIONS) let mut c_wrapped: AsyncConnectionWrapper<AsyncPgPoolConnection> =
.map_err(|e| anyhow!("{}", e))?; AsyncConnectionWrapper::from(c);
conn.run_pending_migrations(MIGRATIONS)
.map_err(|e| anyhow!("{}", e))?;
Ok(()) tokio::task::spawn_blocking(move || -> Result<()> {
c_wrapped
.revert_all_migrations(MIGRATIONS)
.map_err(|e| anyhow!("{}", e))?;
c_wrapped
.run_pending_migrations(MIGRATIONS)
.map_err(|e| anyhow!("{}", e))?;
Ok(())
})
.await?
} }
#[cfg(test)] #[cfg(test)]

457
chirpstack/src/storage/multicast.rs
View File

@ -1,8 +1,7 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
@ -14,7 +13,7 @@ use super::schema::{
application, device, gateway, multicast_group, multicast_group_device, multicast_group_gateway, application, device, gateway, multicast_group, multicast_group_device, multicast_group_gateway,
multicast_group_queue_item, multicast_group_queue_item,
}; };
use super::{fields, get_db_conn}; use super::{fields, get_async_db_conn};
use crate::downlink::classb; use crate::downlink::classb;
use crate::{config, gpstime::ToDateTime, gpstime::ToGpsTime}; use crate::{config, gpstime::ToDateTime, gpstime::ToGpsTime};
@ -133,104 +132,79 @@ impl Default for MulticastGroupQueueItem {
pub async fn create(mg: MulticastGroup) -> Result<MulticastGroup, Error> { pub async fn create(mg: MulticastGroup) -> Result<MulticastGroup, Error> {
mg.validate()?; mg.validate()?;
let mg = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<MulticastGroup, Error> { let mg: MulticastGroup = diesel::insert_into(multicast_group::table)
let mut c = get_db_conn()?; .values(&mg)
diesel::insert_into(multicast_group::table) .get_result(&mut c)
.values(&mg) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, mg.id.to_string()))?;
.map_err(|e| Error::from_diesel(e, mg.id.to_string()))
}
})
.await??;
info!(id = %mg.id, "Multicast-group created"); info!(id = %mg.id, "Multicast-group created");
Ok(mg) Ok(mg)
} }
pub async fn get(id: &Uuid) -> Result<MulticastGroup, Error> { pub async fn get(id: &Uuid) -> Result<MulticastGroup, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; multicast_group::dsl::multicast_group
move || -> Result<MulticastGroup, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
multicast_group::dsl::multicast_group .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))
.first(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))
}
})
.await?
} }
pub async fn update(mg: MulticastGroup) -> Result<MulticastGroup, Error> { pub async fn update(mg: MulticastGroup) -> Result<MulticastGroup, Error> {
mg.validate()?; mg.validate()?;
let mg = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<MulticastGroup, Error> { let mg: MulticastGroup = diesel::update(multicast_group::dsl::multicast_group.find(&mg.id))
let mut c = get_db_conn()?; .set((
multicast_group::updated_at.eq(Utc::now()),
diesel::update(multicast_group::dsl::multicast_group.find(&mg.id)) multicast_group::name.eq(&mg.name),
.set(( multicast_group::region.eq(&mg.region),
multicast_group::updated_at.eq(Utc::now()), multicast_group::mc_addr.eq(&mg.mc_addr),
multicast_group::name.eq(&mg.name), multicast_group::mc_nwk_s_key.eq(&mg.mc_nwk_s_key),
multicast_group::region.eq(&mg.region), multicast_group::mc_app_s_key.eq(&mg.mc_app_s_key),
multicast_group::mc_addr.eq(&mg.mc_addr), multicast_group::f_cnt.eq(&mg.f_cnt),
multicast_group::mc_nwk_s_key.eq(&mg.mc_nwk_s_key), multicast_group::group_type.eq(&mg.group_type),
multicast_group::mc_app_s_key.eq(&mg.mc_app_s_key), multicast_group::dr.eq(&mg.dr),
multicast_group::f_cnt.eq(&mg.f_cnt), multicast_group::frequency.eq(&mg.frequency),
multicast_group::group_type.eq(&mg.group_type), multicast_group::class_b_ping_slot_period.eq(&mg.class_b_ping_slot_period),
multicast_group::dr.eq(&mg.dr), multicast_group::class_c_scheduling_type.eq(&mg.class_c_scheduling_type),
multicast_group::frequency.eq(&mg.frequency), ))
multicast_group::class_b_ping_slot_period.eq(&mg.class_b_ping_slot_period), .get_result(&mut c)
multicast_group::class_c_scheduling_type.eq(&mg.class_c_scheduling_type), .await
)) .map_err(|e| Error::from_diesel(e, mg.id.to_string()))?;
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, mg.id.to_string()))
}
})
.await??;
info!(id = %mg.id, "Multicast-group updated"); info!(id = %mg.id, "Multicast-group updated");
Ok(mg) Ok(mg)
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(multicast_group::dsl::multicast_group.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = if ra == 0 {
diesel::delete(multicast_group::dsl::multicast_group.find(&id)).execute(&mut c)?; return Err(Error::NotFound(id.to_string()));
if ra == 0 { }
return Err(Error::NotFound(id.to_string()));
}
Ok(())
}
})
.await??;
info!(id = %id, "Multicast-group deleted"); info!(id = %id, "Multicast-group deleted");
Ok(()) Ok(())
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = multicast_group::dsl::multicast_group
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .into_boxed();
let mut q = multicast_group::dsl::multicast_group
.select(dsl::count_star())
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(multicast_group::dsl::application_id.eq(application_id)); q = q.filter(multicast_group::dsl::application_id.eq(application_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(multicast_group::dsl::name.ilike(format!("%{}%", search))); q = q.filter(multicast_group::dsl::name.ilike(format!("%{}%", search)));
} }
q.first(&mut c) q.first(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn list( pub async fn list(
@ -238,56 +212,51 @@ pub async fn list(
offset: i64, offset: i64,
filters: &Filters, filters: &Filters,
) -> Result<Vec<MulticastGroupListItem>, Error> { ) -> Result<Vec<MulticastGroupListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = multicast_group::dsl::multicast_group
move || -> Result<Vec<MulticastGroupListItem>, Error> { .select((
let mut c = get_db_conn()?; multicast_group::id,
let mut q = multicast_group::dsl::multicast_group multicast_group::created_at,
.select(( multicast_group::updated_at,
multicast_group::id, multicast_group::name,
multicast_group::created_at, multicast_group::region,
multicast_group::updated_at, multicast_group::group_type,
multicast_group::name, ))
multicast_group::region, .into_boxed();
multicast_group::group_type,
))
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(multicast_group::dsl::application_id.eq(application_id)); q = q.filter(multicast_group::dsl::application_id.eq(application_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(multicast_group::dsl::name.ilike(format!("%{}%", search))); q = q.filter(multicast_group::dsl::name.ilike(format!("%{}%", search)));
} }
q.order_by(multicast_group::dsl::name) q.order_by(multicast_group::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c) .load(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn add_device(group_id: &Uuid, dev_eui: &EUI64) -> Result<(), Error> { pub async fn add_device(group_id: &Uuid, dev_eui: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; c.build_transaction()
let dev_eui = *dev_eui; .run::<(), Error, _>(|c| {
move || -> Result<(), Error> { Box::pin(async move {
let mut c = get_db_conn()?;
c.transaction::<(), Error, _>(|c| {
let d: super::device::Device = device::dsl::device let d: super::device::Device = device::dsl::device
.find(&dev_eui) .find(&dev_eui)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?; .map_err(|e| Error::from_diesel(e, dev_eui.to_string()))?;
let mg: MulticastGroup = multicast_group::dsl::multicast_group let mg: MulticastGroup = multicast_group::dsl::multicast_group
.find(&group_id) .find(&group_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, group_id.to_string()))?; .map_err(|e| Error::from_diesel(e, group_id.to_string()))?;
if d.application_id != mg.application_id { if d.application_id != mg.application_id {
@ -302,65 +271,59 @@ pub async fn add_device(group_id: &Uuid, dev_eui: &EUI64) -> Result<(), Error> {
multicast_group_device::created_at.eq(Utc::now()), multicast_group_device::created_at.eq(Utc::now()),
)) ))
.execute(c) .execute(c)
.await
.map_err(|e| Error::from_diesel(e, "".into()))?; .map_err(|e| Error::from_diesel(e, "".into()))?;
Ok(()) Ok(())
}) })
} })
}) .await?;
.await??;
info!(multicast_group_id = %group_id, dev_eui = %dev_eui, "Device added to multicast-group"); info!(multicast_group_id = %group_id, dev_eui = %dev_eui, "Device added to multicast-group");
Ok(()) Ok(())
} }
pub async fn remove_device(group_id: &Uuid, dev_eui: &EUI64) -> Result<(), Error> { pub async fn remove_device(group_id: &Uuid, dev_eui: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; let ra = diesel::delete(
let dev_eui = *dev_eui; multicast_group_device::dsl::multicast_group_device
move || -> Result<(), Error> { .filter(multicast_group_device::multicast_group_id.eq(&group_id))
let mut c = get_db_conn()?; .filter(multicast_group_device::dev_eui.eq(&dev_eui)),
let ra = diesel::delete( )
multicast_group_device::dsl::multicast_group_device .execute(&mut c)
.filter(multicast_group_device::multicast_group_id.eq(&group_id)) .await?;
.filter(multicast_group_device::dev_eui.eq(&dev_eui)), if ra == 0 {
) return Err(Error::NotFound(format!(
.execute(&mut c)?; "multicast-group: {}, device: {}",
if ra == 0 { group_id, dev_eui
return Err(Error::NotFound(format!( )));
"multicast-group: {}, device: {}", }
group_id, dev_eui
)));
}
Ok(())
}
})
.await??;
info!(multicast_group_id = %group_id, dev_eui = %dev_eui, "Device removed from multicast-group"); info!(multicast_group_id = %group_id, dev_eui = %dev_eui, "Device removed from multicast-group");
Ok(()) Ok(())
} }
pub async fn add_gateway(group_id: &Uuid, gateway_id: &EUI64) -> Result<(), Error> { pub async fn add_gateway(group_id: &Uuid, gateway_id: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; c.build_transaction()
let gateway_id = *gateway_id; .run::<(), Error, _>(|c| {
move || -> Result<(), Error> { Box::pin(async move {
let mut c = get_db_conn()?;
c.transaction::<(), Error, _>(|c| {
let gw: super::gateway::Gateway = gateway::dsl::gateway let gw: super::gateway::Gateway = gateway::dsl::gateway
.find(&gateway_id) .find(&gateway_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?; .map_err(|e| Error::from_diesel(e, gateway_id.to_string()))?;
let mg: MulticastGroup = multicast_group::dsl::multicast_group let mg: MulticastGroup = multicast_group::dsl::multicast_group
.find(&group_id) .find(&group_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, group_id.to_string()))?; .map_err(|e| Error::from_diesel(e, group_id.to_string()))?;
let a: super::application::Application = application::dsl::application let a: super::application::Application = application::dsl::application
.find(&mg.application_id) .find(&mg.application_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, mg.application_id.to_string()))?; .map_err(|e| Error::from_diesel(e, mg.application_id.to_string()))?;
if a.tenant_id != gw.tenant_id { if a.tenant_id != gw.tenant_id {
@ -375,70 +338,53 @@ pub async fn add_gateway(group_id: &Uuid, gateway_id: &EUI64) -> Result<(), Erro
multicast_group_gateway::created_at.eq(Utc::now()), multicast_group_gateway::created_at.eq(Utc::now()),
)) ))
.execute(c) .execute(c)
.await
.map_err(|e| Error::from_diesel(e, "".into()))?; .map_err(|e| Error::from_diesel(e, "".into()))?;
Ok(()) Ok(())
}) })
} })
}) .await?;
.await??;
info!(multicast_group_id = %group_id, gateway_id = %gateway_id, "Gateway added to multicast-group"); info!(multicast_group_id = %group_id, gateway_id = %gateway_id, "Gateway added to multicast-group");
Ok(()) Ok(())
} }
pub async fn remove_gateway(group_id: &Uuid, gateway_id: &EUI64) -> Result<(), Error> { pub async fn remove_gateway(group_id: &Uuid, gateway_id: &EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; let ra = diesel::delete(
let gateway_id = *gateway_id; multicast_group_gateway::dsl::multicast_group_gateway
move || -> Result<(), Error> { .filter(multicast_group_gateway::multicast_group_id.eq(&group_id))
let mut c = get_db_conn()?; .filter(multicast_group_gateway::gateway_id.eq(&gateway_id)),
let ra = diesel::delete( )
multicast_group_gateway::dsl::multicast_group_gateway .execute(&mut c)
.filter(multicast_group_gateway::multicast_group_id.eq(&group_id)) .await?;
.filter(multicast_group_gateway::gateway_id.eq(&gateway_id)), if ra == 0 {
) return Err(Error::NotFound(format!(
.execute(&mut c)?; "multicast-group: {}, gateway: {}",
if ra == 0 { group_id, gateway_id
return Err(Error::NotFound(format!( )));
"multicast-group: {}, gateway: {}", }
group_id, gateway_id
)));
}
Ok(())
}
})
.await??;
info!(multicast_group_id = %group_id, gateway_id = %gateway_id, "Gateway removed from multicast-group"); info!(multicast_group_id = %group_id, gateway_id = %gateway_id, "Gateway removed from multicast-group");
Ok(()) Ok(())
} }
pub async fn get_dev_euis(group_id: &Uuid) -> Result<Vec<EUI64>, Error> { pub async fn get_dev_euis(group_id: &Uuid) -> Result<Vec<EUI64>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; multicast_group_device::dsl::multicast_group_device
move || -> Result<Vec<EUI64>, Error> { .select(multicast_group_device::dev_eui)
let mut c = get_db_conn()?; .filter(multicast_group_device::dsl::multicast_group_id.eq(&group_id))
multicast_group_device::dsl::multicast_group_device .load(&mut c)
.select(multicast_group_device::dev_eui) .await
.filter(multicast_group_device::dsl::multicast_group_id.eq(&group_id)) .map_err(|e| Error::from_diesel(e, group_id.to_string()))
.load(&mut c)
.map_err(|e| Error::from_diesel(e, group_id.to_string()))
}
})
.await?
} }
pub async fn get_gateway_ids(group_id: &Uuid) -> Result<Vec<EUI64>, Error> { pub async fn get_gateway_ids(group_id: &Uuid) -> Result<Vec<EUI64>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let group_id = *group_id; multicast_group_gateway::dsl::multicast_group_gateway
move || -> Result<Vec<EUI64>, Error> { .select(multicast_group_gateway::gateway_id)
let mut c = get_db_conn()?; .filter(multicast_group_gateway::dsl::multicast_group_id.eq(&group_id))
multicast_group_gateway::dsl::multicast_group_gateway .load(&mut c)
.select(multicast_group_gateway::gateway_id) .await
.filter(multicast_group_gateway::dsl::multicast_group_id.eq(&group_id)) .map_err(|e| Error::from_diesel(e, group_id.to_string()))
.load(&mut c)
.map_err(|e| Error::from_diesel(e, group_id.to_string()))
}
})
.await?
} }
// This enqueues a multicast-group queue item for the given gateways and returns the frame-counter // This enqueues a multicast-group queue item for the given gateways and returns the frame-counter
@ -451,17 +397,18 @@ pub async fn enqueue(
) -> Result<(Vec<Uuid>, u32), Error> { ) -> Result<(Vec<Uuid>, u32), Error> {
qi.validate()?; qi.validate()?;
let (ids, f_cnt) = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let gateway_ids = gateway_ids.to_vec(); let conf = config::get();
move || -> Result<(Vec<Uuid>, u32), Error> { let (ids, f_cnt) = c
let mut c = get_db_conn()?; .build_transaction()
let conf = config::get(); .run::<(Vec<Uuid>, u32), Error, _>(|c| {
c.transaction::<(Vec<Uuid>, u32), Error, _>(|c| { Box::pin(async move {
let mut ids: Vec<Uuid> = Vec::new(); let mut ids: Vec<Uuid> = Vec::new();
let mg: MulticastGroup = multicast_group::dsl::multicast_group let mg: MulticastGroup = multicast_group::dsl::multicast_group
.find(&qi.multicast_group_id) .find(&qi.multicast_group_id)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, qi.multicast_group_id.to_string()))?; .map_err(|e| Error::from_diesel(e, qi.multicast_group_id.to_string()))?;
match mg.group_type.as_ref() { match mg.group_type.as_ref() {
@ -483,7 +430,8 @@ pub async fn enqueue(
multicast_group_queue_item::dsl::multicast_group_id multicast_group_queue_item::dsl::multicast_group_id
.eq(&qi.multicast_group_id), .eq(&qi.multicast_group_id),
) )
.first(c)?; .first(c)
.await?;
// Get timestamp after which we must generate the next ping-slot. // Get timestamp after which we must generate the next ping-slot.
let ping_slot_after_gps_time = match res { let ping_slot_after_gps_time = match res {
@ -505,7 +453,7 @@ pub async fn enqueue(
let scheduler_run_after_ts = emit_at_time_since_gps_epoch.to_date_time() let scheduler_run_after_ts = emit_at_time_since_gps_epoch.to_date_time()
- Duration::from_std(2 * conf.network.scheduler.interval).unwrap(); - Duration::from_std(2 * conf.network.scheduler.interval).unwrap();
for gateway_id in &gateway_ids { for gateway_id in gateway_ids {
let qi = MulticastGroupQueueItem { let qi = MulticastGroupQueueItem {
scheduler_run_after: scheduler_run_after_ts, scheduler_run_after: scheduler_run_after_ts,
multicast_group_id: mg.id, multicast_group_id: mg.id,
@ -523,6 +471,7 @@ pub async fn enqueue(
diesel::insert_into(multicast_group_queue_item::table) diesel::insert_into(multicast_group_queue_item::table)
.values(&qi) .values(&qi)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, mg.id.to_string()))?; .map_err(|e| Error::from_diesel(e, mg.id.to_string()))?;
ids.push(qi.id); ids.push(qi.id);
} }
@ -538,7 +487,8 @@ pub async fn enqueue(
multicast_group_queue_item::dsl::multicast_group_id multicast_group_queue_item::dsl::multicast_group_id
.eq(&qi.multicast_group_id), .eq(&qi.multicast_group_id),
) )
.first(c)?; .first(c)
.await?;
let mut scheduler_run_after_ts = match res { let mut scheduler_run_after_ts = match res {
Some(v) => { Some(v) => {
@ -563,7 +513,7 @@ pub async fn enqueue(
None None
}; };
for gateway_id in &gateway_ids { for gateway_id in gateway_ids {
let qi = MulticastGroupQueueItem { let qi = MulticastGroupQueueItem {
scheduler_run_after: scheduler_run_after_ts, scheduler_run_after: scheduler_run_after_ts,
multicast_group_id: mg.id, multicast_group_id: mg.id,
@ -579,6 +529,7 @@ pub async fn enqueue(
diesel::insert_into(multicast_group_queue_item::table) diesel::insert_into(multicast_group_queue_item::table)
.values(&qi) .values(&qi)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, mg.id.to_string()))?; .map_err(|e| Error::from_diesel(e, mg.id.to_string()))?;
ids.push(qi.id); ids.push(qi.id);
@ -604,77 +555,58 @@ pub async fn enqueue(
diesel::update(multicast_group::dsl::multicast_group.find(&qi.multicast_group_id)) diesel::update(multicast_group::dsl::multicast_group.find(&qi.multicast_group_id))
.set(multicast_group::f_cnt.eq(mg.f_cnt + 1)) .set(multicast_group::f_cnt.eq(mg.f_cnt + 1))
.execute(c) .execute(c)
.await
.map_err(|e| Error::from_diesel(e, qi.multicast_group_id.to_string()))?; .map_err(|e| Error::from_diesel(e, qi.multicast_group_id.to_string()))?;
// Return value before it was incremented // Return value before it was incremented
Ok((ids, mg.f_cnt as u32)) Ok((ids, mg.f_cnt as u32))
}) })
} })
}) .await?;
.await??;
info!(multicast_group_id = %qi.multicast_group_id, f_cnt = f_cnt, "Multicast-group queue item created"); info!(multicast_group_id = %qi.multicast_group_id, f_cnt = f_cnt, "Multicast-group queue item created");
Ok((ids, f_cnt)) Ok((ids, f_cnt))
} }
pub async fn delete_queue_item(id: &Uuid) -> Result<(), Error> { pub async fn delete_queue_item(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(multicast_group_queue_item::dsl::multicast_group_queue_item.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await?;
let ra = diesel::delete( if ra == 0 {
multicast_group_queue_item::dsl::multicast_group_queue_item.find(&id), return Err(Error::NotFound(id.to_string()));
) }
.execute(&mut c)?;
if ra == 0 {
return Err(Error::NotFound(id.to_string()));
}
Ok(())
}
})
.await??;
info!(id = %id, "Multicast-group queue item deleted"); info!(id = %id, "Multicast-group queue item deleted");
Ok(()) Ok(())
} }
pub async fn flush_queue(multicast_group_id: &Uuid) -> Result<(), Error> { pub async fn flush_queue(multicast_group_id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let multicast_group_id = *multicast_group_id; let _ = diesel::delete(
move || -> Result<(), Error> { multicast_group_queue_item::dsl::multicast_group_queue_item
let mut c = get_db_conn()?; .filter(multicast_group_queue_item::multicast_group_id.eq(&multicast_group_id)),
let _ = diesel::delete( )
multicast_group_queue_item::dsl::multicast_group_queue_item .execute(&mut c)
.filter(multicast_group_queue_item::multicast_group_id.eq(&multicast_group_id)), .await
) .map_err(|e| Error::from_diesel(e, multicast_group_id.to_string()))?;
.execute(&mut c)
.map_err(|e| Error::from_diesel(e, multicast_group_id.to_string()))?;
Ok(())
}
})
.await??;
info!(multicast_group_id = %multicast_group_id, "Multicast-group queue flushed"); info!(multicast_group_id = %multicast_group_id, "Multicast-group queue flushed");
Ok(()) Ok(())
} }
pub async fn get_queue(multicast_group_id: &Uuid) -> Result<Vec<MulticastGroupQueueItem>, Error> { pub async fn get_queue(multicast_group_id: &Uuid) -> Result<Vec<MulticastGroupQueueItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let multicast_group_id = *multicast_group_id; multicast_group_queue_item::dsl::multicast_group_queue_item
move || -> Result<Vec<MulticastGroupQueueItem>, Error> { .filter(multicast_group_queue_item::dsl::multicast_group_id.eq(&multicast_group_id))
let mut c = get_db_conn()?; .order_by(multicast_group_queue_item::created_at)
multicast_group_queue_item::dsl::multicast_group_queue_item .load(&mut c)
.filter(multicast_group_queue_item::dsl::multicast_group_id.eq(&multicast_group_id)) .await
.order_by(multicast_group_queue_item::created_at) .map_err(|e| Error::from_diesel(e, multicast_group_id.to_string()))
.load(&mut c)
.map_err(|e| Error::from_diesel(e, multicast_group_id.to_string()))
}
})
.await?
} }
pub async fn get_schedulable_queue_items(limit: usize) -> Result<Vec<MulticastGroupQueueItem>> { pub async fn get_schedulable_queue_items(limit: usize) -> Result<Vec<MulticastGroupQueueItem>> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Vec<MulticastGroupQueueItem>> { c.build_transaction()
let mut c = get_db_conn()?; .run::<Vec<MulticastGroupQueueItem>, Error, _>(|c| {
c.transaction::<Vec<MulticastGroupQueueItem>, Error, _>(|c| { Box::pin(async move {
let conf = config::get(); let conf = config::get();
diesel::sql_query( diesel::sql_query(
r#" r#"
@ -703,12 +635,12 @@ pub async fn get_schedulable_queue_items(limit: usize) -> Result<Vec<MulticastGr
Utc::now() + Duration::from_std(2 * conf.network.scheduler.interval).unwrap(), Utc::now() + Duration::from_std(2 * conf.network.scheduler.interval).unwrap(),
) )
.load(c) .load(c)
.await
.map_err(|e| Error::from_diesel(e, "".into())) .map_err(|e| Error::from_diesel(e, "".into()))
}) })
.context("Get schedulable multicast-group queue items") })
} .await
}) .context("Get schedulable multicast-group queue items")
.await?
} }
#[cfg(test)] #[cfg(test)]
@ -718,17 +650,12 @@ pub mod test {
use crate::test; use crate::test;
pub async fn get_queue_item(id: &Uuid) -> Result<MulticastGroupQueueItem, Error> { pub async fn get_queue_item(id: &Uuid) -> Result<MulticastGroupQueueItem, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; multicast_group_queue_item::dsl::multicast_group_queue_item
move || -> Result<MulticastGroupQueueItem, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
multicast_group_queue_item::dsl::multicast_group_queue_item .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))
.first(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))
}
})
.await?
} }
struct FilterTest<'a> { struct FilterTest<'a> {

201
chirpstack/src/storage/relay.rs
View File

@ -1,15 +1,14 @@
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use lrwn::{DevAddr, EUI64}; use lrwn::{DevAddr, EUI64};
use super::schema::{device, device_profile, relay_device}; use super::schema::{device, device_profile, relay_device};
use super::{device::Device, error::Error, get_db_conn}; use super::{device::Device, error::Error, get_async_db_conn};
// This is set to 15, because the FilterList must contain a "catch-all" record to filter all // This is set to 15, because the FilterList must contain a "catch-all" record to filter all
// uplinks that do not match the remaining records. This means that we can use 16 - 1 FilterList // uplinks that do not match the remaining records. This means that we can use 16 - 1 FilterList
@ -44,24 +43,18 @@ pub struct DeviceListItem {
} }
pub async fn get_relay_count(filters: &RelayFilters) -> Result<i64, Error> { pub async fn get_relay_count(filters: &RelayFilters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device::dsl::device
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .inner_join(device_profile::table)
let mut q = device::dsl::device .filter(device_profile::dsl::is_relay.eq(true))
.select(dsl::count_star()) .into_boxed();
.inner_join(device_profile::table)
.filter(device_profile::dsl::is_relay.eq(true))
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(device::dsl::application_id.eq(application_id)); q = q.filter(device::dsl::application_id.eq(application_id));
} }
Ok(q.first(&mut c)?) Ok(q.first(&mut c).await?)
}
})
.await?
} }
pub async fn list_relays( pub async fn list_relays(
@ -69,48 +62,38 @@ pub async fn list_relays(
offset: i64, offset: i64,
filters: &RelayFilters, filters: &RelayFilters,
) -> Result<Vec<RelayListItem>, Error> { ) -> Result<Vec<RelayListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = device::dsl::device
move || -> Result<Vec<RelayListItem>, Error> { .inner_join(device_profile::table)
let mut c = get_db_conn()?; .select((device::dev_eui, device::name))
let mut q = device::dsl::device .filter(device_profile::dsl::is_relay.eq(true))
.inner_join(device_profile::table) .into_boxed();
.select((device::dev_eui, device::name))
.filter(device_profile::dsl::is_relay.eq(true))
.into_boxed();
if let Some(application_id) = &filters.application_id { if let Some(application_id) = &filters.application_id {
q = q.filter(device::dsl::application_id.eq(application_id)); q = q.filter(device::dsl::application_id.eq(application_id));
} }
q.order_by(device::dsl::name) q.order_by(device::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c) .load(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn get_device_count(filters: &DeviceFilters) -> Result<i64, Error> { pub async fn get_device_count(filters: &DeviceFilters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = relay_device::dsl::relay_device
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .into_boxed();
let mut q = relay_device::dsl::relay_device
.select(dsl::count_star())
.into_boxed();
if let Some(relay_dev_eui) = &filters.relay_dev_eui { if let Some(relay_dev_eui) = &filters.relay_dev_eui {
q = q.filter(relay_device::dsl::relay_dev_eui.eq(relay_dev_eui)); q = q.filter(relay_device::dsl::relay_dev_eui.eq(relay_dev_eui));
} }
q.first(&mut c) q.first(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn list_devices( pub async fn list_devices(
@ -118,57 +101,53 @@ pub async fn list_devices(
offset: i64, offset: i64,
filters: &DeviceFilters, filters: &DeviceFilters,
) -> Result<Vec<DeviceListItem>, Error> { ) -> Result<Vec<DeviceListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = relay_device::dsl::relay_device
move || -> Result<Vec<DeviceListItem>, Error> { .inner_join(device::table.on(relay_device::dsl::dev_eui.eq(device::dsl::dev_eui)))
let mut c = get_db_conn()?; .inner_join(
let mut q = relay_device::dsl::relay_device device_profile::table.on(device::dsl::device_profile_id.eq(device_profile::dsl::id)),
.inner_join(device::table.on(relay_device::dsl::dev_eui.eq(device::dsl::dev_eui))) )
.inner_join( .select((
device_profile::table relay_device::dev_eui,
.on(device::dsl::device_profile_id.eq(device_profile::dsl::id)), device::join_eui,
) device::dev_addr,
.select(( relay_device::created_at,
relay_device::dev_eui, device::name,
device::join_eui, device_profile::relay_ed_uplink_limit_bucket_size,
device::dev_addr, device_profile::relay_ed_uplink_limit_reload_rate,
relay_device::created_at, ))
device::name, .into_boxed();
device_profile::relay_ed_uplink_limit_bucket_size,
device_profile::relay_ed_uplink_limit_reload_rate,
))
.into_boxed();
if let Some(relay_dev_eui) = &filters.relay_dev_eui { if let Some(relay_dev_eui) = &filters.relay_dev_eui {
q = q.filter(relay_device::dsl::relay_dev_eui.eq(relay_dev_eui)); q = q.filter(relay_device::dsl::relay_dev_eui.eq(relay_dev_eui));
} }
q.order_by(device::dsl::name) q.order_by(device::dsl::name)
.limit(limit) .limit(limit)
.offset(offset) .offset(offset)
.load(&mut c) .load(&mut c)
.map_err(|e| Error::from_diesel(e, "".into())) .await
} .map_err(|e| Error::from_diesel(e, "".into()))
})
.await?
} }
pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(), Error> { pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<(), Error> { c.build_transaction()
let mut c = get_db_conn()?; .run::<(), Error, _>(|c| {
c.transaction::<(), Error, _>(|c| { Box::pin(async move {
// We lock the relay device to avoid race-conditions in the validation. // We lock the relay device to avoid race-conditions in the validation.
let rd: Device = device::dsl::device let rd: Device = device::dsl::device
.find(&relay_dev_eui) .find(&relay_dev_eui)
.for_update() .for_update()
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, relay_dev_eui.to_string()))?; .map_err(|e| Error::from_diesel(e, relay_dev_eui.to_string()))?;
// Is the given relay_dev_eui a Relay? // Is the given relay_dev_eui a Relay?
let rdp: super::device_profile::DeviceProfile = device_profile::dsl::device_profile let rdp: super::device_profile::DeviceProfile = device_profile::dsl::device_profile
.find(&rd.device_profile_id) .find(&rd.device_profile_id)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, rd.device_profile_id.to_string()))?; .map_err(|e| Error::from_diesel(e, rd.device_profile_id.to_string()))?;
if !rdp.is_relay { if !rdp.is_relay {
return Err(Error::Validation("Device is not a relay".to_string())); return Err(Error::Validation("Device is not a relay".to_string()));
@ -178,6 +157,7 @@ pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(
let d: Device = device::dsl::device let d: Device = device::dsl::device
.find(&device_dev_eui) .find(&device_dev_eui)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, device_dev_eui.to_string()))?; .map_err(|e| Error::from_diesel(e, device_dev_eui.to_string()))?;
if rd.application_id != d.application_id { if rd.application_id != d.application_id {
@ -190,6 +170,7 @@ pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(
let dp: super::device_profile::DeviceProfile = device_profile::dsl::device_profile let dp: super::device_profile::DeviceProfile = device_profile::dsl::device_profile
.find(&d.device_profile_id) .find(&d.device_profile_id)
.get_result(c) .get_result(c)
.await
.map_err(|e| Error::from_diesel(e, d.device_profile_id.to_string()))?; .map_err(|e| Error::from_diesel(e, d.device_profile_id.to_string()))?;
if rdp.region != dp.region { if rdp.region != dp.region {
return Err(Error::Validation( return Err(Error::Validation(
@ -207,6 +188,7 @@ pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(
.select(dsl::count_star()) .select(dsl::count_star())
.filter(relay_device::dsl::relay_dev_eui.eq(&relay_dev_eui)) .filter(relay_device::dsl::relay_dev_eui.eq(&relay_dev_eui))
.first(c) .first(c)
.await
.map_err(|e| Error::from_diesel(e, "".into()))?; .map_err(|e| Error::from_diesel(e, "".into()))?;
if count > RELAY_MAX_DEVICES { if count > RELAY_MAX_DEVICES {
@ -223,13 +205,13 @@ pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(
relay_device::created_at.eq(Utc::now()), relay_device::created_at.eq(Utc::now()),
)) ))
.execute(c) .execute(c)
.await
.map_err(|e| Error::from_diesel(e, "".into()))?; .map_err(|e| Error::from_diesel(e, "".into()))?;
Ok(()) Ok(())
}) })
} })
}) .await?;
.await??;
info!(relay_dev_eui = %relay_dev_eui, device_dev_eui = %device_dev_eui, "Device added to relay"); info!(relay_dev_eui = %relay_dev_eui, device_dev_eui = %device_dev_eui, "Device added to relay");
@ -237,25 +219,20 @@ pub async fn add_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(
} }
pub async fn remove_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(), Error> { pub async fn remove_device(relay_dev_eui: EUI64, device_dev_eui: EUI64) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<(), Error> { let ra = diesel::delete(
let mut c = get_db_conn()?; relay_device::dsl::relay_device
let ra = diesel::delete( .filter(relay_device::relay_dev_eui.eq(&relay_dev_eui))
relay_device::dsl::relay_device .filter(relay_device::dev_eui.eq(&device_dev_eui)),
.filter(relay_device::relay_dev_eui.eq(&relay_dev_eui)) )
.filter(relay_device::dev_eui.eq(&device_dev_eui)), .execute(&mut c)
) .await?;
.execute(&mut c)?; if ra == 0 {
if ra == 0 { return Err(Error::NotFound(format!(
return Err(Error::NotFound(format!( "relay_dev_eui: {}, device_dev_eui: {}",
"relay_dev_eui: {}, device_dev_eui: {}", relay_dev_eui, device_dev_eui
relay_dev_eui, device_dev_eui )));
))); }
}
Ok(())
}
})
.await??;
info!(relay_dev_eui = %relay_dev_eui, device_dev_eui = %device_dev_eui, "Device removed from relay"); info!(relay_dev_eui = %relay_dev_eui, device_dev_eui = %device_dev_eui, "Device removed from relay");

227
chirpstack/src/storage/search.rs
View File

@ -1,13 +1,12 @@
use std::collections::HashMap; use std::collections::HashMap;
use crate::diesel::RunQueryDsl;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use diesel_async::RunQueryDsl;
use regex::Regex; use regex::Regex;
use tokio::task;
use uuid::Uuid; use uuid::Uuid;
use super::error::Error; use super::error::Error;
use super::get_db_conn; use super::get_async_db_conn;
use lrwn::EUI64; use lrwn::EUI64;
lazy_static! { lazy_static! {
@ -45,117 +44,112 @@ pub async fn global_search(
limit: usize, limit: usize,
offset: usize, offset: usize,
) -> Result<Vec<SearchResult>, Error> { ) -> Result<Vec<SearchResult>, Error> {
task::spawn_blocking({ let (query, tags) = parse_search_query(search);
let user_id = *user_id; let query = format!("%{}%", query);
let search = search.to_string(); let tags = serde_json::to_value(tags).context("To serde_json value")?;
let (query, tags) = parse_search_query(&search);
let query = format!("%{}%", query);
let tags = serde_json::to_value(tags).context("To serde_json value")?;
move || -> Result<Vec<SearchResult>, Error> { let mut c = get_async_db_conn().await?;
let mut c = get_db_conn()?; let res: Vec<SearchResult> = diesel::sql_query(
let res = diesel::sql_query( r#"
r#" -- device
-- device select
select 'device' as kind,
'device' as kind, greatest(similarity(d.name, $1), similarity(encode(d.dev_eui, 'hex'), $1), similarity(encode(d.dev_addr, 'hex'), $1)) as score,
greatest(similarity(d.name, $1), similarity(encode(d.dev_eui, 'hex'), $1), similarity(encode(d.dev_addr, 'hex'), $1)) as score, t.id as tenant_id,
t.id as tenant_id, t.name as tenant_name,
t.name as tenant_name, a.id as application_id,
a.id as application_id, a.name as application_name,
a.name as application_name, d.dev_eui as device_dev_eui,
d.dev_eui as device_dev_eui, d.name as device_name,
d.name as device_name, null as gateway_id,
null as gateway_id, null as gateway_name
null as gateway_name from device d
from device d inner join application a
inner join application a on a.id = d.application_id
on a.id = d.application_id inner join tenant t
inner join tenant t on t.id = a.tenant_id
on t.id = a.tenant_id left join tenant_user tu
left join tenant_user tu on tu.tenant_id = t.id
on tu.tenant_id = t.id left join "user" u
left join "user" u on u.id = tu.user_id
on u.id = tu.user_id where
where ($3 = true or u.id = $4)
($3 = true or u.id = $4) and (d.name ilike $2 or encode(d.dev_eui, 'hex') ilike $2 or encode(d.dev_addr, 'hex') ilike $2 or ($7 != '{}'::jsonb and d.tags @> $7))
and (d.name ilike $2 or encode(d.dev_eui, 'hex') ilike $2 or encode(d.dev_addr, 'hex') ilike $2 or ($7 != '{}'::jsonb and d.tags @> $7)) -- gateway
-- gateway union
union select
select 'gateway' as kind,
'gateway' as kind, greatest(similarity(g.name, $1), similarity(encode(g.gateway_id, 'hex'), $1)) as score,
greatest(similarity(g.name, $1), similarity(encode(g.gateway_id, 'hex'), $1)) as score, t.id as tenant_id,
t.id as tenant_id, t.name as tenant_name,
t.name as tenant_name, null as application_id,
null as application_id, null as application_name,
null as application_name, null as device_dev_eui,
null as device_dev_eui, null as device_name,
null as device_name, g.gateway_id as gateway_id,
g.gateway_id as gateway_id, g.name as gateway_name
g.name as gateway_name from
from gateway g
gateway g inner join tenant t
inner join tenant t on t.id = g.tenant_id
on t.id = g.tenant_id left join tenant_user tu
left join tenant_user tu on tu.tenant_id = t.id
on tu.tenant_id = t.id left join "user" u
left join "user" u on u.id = tu.user_id
on u.id = tu.user_id where
where ($3 = true or u.id = $4)
($3 = true or u.id = $4) and (g.name ilike $2 or encode(g.gateway_id, 'hex') ilike $2 or ($7 != '{}'::jsonb and g.tags @> $7))
and (g.name ilike $2 or encode(g.gateway_id, 'hex') ilike $2 or ($7 != '{}'::jsonb and g.tags @> $7)) -- tenant
-- tenant union
union select
select 'tenant' as kind,
'tenant' as kind, similarity(t.name, $1) as score,
similarity(t.name, $1) as score, t.id as tenant_id,
t.id as tenant_id, t.name as tenant_name,
t.name as tenant_name, null as application_id,
null as application_id, null as application_name,
null as application_name, null as device_dev_eui,
null as device_dev_eui, null as device_name,
null as device_name, null as gateway_id,
null as gateway_id, null as gateway_name
null as gateway_name from
from tenant t
tenant t left join tenant_user tu
left join tenant_user tu on tu.tenant_id = t.id
on tu.tenant_id = t.id left join "user" u
left join "user" u on u.id = tu.user_id
on u.id = tu.user_id where
where ($3 = true or u.id = $4)
($3 = true or u.id = $4) and t.name ilike $2
and t.name ilike $2 -- application
-- application union
union select
select 'application' as kind,
'application' as kind, similarity(a.name, $1) as score,
similarity(a.name, $1) as score, t.id as tenant_id,
t.id as tenant_id, t.name as tenant_name,
t.name as tenant_name, a.id as application_id,
a.id as application_id, a.name as application_name,
a.name as application_name, null as device_dev_eui,
null as device_dev_eui, null as device_name,
null as device_name, null as gateway_id,
null as gateway_id, null as gateway_name
null as gateway_name from
from application a
application a inner join tenant t
inner join tenant t on t.id = a.tenant_id
on t.id = a.tenant_id left join tenant_user tu
left join tenant_user tu on tu.tenant_id = t.id
on tu.tenant_id = t.id left join "user" u
left join "user" u on u.id = tu.user_id
on u.id = tu.user_id where
where ($3 = true or u.id = $4)
($3 = true or u.id = $4) and a.name ilike $2
and a.name ilike $2 order by
order by score desc
score desc limit $5
limit $5 offset $6
offset $6 "#)
"#,
)
.bind::<diesel::sql_types::Text, _>(&search) .bind::<diesel::sql_types::Text, _>(&search)
.bind::<diesel::sql_types::Text, _>(&query) .bind::<diesel::sql_types::Text, _>(&query)
.bind::<diesel::sql_types::Bool, _>(global_admin) .bind::<diesel::sql_types::Bool, _>(global_admin)
@ -163,12 +157,9 @@ pub async fn global_search(
.bind::<diesel::sql_types::BigInt, _>(limit as i64) .bind::<diesel::sql_types::BigInt, _>(limit as i64)
.bind::<diesel::sql_types::BigInt, _>(offset as i64) .bind::<diesel::sql_types::BigInt, _>(offset as i64)
.bind::<diesel::sql_types::Jsonb, _>(tags) .bind::<diesel::sql_types::Jsonb, _>(tags)
.load(&mut c)?; .load(&mut c).await?;
Ok(res) Ok(res)
}
})
.await?
} }
fn parse_search_query(q: &str) -> (String, HashMap<String, String>) { fn parse_search_query(q: &str) -> (String, HashMap<String, String>) {

348
chirpstack/src/storage/tenant.rs
View File

@ -2,15 +2,14 @@ use std::collections::HashMap;
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use super::error::Error; use super::error::Error;
use super::schema::{tenant, tenant_user, user}; use super::schema::{tenant, tenant_user, user};
use super::{fields, get_db_conn}; use super::{fields, get_async_db_conn};
#[derive(Queryable, Insertable, PartialEq, Eq, Debug, Clone)] #[derive(Queryable, Insertable, PartialEq, Eq, Debug, Clone)]
#[diesel(table_name = tenant)] #[diesel(table_name = tenant)]
@ -105,148 +104,113 @@ pub struct Filters {
pub async fn create(t: Tenant) -> Result<Tenant, Error> { pub async fn create(t: Tenant) -> Result<Tenant, Error> {
t.validate()?; t.validate()?;
let t = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Tenant, Error> { let t: Tenant = diesel::insert_into(tenant::table)
let mut c = get_db_conn()?; .values(&t)
diesel::insert_into(tenant::table) .get_result(&mut c)
.values(&t) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, t.id.to_string()))?;
.map_err(|e| Error::from_diesel(e, t.id.to_string()))
}
})
.await??;
info!(id = %t.id, "Tenant created"); info!(id = %t.id, "Tenant created");
Ok(t) Ok(t)
} }
pub async fn get(id: &Uuid) -> Result<Tenant, Error> { pub async fn get(id: &Uuid) -> Result<Tenant, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let t = tenant::dsl::tenant
move || -> Result<Tenant, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
let t = tenant::dsl::tenant .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.first(&mut c) Ok(t)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(t)
}
})
.await?
} }
pub async fn update(t: Tenant) -> Result<Tenant, Error> { pub async fn update(t: Tenant) -> Result<Tenant, Error> {
t.validate()?; t.validate()?;
let t = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Tenant, Error> { let t: Tenant = diesel::update(tenant::dsl::tenant.find(&t.id))
let mut c = get_db_conn()?; .set((
diesel::update(tenant::dsl::tenant.find(&t.id)) tenant::updated_at.eq(Utc::now()),
.set(( tenant::name.eq(&t.name),
tenant::updated_at.eq(Utc::now()), tenant::description.eq(&t.description),
tenant::name.eq(&t.name), tenant::can_have_gateways.eq(&t.can_have_gateways),
tenant::description.eq(&t.description), tenant::max_device_count.eq(&t.max_device_count),
tenant::can_have_gateways.eq(&t.can_have_gateways), tenant::max_gateway_count.eq(&t.max_gateway_count),
tenant::max_device_count.eq(&t.max_device_count), tenant::private_gateways_up.eq(&t.private_gateways_up),
tenant::max_gateway_count.eq(&t.max_gateway_count), tenant::private_gateways_down.eq(&t.private_gateways_down),
tenant::private_gateways_up.eq(&t.private_gateways_up), tenant::tags.eq(&t.tags),
tenant::private_gateways_down.eq(&t.private_gateways_down), ))
tenant::tags.eq(&t.tags), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, t.id.to_string()))?;
.map_err(|e| Error::from_diesel(e, t.id.to_string()))
}
})
.await??;
info!(id = %t.id, "Tenant updated"); info!(id = %t.id, "Tenant updated");
Ok(t) Ok(t)
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(tenant::dsl::tenant.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await
let ra = diesel::delete(tenant::dsl::tenant.find(&id)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.execute(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
if ra == 0 { if ra == 0 {
return Err(Error::NotFound(id.to_string())); return Err(Error::NotFound(id.to_string()));
} }
Ok(())
}
})
.await??;
info!(id = %id, "Tenant deleted"); info!(id = %id, "Tenant deleted");
Ok(()) Ok(())
} }
pub async fn get_count(filters: &Filters) -> Result<i64, Error> { pub async fn get_count(filters: &Filters) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = tenant::dsl::tenant
move || -> Result<i64, Error> { .left_join(tenant_user::table)
let mut c = get_db_conn()?; .into_boxed();
let mut q = tenant::dsl::tenant
.left_join(tenant_user::table)
.into_boxed();
if let Some(user_id) = &filters.user_id { if let Some(user_id) = &filters.user_id {
q = q.filter(tenant_user::dsl::user_id.eq(user_id)); q = q.filter(tenant_user::dsl::user_id.eq(user_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(tenant::dsl::name.ilike(format!("%{}%", search))); q = q.filter(tenant::dsl::name.ilike(format!("%{}%", search)));
} }
Ok( Ok(
q.select(dsl::sql::<diesel::sql_types::BigInt>("count(distinct id)")) q.select(dsl::sql::<diesel::sql_types::BigInt>("count(distinct id)"))
.first(&mut c)?, .first(&mut c)
) .await?,
} )
})
.await?
} }
pub async fn list(limit: i64, offset: i64, filters: &Filters) -> Result<Vec<Tenant>, Error> { pub async fn list(limit: i64, offset: i64, filters: &Filters) -> Result<Vec<Tenant>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let filters = filters.clone(); let mut q = tenant::dsl::tenant
move || -> Result<Vec<Tenant>, Error> { .left_join(tenant_user::table)
let mut c = get_db_conn()?; .select(tenant::all_columns)
let mut q = tenant::dsl::tenant .group_by(tenant::dsl::id)
.left_join(tenant_user::table) .order_by(tenant::dsl::name)
.select(tenant::all_columns) .limit(limit)
.group_by(tenant::dsl::id) .offset(offset)
.order_by(tenant::dsl::name) .into_boxed();
.limit(limit)
.offset(offset)
.into_boxed();
if let Some(user_id) = &filters.user_id { if let Some(user_id) = &filters.user_id {
q = q.filter(tenant_user::dsl::user_id.eq(user_id)); q = q.filter(tenant_user::dsl::user_id.eq(user_id));
} }
if let Some(search) = &filters.search { if let Some(search) = &filters.search {
q = q.filter(tenant::dsl::name.ilike(format!("%{}%", search))); q = q.filter(tenant::dsl::name.ilike(format!("%{}%", search)));
} }
let items = q.load(&mut c)?; let items = q.load(&mut c).await?;
Ok(items)
Ok(items)
}
})
.await?
} }
pub async fn add_user(tu: TenantUser) -> Result<TenantUser, Error> { pub async fn add_user(tu: TenantUser) -> Result<TenantUser, Error> {
let tu = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<TenantUser, Error> { let tu: TenantUser = diesel::insert_into(tenant_user::table)
let mut c = get_db_conn()?; .values(&tu)
diesel::insert_into(tenant_user::table) .get_result(&mut c)
.values(&tu) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, tu.user_id.to_string()))?;
.map_err(|e| Error::from_diesel(e, tu.user_id.to_string()))
}
})
.await??;
info!( info!(
tenant_id = %tu.tenant_id, tenant_id = %tu.tenant_id,
user_id = %tu.user_id, user_id = %tu.user_id,
@ -256,20 +220,16 @@ pub async fn add_user(tu: TenantUser) -> Result<TenantUser, Error> {
} }
pub async fn update_user(tu: TenantUser) -> Result<TenantUser, Error> { pub async fn update_user(tu: TenantUser) -> Result<TenantUser, Error> {
let tu = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<TenantUser, Error> { let tu: TenantUser = diesel::update(
let mut c = get_db_conn()?; tenant_user::dsl::tenant_user
diesel::update( .filter(tenant_user::dsl::tenant_id.eq(&tu.tenant_id))
tenant_user::dsl::tenant_user .filter(tenant_user::dsl::user_id.eq(&tu.user_id)),
.filter(tenant_user::dsl::tenant_id.eq(&tu.tenant_id)) )
.filter(tenant_user::dsl::user_id.eq(&tu.user_id)), .set(&tu)
) .get_result(&mut c)
.set(&tu) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, tu.user_id.to_string()))?;
.map_err(|e| Error::from_diesel(e, tu.user_id.to_string()))
}
})
.await??;
info!( info!(
tenant_id = %tu.tenant_id, tenant_id = %tu.tenant_id,
user_id = %tu.user_id, user_id = %tu.user_id,
@ -279,35 +239,24 @@ pub async fn update_user(tu: TenantUser) -> Result<TenantUser, Error> {
} }
pub async fn get_user(tenant_id: &Uuid, user_id: &Uuid) -> Result<TenantUser, Error> { pub async fn get_user(tenant_id: &Uuid, user_id: &Uuid) -> Result<TenantUser, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let tu: TenantUser = tenant_user::dsl::tenant_user
let user_id = *user_id; .filter(tenant_user::dsl::tenant_id.eq(&tenant_id))
move || -> Result<TenantUser, Error> { .filter(tenant_user::dsl::user_id.eq(&user_id))
let mut c = get_db_conn()?; .first(&mut c)
let tu: TenantUser = tenant_user::dsl::tenant_user .await
.filter(tenant_user::dsl::tenant_id.eq(&tenant_id)) .map_err(|e| Error::from_diesel(e, user_id.to_string()))?;
.filter(tenant_user::dsl::user_id.eq(&user_id)) Ok(tu)
.first(&mut c)
.map_err(|e| Error::from_diesel(e, user_id.to_string()))?;
Ok(tu)
}
})
.await?
} }
pub async fn get_user_count(tenant_id: &Uuid) -> Result<i64, Error> { pub async fn get_user_count(tenant_id: &Uuid) -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let count = tenant_user::dsl::tenant_user
move || -> Result<i64, Error> { .select(dsl::count_star())
let mut c = get_db_conn()?; .filter(tenant_user::dsl::tenant_id.eq(&tenant_id))
let count = tenant_user::dsl::tenant_user .first(&mut c)
.select(dsl::count_star()) .await?;
.filter(tenant_user::dsl::tenant_id.eq(&tenant_id)) Ok(count)
.first(&mut c)?;
Ok(count)
}
})
.await?
} }
pub async fn get_users( pub async fn get_users(
@ -315,53 +264,41 @@ pub async fn get_users(
limit: i64, limit: i64,
offset: i64, offset: i64,
) -> Result<Vec<TenantUserListItem>, Error> { ) -> Result<Vec<TenantUserListItem>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let items = tenant_user::dsl::tenant_user
move || -> Result<Vec<TenantUserListItem>, Error> { .inner_join(user::table)
let mut c = get_db_conn()?; .select((
let items = tenant_user::dsl::tenant_user tenant_user::dsl::tenant_id,
.inner_join(user::table) tenant_user::dsl::user_id,
.select(( tenant_user::dsl::created_at,
tenant_user::dsl::tenant_id, tenant_user::dsl::updated_at,
tenant_user::dsl::user_id, user::dsl::email,
tenant_user::dsl::created_at, tenant_user::dsl::is_admin,
tenant_user::dsl::updated_at, tenant_user::dsl::is_device_admin,
user::dsl::email, tenant_user::dsl::is_gateway_admin,
tenant_user::dsl::is_admin, ))
tenant_user::dsl::is_device_admin, .filter(tenant_user::dsl::tenant_id.eq(&tenant_id))
tenant_user::dsl::is_gateway_admin, .order_by(user::dsl::email)
)) .limit(limit)
.filter(tenant_user::dsl::tenant_id.eq(&tenant_id)) .offset(offset)
.order_by(user::dsl::email) .load(&mut c)
.limit(limit) .await?;
.offset(offset)
.load(&mut c)?;
Ok(items) Ok(items)
}
})
.await?
} }
pub async fn delete_user(tenant_id: &Uuid, user_id: &Uuid) -> Result<(), Error> { pub async fn delete_user(tenant_id: &Uuid, user_id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let tenant_id = *tenant_id; let ra = diesel::delete(
let user_id = *user_id; tenant_user::dsl::tenant_user
move || -> Result<(), Error> { .filter(tenant_user::dsl::tenant_id.eq(&tenant_id))
let mut c = get_db_conn()?; .filter(tenant_user::dsl::user_id.eq(&user_id)),
let ra = diesel::delete( )
tenant_user::dsl::tenant_user .execute(&mut c)
.filter(tenant_user::dsl::tenant_id.eq(&tenant_id)) .await?;
.filter(tenant_user::dsl::user_id.eq(&user_id)), if ra == 0 {
) return Err(Error::NotFound(user_id.to_string()));
.execute(&mut c)?; }
if ra == 0 {
return Err(Error::NotFound(user_id.to_string()));
}
Ok(())
}
})
.await??;
info!( info!(
tenant_id = %tenant_id, tenant_id = %tenant_id,
user_id = %user_id, user_id = %user_id,
@ -371,17 +308,12 @@ pub async fn delete_user(tenant_id: &Uuid, user_id: &Uuid) -> Result<(), Error>
} }
pub async fn get_tenant_users_for_user(user_id: &Uuid) -> Result<Vec<TenantUser>, Error> { pub async fn get_tenant_users_for_user(user_id: &Uuid) -> Result<Vec<TenantUser>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let user_id = *user_id; let items = tenant_user::dsl::tenant_user
move || -> Result<Vec<TenantUser>, Error> { .filter(tenant_user::dsl::user_id.eq(&user_id))
let mut c = get_db_conn()?; .load(&mut c)
let items = tenant_user::dsl::tenant_user .await?;
.filter(tenant_user::dsl::user_id.eq(&user_id)) Ok(items)
.load(&mut c)?;
Ok(items)
}
})
.await?
} }
#[cfg(test)] #[cfg(test)]

232
chirpstack/src/storage/user.rs
View File

@ -1,19 +1,18 @@
use anyhow::Result; use anyhow::Result;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use diesel::dsl; use diesel::{dsl, prelude::*};
use diesel::prelude::*; use diesel_async::RunQueryDsl;
use pbkdf2::{ use pbkdf2::{
password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString}, password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString},
Algorithm, Pbkdf2, Algorithm, Pbkdf2,
}; };
use rand_core::OsRng; use rand_core::OsRng;
use tokio::task;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
use validator::validate_email; use validator::validate_email;
use super::error::Error; use super::error::Error;
use super::get_db_conn; use super::get_async_db_conn;
use super::schema::user; use super::schema::user;
#[derive(Queryable, Insertable, PartialEq, Eq, Debug, Clone)] #[derive(Queryable, Insertable, PartialEq, Eq, Debug, Clone)]
@ -67,181 +66,134 @@ impl User {
pub async fn create(u: User) -> Result<User, Error> { pub async fn create(u: User) -> Result<User, Error> {
u.validate()?; u.validate()?;
let u = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<User, Error> {
let mut c = get_db_conn()?;
diesel::insert_into(user::table) let u: User = diesel::insert_into(user::table)
.values(&u) .values(&u)
.get_result(&mut c) .get_result(&mut c)
.map_err(|e| Error::from_diesel(e, u.id.to_string())) .await
} .map_err(|e| Error::from_diesel(e, u.id.to_string()))?;
})
.await??;
info!(id = %u.id, "User created"); info!(id = %u.id, "User created");
Ok(u) Ok(u)
} }
pub async fn get(id: &Uuid) -> Result<User, Error> { pub async fn get(id: &Uuid) -> Result<User, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let u = user::dsl::user
move || -> Result<User, Error> { .find(&id)
let mut c = get_db_conn()?; .first(&mut c)
let u = user::dsl::user .await
.find(&id) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.first(&mut c) Ok(u)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
Ok(u)
}
})
.await?
} }
pub async fn get_by_email(email: &str) -> Result<User, Error> { pub async fn get_by_email(email: &str) -> Result<User, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let email = email.to_string(); let u = user::dsl::user
move || -> Result<User, Error> { .filter(user::dsl::email.eq(email))
let mut c = get_db_conn()?; .first(&mut c)
let u = user::dsl::user .await
.filter(user::dsl::email.eq(&email)) .map_err(|e| Error::from_diesel(e, email.to_string()))?;
.first(&mut c) Ok(u)
.map_err(|e| Error::from_diesel(e, email))?;
Ok(u)
}
})
.await?
} }
pub async fn get_by_external_id(external_id: &str) -> Result<User, Error> { pub async fn get_by_external_id(external_id: &str) -> Result<User, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let external_id = external_id.to_string(); let u = user::dsl::user
move || -> Result<User, Error> { .filter(user::dsl::external_id.eq(external_id))
let mut c = get_db_conn()?; .first(&mut c)
let u = user::dsl::user .await
.filter(user::dsl::external_id.eq(&external_id)) .map_err(|e| Error::from_diesel(e, external_id.to_string()))?;
.first(&mut c) Ok(u)
.map_err(|e| Error::from_diesel(e, external_id))?;
Ok(u)
}
})
.await?
} }
pub async fn get_by_email_and_pw(email: &str, pw: &str) -> Result<User, Error> { pub async fn get_by_email_and_pw(email: &str, pw: &str) -> Result<User, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let email = email.to_string(); let u: User = match user::dsl::user
let pw = pw.to_string(); .filter(user::dsl::email.eq(email))
move || -> Result<User, Error> { .first(&mut c)
let mut c = get_db_conn()?; .await
let u: User = match user::dsl::user .map_err(|e| Error::from_diesel(e, email.to_string()))
.filter(user::dsl::email.eq(&email)) {
.first(&mut c) Ok(v) => v,
.map_err(|e| Error::from_diesel(e, email)) Err(Error::NotFound(_)) => {
{ return Err(Error::InvalidUsernameOrPassword);
Ok(v) => v,
Err(Error::NotFound(_)) => {
return Err(Error::InvalidUsernameOrPassword);
}
Err(v) => {
return Err(v);
}
};
if verify_password(&pw, &u.password_hash) {
return Ok(u);
}
Err(Error::InvalidUsernameOrPassword)
} }
}) Err(v) => {
.await? return Err(v);
}
};
if verify_password(&pw, &u.password_hash) {
return Ok(u);
}
Err(Error::InvalidUsernameOrPassword)
} }
pub async fn update(u: User) -> Result<User, Error> { pub async fn update(u: User) -> Result<User, Error> {
u.validate()?; u.validate()?;
let u = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<User, Error> { let u: User = diesel::update(user::dsl::user.find(&u.id))
let mut c = get_db_conn()?; .set((
diesel::update(user::dsl::user.find(&u.id)) user::updated_at.eq(Utc::now()),
.set(( user::is_admin.eq(&u.is_admin),
user::updated_at.eq(Utc::now()), user::is_active.eq(&u.is_active),
user::is_admin.eq(&u.is_admin), user::email.eq(&u.email),
user::is_active.eq(&u.is_active), user::email_verified.eq(&u.email_verified),
user::email.eq(&u.email), user::note.eq(&u.note),
user::email_verified.eq(&u.email_verified), user::external_id.eq(&u.external_id),
user::note.eq(&u.note), ))
user::external_id.eq(&u.external_id), .get_result(&mut c)
)) .await
.get_result(&mut c) .map_err(|e| Error::from_diesel(e, u.id.to_string()))?;
.map_err(|e| Error::from_diesel(e, u.id.to_string()))
}
})
.await??;
info!(user_id = %u.id, "User updated"); info!(user_id = %u.id, "User updated");
Ok(u) Ok(u)
} }
pub async fn set_password_hash(id: &Uuid, hash: &str) -> Result<User, Error> { pub async fn set_password_hash(id: &Uuid, hash: &str) -> Result<User, Error> {
let u = task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let u: User = diesel::update(user::dsl::user.find(&id))
let hash = hash.to_string(); .set(user::password_hash.eq(&hash))
move || -> Result<User, Error> { .get_result(&mut c)
let mut c = get_db_conn()?; .await
diesel::update(user::dsl::user.find(&id)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.set(user::password_hash.eq(&hash))
.get_result(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))
}
})
.await??;
info!(id = %id, "Password set"); info!(id = %id, "Password set");
Ok(u) Ok(u)
} }
pub async fn delete(id: &Uuid) -> Result<(), Error> { pub async fn delete(id: &Uuid) -> Result<(), Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
let id = *id; let ra = diesel::delete(user::dsl::user.find(&id))
move || -> Result<(), Error> { .execute(&mut c)
let mut c = get_db_conn()?; .await
let ra = diesel::delete(user::dsl::user.find(&id)) .map_err(|e| Error::from_diesel(e, id.to_string()))?;
.execute(&mut c)
.map_err(|e| Error::from_diesel(e, id.to_string()))?;
if ra == 0 { if ra == 0 {
return Err(Error::NotFound(id.to_string())); return Err(Error::NotFound(id.to_string()));
} }
Ok(())
}
})
.await??;
info!(user_id = %id, "User deleted"); info!(user_id = %id, "User deleted");
Ok(()) Ok(())
} }
pub async fn get_count() -> Result<i64, Error> { pub async fn get_count() -> Result<i64, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<i64, Error> { let count = user::dsl::user
let mut c = get_db_conn()?; .select(dsl::count_star())
let count = user::dsl::user.select(dsl::count_star()).first(&mut c)?; .first(&mut c)
Ok(count) .await?;
} Ok(count)
})
.await?
} }
pub async fn list(limit: i64, offset: i64) -> Result<Vec<User>, Error> { pub async fn list(limit: i64, offset: i64) -> Result<Vec<User>, Error> {
task::spawn_blocking({ let mut c = get_async_db_conn().await?;
move || -> Result<Vec<User>, Error> { let items = user::dsl::user
let mut c = get_db_conn()?; .order_by(user::dsl::email)
let items = user::dsl::user .limit(limit)
.order_by(user::dsl::email) .offset(offset)
.limit(limit) .load(&mut c)
.offset(offset) .await?;
.load(&mut c)?; Ok(items)
Ok(items)
}
})
.await?
} }
// The output format is documented here: // The output format is documented here:

2
chirpstack/src/test/mod.rs
View File

@ -80,7 +80,7 @@ pub async fn prepare<'a>() -> std::sync::MutexGuard<'a, ()> {
storage::setup().await.unwrap(); storage::setup().await.unwrap();
// reset db // reset db
storage::reset_db().unwrap(); storage::reset_db().await.unwrap();
// flush redis db // flush redis db
storage::reset_redis().await.unwrap(); storage::reset_redis().await.unwrap();

26
cross/Dockerfile.aarch64-unknown-linux-musl
View File

@ -1,17 +1,13 @@
FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:latest FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:latest
ENV ZLIB_VERSION=1.3
ENV POSTGRESQL_VERSION=11.21
ENV OPENSSL_VERSION=3.1.2 ENV OPENSSL_VERSION=3.1.2
ENV OPENSSL_TARGET=linux-aarch64 ENV OPENSSL_TARGET=linux-aarch64
ENV MUSL_PREFIX=aarch64-linux-musl ENV MUSL_PREFIX=aarch64-linux-musl
ENV POSTGRESQL_HOST=aarch64-unknown-linux-musl
RUN apt-get update && \ RUN apt-get update && \
apt-get --assume-yes install \ apt-get --assume-yes install \
protobuf-compiler \ protobuf-compiler \
libprotobuf-dev \ libprotobuf-dev
binutils-aarch64-linux-gnu
RUN echo "Building OpenSSL" && \ RUN echo "Building OpenSSL" && \
cd /tmp && \ cd /tmp && \
@ -23,24 +19,4 @@ RUN echo "Building OpenSSL" && \
make install_sw && \ make install_sw && \
rm -r /tmp/* rm -r /tmp/*
RUN echo "Building zlib" && \
cd /tmp && \
curl -fLO "https://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \
tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \
CC=$MUSL_PREFIX-gcc ./configure --static --prefix=/usr/local/$MUSL_PREFIX-target && \
make && make install && \
rm -r /tmp/*
RUN echo "Building libpq" && \
cd /tmp && \
curl -fLO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \
tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \
CC=$MUSL_PREFIX-gcc CPPFLAGS="-I/usr/local/$MUSL_PREFIX/include -I/usr/local/$MUSL_PREFIX-target/include" LDFLAGS="-L/usr/local/$MUSL_PREFIX/lib -L/usr/local/$MUSL_PREFIX-target/lib" ./configure --with-openssl --without-readline --prefix=/usr/local/$MUSL_PREFIX-target --host $POSTGRESQL_HOST && \
cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \
rm -r /tmp/*
# Workaround for re-defined unicode_to_utf8 which also exists in quickjs library.
RUN /usr/aarch64-linux-gnu/bin/objcopy --redefine-sym unicode_to_utf8=unicode_to_utf8_rename /usr/local/$MUSL_PREFIX-target/lib/libpq.a
ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig

26
cross/Dockerfile.armv7-unknown-linux-musleabihf
View File

@ -1,17 +1,13 @@
FROM ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:latest FROM ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:latest
ENV ZLIB_VERSION=1.3
ENV POSTGRESQL_VERSION=11.21
ENV OPENSSL_VERSION=3.1.2 ENV OPENSSL_VERSION=3.1.2
ENV OPENSSL_TARGET=linux-generic32 ENV OPENSSL_TARGET=linux-generic32
ENV MUSL_PREFIX=arm-linux-musleabihf ENV MUSL_PREFIX=arm-linux-musleabihf
ENV POSTGRESQL_HOST=armv7-unknown-linux-musleabihf
RUN apt-get update && \ RUN apt-get update && \
apt-get --assume-yes install \ apt-get --assume-yes install \
protobuf-compiler \ protobuf-compiler \
libprotobuf-dev \ libprotobuf-dev
binutils-arm-linux-gnueabihf
RUN echo "Building OpenSSL" && \ RUN echo "Building OpenSSL" && \
cd /tmp && \ cd /tmp && \
@ -23,24 +19,4 @@ RUN echo "Building OpenSSL" && \
make install_sw && \ make install_sw && \
rm -r /tmp/* rm -r /tmp/*
RUN echo "Building zlib" && \
cd /tmp && \
curl -fLO "https://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \
tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \
CC=$MUSL_PREFIX-gcc ./configure --static --prefix=/usr/local/$MUSL_PREFIX-target && \
make && make install && \
rm -r /tmp/*
RUN echo "Building libpq" && \
cd /tmp && \
curl -fLO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \
tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \
CC=$MUSL_PREFIX-gcc CPPFLAGS="-I/usr/local/$MUSL_PREFIX/include -I/usr/local/$MUSL_PREFIX-target/include" LDFLAGS="-L/usr/local/$MUSL_PREFIX/lib -L/usr/local/$MUSL_PREFIX-target/lib" ./configure --with-openssl --without-readline --prefix=/usr/local/$MUSL_PREFIX-target --host $POSTGRESQL_HOST && \
cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \
rm -r /tmp/*
# Workaround for re-defined unicode_to_utf8 which also exists in quickjs library.
RUN /usr/bin/arm-linux-gnueabihf-objcopy --redefine-sym unicode_to_utf8=unicode_to_utf8_rename /usr/local/$MUSL_PREFIX-target/lib/libpq.a
ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig

22
cross/Dockerfile.x86_64-unknown-linux-musl
View File

@ -1,7 +1,5 @@
FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:latest FROM ghcr.io/cross-rs/x86_64-unknown-linux-musl:latest
ENV ZLIB_VERSION=1.3
ENV POSTGRESQL_VERSION=11.21
ENV OPENSSL_VERSION=3.1.2 ENV OPENSSL_VERSION=3.1.2
ENV OPENSSL_TARGET=linux-x86_64 ENV OPENSSL_TARGET=linux-x86_64
ENV MUSL_PREFIX=x86_64-linux-musl ENV MUSL_PREFIX=x86_64-linux-musl
@ -21,24 +19,4 @@ RUN echo "Building OpenSSL" && \
make install_sw && \ make install_sw && \
rm -r /tmp/* rm -r /tmp/*
RUN echo "Building zlib" && \
cd /tmp && \
curl -fLO "https://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \
tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \
CC=$MUSL_PREFIX-gcc ./configure --static --prefix=/usr/local/$MUSL_PREFIX-target && \
make && make install && \
rm -r /tmp/*
RUN echo "Building libpq" && \
cd /tmp && \
curl -fLO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \
tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \
CC=$MUSL_PREFIX-gcc CPPFLAGS="-I/usr/local/$MUSL_PREFIX/include -I/usr/local/$MUSL_PREFIX-target/include" LDFLAGS="-L/usr/local/$MUSL_PREFIX/lib -L/usr/local/$MUSL_PREFIX-target/lib -L/usr/local/$MUSL_PREFIX-target/lib64" ./configure --with-openssl --without-readline --prefix=/usr/local/$MUSL_PREFIX-target && \
cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \
rm -r /tmp/*
# Workaround for re-defined unicode_to_utf8 which also exists in quickjs library.
RUN objcopy --redefine-sym unicode_to_utf8=unicode_to_utf8_rename /usr/local/$MUSL_PREFIX-target/lib/libpq.a
ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig:/usr/local/$MUSL_PREFIX-target/lib64/pkgconfig ENV PKG_CONFIG_PATH=/usr/local/$MUSL_PREFIX-target/lib/pkgconfig:/usr/local/$MUSL_PREFIX-target/lib64/pkgconfig

2
lrwn/Cargo.toml
View File

@ -16,7 +16,7 @@ hex = "0.4"
cmac = { version = "0.7", optional = true } cmac = { version = "0.7", optional = true }
aes = { version = "0.8", optional = true } aes = { version = "0.8", optional = true }
serde = { version = "1.0", features = ["derive"], optional = true } serde = { version = "1.0", features = ["derive"], optional = true }
diesel = { version = "2.1", features = ["postgres"], optional = true } diesel = { version = "2.1", features = ["postgres_backend"], optional = true }
# Error handling # Error handling
thiserror = "1.0" thiserror = "1.0"

1
shell.nix
View File

@ -11,7 +11,6 @@ pkgs.mkShell {
pkgs.perl pkgs.perl
pkgs.cmake pkgs.cmake
pkgs.clang pkgs.clang
pkgs.postgresql
pkgs.openssl pkgs.openssl
]; ];
LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";