ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-18 12:48:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to tokio 1.5.0 (#800)

Browse Source 
Upgrade to tokio 1.5.0
replaced tokio::sync::mpsc:: with [flume ](https://crates.io/crates/flume) because try_recv was removed
This commit is contained in:
Cheick Keita
2021-04-14 03:07:28 -07:00
committed by GitHub
parent 3f52f34b6a
commit e413aec03d
44 changed files with 683 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
src/agent/Cargo.lock generated
View File

@ -66,7 +66,7 @@ dependencies = [
"http", "http",
"log", "log",
"paste", "paste",
"reqwest", "reqwest 0.10.10",
"serde", "serde",
"serde_json", "serde_json",
"sm", "sm",
@ -751,6 +751,19 @@ dependencies = [
"miniz_oxide", "miniz_oxide",
] ]
[[package]]
name = "flume"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11fce69af4d4582ea989e6adfc5c9b81fd2071ff89234e5c14675c82a85217df"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"pin-project",
"spinning_top",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@ -865,9 +878,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -880,9 +893,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -890,15 +903,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -907,9 +920,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
[[package]] [[package]]
name = "futures-lite" name = "futures-lite"
@ -928,9 +941,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2 1.0.24", "proc-macro2 1.0.24",
@ -940,21 +953,21 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -998,8 +1011,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"js-sys",
"libc", "libc",
"wasi 0.10.2+wasi-snapshot-preview1", "wasi 0.10.2+wasi-snapshot-preview1",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -1053,11 +1068,30 @@ dependencies = [
"indexmap", "indexmap",
"slab", "slab",
"tokio 0.2.25", "tokio 0.2.25",
"tokio-util", "tokio-util 0.3.1",
"tracing", "tracing",
"tracing-futures", "tracing-futures",
] ]
[[package]]
name = "h2"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00"
dependencies = [
"bytes 1.0.1",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http",
"indexmap",
"slab",
"tokio 1.5.0",
"tokio-util 0.6.6",
"tracing",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.9.1" version = "0.9.1"
@ -1120,6 +1154,17 @@ dependencies = [
"http", "http",
] ]
[[package]]
name = "http-body"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737"
dependencies = [
"bytes 1.0.1",
"http",
"pin-project-lite 0.2.6",
]
[[package]] [[package]]
name = "httparse" name = "httparse"
version = "1.3.5" version = "1.3.5"
@ -1148,20 +1193,44 @@ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-util", "futures-util",
"h2", "h2 0.2.7",
"http", "http",
"http-body", "http-body 0.3.1",
"httparse", "httparse",
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project", "pin-project",
"socket2", "socket2 0.3.19",
"tokio 0.2.25", "tokio 0.2.25",
"tower-service", "tower-service",
"tracing", "tracing",
"want", "want",
] ]
[[package]]
name = "hyper"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1"
dependencies = [
"bytes 1.0.1",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.2",
"http",
"http-body 0.4.1",
"httparse",
"httpdate",
"itoa",
"pin-project",
"socket2 0.4.0",
"tokio 1.5.0",
"tower-service",
"tracing",
"want",
]
[[package]] [[package]]
name = "hyper-tls" name = "hyper-tls"
version = "0.4.3" version = "0.4.3"
@ -1169,12 +1238,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 0.5.6",
"hyper", "hyper 0.13.10",
"native-tls", "native-tls",
"tokio 0.2.25", "tokio 0.2.25",
"tokio-tls", "tokio-tls",
] ]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes 1.0.1",
"hyper 0.14.5",
"native-tls",
"tokio 1.5.0",
"tokio-native-tls",
]
[[package]] [[package]]
name = "iced-x86" name = "iced-x86"
version = "1.10.3" version = "1.10.3"
@ -1491,29 +1573,6 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "mio-named-pipes"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656"
dependencies = [
"log",
"mio 0.6.23",
"miow 0.3.6",
"winapi 0.3.9",
]
[[package]]
name = "mio-uds"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
dependencies = [
"iovec",
"libc",
"mio 0.6.23",
]
[[package]] [[package]]
name = "miow" name = "miow"
version = "0.2.2" version = "0.2.2"
@ -1532,7 +1591,7 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
dependencies = [ dependencies = [
"socket2", "socket2 0.3.19",
"winapi 0.3.9", "winapi 0.3.9",
] ]
@ -1545,6 +1604,15 @@ dependencies = [
"bitflags", "bitflags",
] ]
[[package]]
name = "nanorand"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1378b66f7c93a1c0f8464a19bf47df8795083842e5090f4b7305973d5a22d0"
dependencies = [
"getrandom 0.2.2",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.7" version = "0.2.7"
@ -1570,7 +1638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f" checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f"
dependencies = [ dependencies = [
"libc", "libc",
"socket2", "socket2 0.3.19",
] ]
[[package]] [[package]]
@ -1704,11 +1772,12 @@ dependencies = [
"async-trait", "async-trait",
"backoff", "backoff",
"base64", "base64",
"bytes 0.5.6", "bytes 1.0.1",
"cpp_demangle", "cpp_demangle",
"debugger", "debugger",
"dunce", "dunce",
"futures", "futures",
"futures-util",
"hex", "hex",
"input-tester", "input-tester",
"lazy_static", "lazy_static",
@ -1721,7 +1790,7 @@ dependencies = [
"process_control", "process_control",
"rand 0.7.3", "rand 0.7.3",
"regex", "regex",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"ring", "ring",
"rstack", "rstack",
@ -1735,8 +1804,9 @@ dependencies = [
"strum_macros", "strum_macros",
"sysinfo 0.16.4", "sysinfo 0.16.4",
"tempfile", "tempfile",
"tokio 0.2.25", "tokio 1.5.0",
"tokio-util", "tokio-stream",
"tokio-util 0.6.6",
"url", "url",
"urlparse", "urlparse",
"uuid", "uuid",
@ -1756,6 +1826,7 @@ dependencies = [
"clap", "clap",
"crossterm", "crossterm",
"env_logger", "env_logger",
"flume",
"futures", "futures",
"hex", "hex",
"lazy_static", "lazy_static",
@ -1764,7 +1835,7 @@ dependencies = [
"onefuzz", "onefuzz",
"onefuzz-telemetry", "onefuzz-telemetry",
"path-absolutize", "path-absolutize",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"serde", "serde",
"serde_json", "serde_json",
@ -1772,9 +1843,9 @@ dependencies = [
"storage-queue", "storage-queue",
"tempfile", "tempfile",
"thiserror", "thiserror",
"tokio 0.2.25", "tokio 1.5.0",
"tokio-stream", "tokio-stream",
"tokio-util", "tokio-util 0.6.6",
"tui", "tui",
"url", "url",
"uuid", "uuid",
@ -1794,13 +1865,13 @@ dependencies = [
"log", "log",
"onefuzz", "onefuzz",
"onefuzz-telemetry", "onefuzz-telemetry",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"serde", "serde",
"serde_json", "serde_json",
"storage-queue", "storage-queue",
"structopt", "structopt",
"tokio 0.2.25", "tokio 1.5.0",
"url", "url",
"users", "users",
"uuid", "uuid",
@ -1815,7 +1886,7 @@ dependencies = [
"lazy_static", "lazy_static",
"log", "log",
"serde", "serde",
"tokio 0.2.25", "tokio 1.5.0",
"uuid", "uuid",
"z3-sys", "z3-sys",
] ]
@ -2328,9 +2399,9 @@ dependencies = [
"futures-core", "futures-core",
"futures-util", "futures-util",
"http", "http",
"http-body", "http-body 0.3.1",
"hyper", "hyper 0.13.10",
"hyper-tls", "hyper-tls 0.4.3",
"ipnet", "ipnet",
"js-sys", "js-sys",
"lazy_static", "lazy_static",
@ -2352,6 +2423,41 @@ dependencies = [
"winreg 0.7.0", "winreg 0.7.0",
] ]
[[package]]
name = "reqwest"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
dependencies = [
"base64",
"bytes 1.0.1",
"encoding_rs",
"futures-core",
"futures-util",
"http",
"http-body 0.4.1",
"hyper 0.14.5",
"hyper-tls 0.5.0",
"ipnet",
"js-sys",
"lazy_static",
"log",
"mime",
"native-tls",
"percent-encoding",
"pin-project-lite 0.2.6",
"serde",
"serde_json",
"serde_urlencoded",
"tokio 1.5.0",
"tokio-native-tls",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg 0.7.0",
]
[[package]] [[package]]
name = "reqwest-retry" name = "reqwest-retry"
version = "0.1.0" version = "0.1.0"
@ -2361,8 +2467,8 @@ dependencies = [
"backoff", "backoff",
"log", "log",
"onefuzz-telemetry", "onefuzz-telemetry",
"reqwest", "reqwest 0.11.3",
"tokio 0.2.25", "tokio 1.5.0",
] ]
[[package]] [[package]]
@ -2648,12 +2754,31 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "socket2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "spin" name = "spin"
version = "0.5.2" version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spinning_top"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bd0ab6b8c375d2d963503b90d3770010d95bc3b5f98036f948dee24bf4e8879"
dependencies = [
"lock_api",
]
[[package]] [[package]]
name = "stacktrace-parser" name = "stacktrace-parser"
version = "0.1.0" version = "0.1.0"
@ -2684,16 +2809,17 @@ dependencies = [
"base64", "base64",
"bytes 0.5.6", "bytes 0.5.6",
"derivative", "derivative",
"flume",
"num_cpus", "num_cpus",
"queue-file", "queue-file",
"regex", "regex",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"serde", "serde",
"serde-xml-rs", "serde-xml-rs",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"tokio 0.2.25", "tokio 1.5.0",
"uuid", "uuid",
"yaque", "yaque",
] ]
@ -2899,34 +3025,38 @@ dependencies = [
"futures-core", "futures-core",
"iovec", "iovec",
"lazy_static", "lazy_static",
"libc",
"memchr", "memchr",
"mio 0.6.23", "mio 0.6.23",
"mio-named-pipes",
"mio-uds",
"num_cpus", "num_cpus",
"pin-project-lite 0.1.12", "pin-project-lite 0.1.12",
"signal-hook-registry",
"slab", "slab",
]
[[package]]
name = "tokio"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5"
dependencies = [
"autocfg",
"bytes 1.0.1",
"libc",
"memchr",
"mio 0.7.10",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite 0.2.6",
"signal-hook-registry",
"tokio-macros", "tokio-macros",
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "tokio"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a"
dependencies = [
"autocfg",
"pin-project-lite 0.2.6",
]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "0.2.6" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [ dependencies = [
"proc-macro2 1.0.24", "proc-macro2 1.0.24",
"quote 1.0.9", "quote 1.0.9",
@ -2934,14 +3064,25 @@ dependencies = [
] ]
[[package]] [[package]]
name = "tokio-stream" name = "tokio-native-tls"
version = "0.1.3" version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio 1.5.0",
]
[[package]]
name = "tokio-stream"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e177a5d8c3bf36de9ebe6d58537d8879e964332f93fb3339e43f618c81361af0"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"pin-project-lite 0.2.6", "pin-project-lite 0.2.6",
"tokio 1.2.0", "tokio 1.5.0",
"tokio-util 0.6.6",
] ]
[[package]] [[package]]
@ -2962,13 +3103,28 @@ checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 0.5.6",
"futures-core", "futures-core",
"futures-io",
"futures-sink", "futures-sink",
"log", "log",
"pin-project-lite 0.1.12", "pin-project-lite 0.1.12",
"tokio 0.2.25", "tokio 0.2.25",
] ]
[[package]]
name = "tokio-util"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e"
dependencies = [
"bytes 1.0.1",
"futures-core",
"futures-io",
"futures-sink",
"log",
"pin-project-lite 0.2.6",
"slab",
"tokio 1.5.0",
]
[[package]] [[package]]
name = "tower-service" name = "tower-service"
version = "0.3.1" version = "0.3.1"

11
src/agent/onefuzz-agent/Cargo.toml
View File

@ -19,12 +19,13 @@ backoff = { version = "0.3", features = ["async-std"] }
clap = "2.33" clap = "2.33"
crossterm = "0.18" crossterm = "0.18"
env_logger = "0.8" env_logger = "0.8"
futures = "0.3" flume = "0.10.4"
futures = "0.3.14"
hex = "0.4" hex = "0.4"
lazy_static = "1.4" lazy_static = "1.4"
log = "0.4" log = "0.4"
num_cpus = "1.13" num_cpus = "1.13"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
serde = "1.0" serde = "1.0"
serde_json = "1.0" serde_json = "1.0"
onefuzz = { path = "../onefuzz" } onefuzz = { path = "../onefuzz" }
@ -35,9 +36,9 @@ stacktrace-parser = { path = "../stacktrace-parser" }
storage-queue = { path = "../storage-queue" } storage-queue = { path = "../storage-queue" }
tempfile = "3.2" tempfile = "3.2"
thiserror = "1.0" thiserror = "1.0"
tokio = { version = "0.2", features = ["full"] } tokio = { version = "1.5.0", features = ["full"] }
tokio-util = { version = "0.3", features = ["full"] } tokio-util = { version = "0.6.6", features = ["full"] }
tokio-stream = "0.1.3" tokio-stream = "0.1.5"
tui = { version = "0.14", default-features = false, features = ['crossterm'] } tui = { version = "0.14", default-features = false, features = ['crossterm'] }
url = { version = "2.2", features = ["serde"] } url = { version = "2.2", features = ["serde"] }
uuid = { version = "0.8", features = ["serde", "v4"] } uuid = { version = "0.8", features = ["serde", "v4"] }

12
src/agent/onefuzz-agent/src/local/common.rs
View File

@ -3,6 +3,7 @@ use crate::tasks::utils::parse_key_value;
use anyhow::Result; use anyhow::Result;
use backoff::{future::retry, Error as BackoffError, ExponentialBackoff}; use backoff::{future::retry, Error as BackoffError, ExponentialBackoff};
use clap::{App, Arg, ArgMatches}; use clap::{App, Arg, ArgMatches};
use flume::Sender;
use onefuzz::jitter::delay_with_jitter; use onefuzz::jitter::delay_with_jitter;
use onefuzz::{blob::url::BlobContainerUrl, monitor::DirectoryMonitor, syncdir::SyncedDir}; use onefuzz::{blob::url::BlobContainerUrl, monitor::DirectoryMonitor, syncdir::SyncedDir};
use path_absolutize::Absolutize; use path_absolutize::Absolutize;
@ -14,7 +15,6 @@ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
time::Duration, time::Duration,
}; };
use tokio::sync::mpsc::UnboundedSender;
use uuid::Uuid; use uuid::Uuid;
pub const SETUP_DIR: &str = "setup_dir"; pub const SETUP_DIR: &str = "setup_dir";
@ -69,7 +69,7 @@ pub enum CmdType {
pub struct LocalContext { pub struct LocalContext {
pub job_path: PathBuf, pub job_path: PathBuf,
pub common_config: CommonConfig, pub common_config: CommonConfig,
pub event_sender: Option<UnboundedSender<UiEvent>>, pub event_sender: Option<Sender<UiEvent>>,
} }
pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result<HashMap<String, String>> { pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result<HashMap<String, String>> {
@ -213,7 +213,7 @@ pub fn get_synced_dir(
pub fn build_local_context( pub fn build_local_context(
args: &ArgMatches<'_>, args: &ArgMatches<'_>,
generate_task_id: bool, generate_task_id: bool,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<LocalContext> { ) -> Result<LocalContext> {
let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil()); let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil());
let task_id = get_uuid("task_id", args).unwrap_or_else(|_| { let task_id = get_uuid("task_id", args).unwrap_or_else(|_| {
@ -322,11 +322,11 @@ pub enum UiEvent {
} }
pub trait SyncCountDirMonitor<T: Sized> { pub trait SyncCountDirMonitor<T: Sized> {
fn monitor_count(self, event_sender: &Option<UnboundedSender<UiEvent>>) -> Result<T>; fn monitor_count(self, event_sender: &Option<Sender<UiEvent>>) -> Result<T>;
} }
impl SyncCountDirMonitor<SyncedDir> for SyncedDir { impl SyncCountDirMonitor<SyncedDir> for SyncedDir {
fn monitor_count(self, event_sender: &Option<UnboundedSender<UiEvent>>) -> Result<Self> { fn monitor_count(self, event_sender: &Option<Sender<UiEvent>>) -> Result<Self> {
if let (Some(event_sender), Some(p)) = (event_sender, self.remote_url()?.as_file_path()) { if let (Some(event_sender), Some(p)) = (event_sender, self.remote_url()?.as_file_path()) {
event_sender.send(UiEvent::MonitorDir(p))?; event_sender.send(UiEvent::MonitorDir(p))?;
} }
@ -335,7 +335,7 @@ impl SyncCountDirMonitor<SyncedDir> for SyncedDir {
} }
impl SyncCountDirMonitor<Option<SyncedDir>> for Option<SyncedDir> { impl SyncCountDirMonitor<Option<SyncedDir>> for Option<SyncedDir> {
fn monitor_count(self, event_sender: &Option<UnboundedSender<UiEvent>>) -> Result<Self> { fn monitor_count(self, event_sender: &Option<Sender<UiEvent>>) -> Result<Self> {
if let Some(sd) = self { if let Some(sd) = self {
let sd = sd.monitor_count(event_sender)?; let sd = sd.monitor_count(event_sender)?;
Ok(Some(sd)) Ok(Some(sd))

9
src/agent/onefuzz-agent/src/local/generic_analysis.rs
View File

@ -15,14 +15,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use storage_queue::QueueClient; use storage_queue::QueueClient;
use tokio::sync::mpsc::UnboundedSender;
pub fn build_analysis_config( pub fn build_analysis_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
input_queue: Option<QueueClient>, input_queue: Option<QueueClient>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_options = get_cmd_arg(CmdType::Target, args); let target_options = get_cmd_arg(CmdType::Target, args);
@ -69,10 +69,7 @@ pub fn build_analysis_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_analysis_config(args, None, context.common_config.clone(), event_sender)?; let config = build_analysis_config(args, None, context.common_config.clone(), event_sender)?;
run_analysis(config).await run_analysis(config).await

9
src/agent/onefuzz-agent/src/local/generic_crash_report.rs
View File

@ -15,14 +15,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use storage_queue::QueueClient; use storage_queue::QueueClient;
use tokio::sync::mpsc::UnboundedSender;
pub fn build_report_config( pub fn build_report_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
input_queue: Option<QueueClient>, input_queue: Option<QueueClient>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -78,10 +78,7 @@ pub fn build_report_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; let config = build_report_config(args, None, context.common_config.clone(), event_sender)?;
ReportTask::new(config).managed_run().await ReportTask::new(config).managed_run().await

9
src/agent/onefuzz-agent/src/local/generic_generator.rs
View File

@ -16,12 +16,12 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use tokio::sync::mpsc::UnboundedSender; use flume::Sender;
pub fn build_fuzz_config( pub fn build_fuzz_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?
.monitor_count(&event_sender)?; .monitor_count(&event_sender)?;
@ -71,10 +71,7 @@ pub fn build_fuzz_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?;
GeneratorTask::new(config).run().await GeneratorTask::new(config).run().await

8
src/agent/onefuzz-agent/src/local/libfuzzer.rs
View File

@ -23,15 +23,13 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, SubCommand}; use clap::{App, SubCommand};
use flume::Sender;
use onefuzz::utils::try_wait_all_join_handles; use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet; use std::collections::HashSet;
use tokio::{sync::mpsc::UnboundedSender, task::spawn}; use tokio::task::spawn;
use uuid::Uuid; use uuid::Uuid;
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?;
let crash_dir = fuzz_config let crash_dir = fuzz_config

9
src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs
View File

@ -14,8 +14,8 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use storage_queue::QueueClient; use storage_queue::QueueClient;
use tokio::sync::mpsc::UnboundedSender;
use super::common::{SyncCountDirMonitor, UiEvent}; use super::common::{SyncCountDirMonitor, UiEvent};
@ -24,7 +24,7 @@ pub fn build_coverage_config(
local_job: bool, local_job: bool,
input_queue: Option<QueueClient>, input_queue: Option<QueueClient>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -61,10 +61,7 @@ pub fn build_coverage_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_coverage_config( let config = build_coverage_config(
args, args,

9
src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs
View File

@ -15,14 +15,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use storage_queue::QueueClient; use storage_queue::QueueClient;
use tokio::sync::mpsc::UnboundedSender;
pub fn build_report_config( pub fn build_report_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
input_queue: Option<QueueClient>, input_queue: Option<QueueClient>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -73,10 +73,7 @@ pub fn build_report_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; let config = build_report_config(args, None, context.common_config.clone(), event_sender)?;
ReportTask::new(config).managed_run().await ReportTask::new(config).managed_run().await

9
src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs
View File

@ -14,14 +14,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use tokio::sync::mpsc::UnboundedSender; use flume::Sender;
const EXPECT_CRASH_ON_FAILURE: &str = "expect_crash_on_failure"; const EXPECT_CRASH_ON_FAILURE: &str = "expect_crash_on_failure";
pub fn build_fuzz_config( pub fn build_fuzz_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?
.monitor_count(&event_sender)?; .monitor_count(&event_sender)?;
@ -56,10 +56,7 @@ pub fn build_fuzz_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?;
LibFuzzerFuzzTask::new(config)?.run().await LibFuzzerFuzzTask::new(config)?.run().await

9
src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs
View File

@ -15,14 +15,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use storage_queue::QueueClient; use storage_queue::QueueClient;
use tokio::sync::mpsc::UnboundedSender;
pub fn build_merge_config( pub fn build_merge_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
input_queue: Option<QueueClient>, input_queue: Option<QueueClient>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -52,10 +52,7 @@ pub fn build_merge_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_merge_config(args, None, context.common_config.clone(), event_sender)?; let config = build_merge_config(args, None, context.common_config.clone(), event_sender)?;
spawn(std::sync::Arc::new(config)).await spawn(std::sync::Arc::new(config)).await

9
src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs
View File

@ -15,14 +15,14 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use tokio::sync::mpsc::UnboundedSender; use flume::Sender;
const REPORT_NAMES: &str = "report_names"; const REPORT_NAMES: &str = "report_names";
pub fn build_regression_config( pub fn build_regression_config(
args: &clap::ArgMatches<'_>, args: &clap::ArgMatches<'_>,
common: CommonConfig, common: CommonConfig,
event_sender: Option<UnboundedSender<UiEvent>>, event_sender: Option<Sender<UiEvent>>,
) -> Result<Config> { ) -> Result<Config> {
let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into();
let target_env = get_cmd_env(CmdType::Target, args)?; let target_env = get_cmd_env(CmdType::Target, args)?;
@ -73,10 +73,7 @@ pub fn build_regression_config(
Ok(config) Ok(config)
} }
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let config = build_regression_config(args, context.common_config.clone(), event_sender)?; let config = build_regression_config(args, context.common_config.clone(), event_sender)?;
LibFuzzerRegressionTask::new(config).run().await LibFuzzerRegressionTask::new(config).run().await

7
src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs
View File

@ -10,13 +10,10 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use std::path::PathBuf; use std::path::PathBuf;
use tokio::sync::mpsc::UnboundedSender;
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender)?; let context = build_local_context(args, true, event_sender)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;

7
src/agent/onefuzz-agent/src/local/radamsa.rs
View File

@ -11,16 +11,13 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, SubCommand}; use clap::{App, SubCommand};
use flume::Sender;
use onefuzz::utils::try_wait_all_join_handles; use onefuzz::utils::try_wait_all_join_handles;
use std::collections::HashSet; use std::collections::HashSet;
use tokio::sync::mpsc::UnboundedSender;
use tokio::task::spawn; use tokio::task::spawn;
use uuid::Uuid; use uuid::Uuid;
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, true, event_sender.clone())?; let context = build_local_context(args, true, event_sender.clone())?;
let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?;
let crash_dir = fuzz_config let crash_dir = fuzz_config

7
src/agent/onefuzz-agent/src/local/test_input.rs
View File

@ -11,13 +11,10 @@ use crate::{
}; };
use anyhow::Result; use anyhow::Result;
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use flume::Sender;
use std::path::PathBuf; use std::path::PathBuf;
use tokio::sync::mpsc::UnboundedSender;
pub async fn run( pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option<Sender<UiEvent>>) -> Result<()> {
args: &clap::ArgMatches<'_>,
event_sender: Option<UnboundedSender<UiEvent>>,
) -> Result<()> {
let context = build_local_context(args, false, event_sender)?; let context = build_local_context(args, false, event_sender)?;
let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?;

70
src/agent/onefuzz-agent/src/local/tui.rs
View File

@ -21,12 +21,11 @@ use std::{
thread::{self, JoinHandle}, thread::{self, JoinHandle},
time::Duration, time::Duration,
}; };
use flume::{Receiver, Sender};
use tokio::{ use tokio::{
sync::{ sync::broadcast::{self, error::TryRecvError},
broadcast::{self, TryRecvError}, time::sleep,
mpsc::{self, UnboundedSender},
},
time::delay_for,
}; };
use tui::{ use tui::{
backend::CrosstermBackend, backend::CrosstermBackend,
@ -91,7 +90,7 @@ struct UiLoopState {
pub file_count: HashMap<PathBuf, usize>, pub file_count: HashMap<PathBuf, usize>,
pub file_count_state: ListState, pub file_count_state: ListState,
pub file_monitors: HashMap<PathBuf, tokio::task::JoinHandle<Result<()>>>, pub file_monitors: HashMap<PathBuf, tokio::task::JoinHandle<Result<()>>>,
pub log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, pub log_event_receiver: Receiver<(Level, String)>,
pub terminal: Terminal<CrosstermBackend<Stdout>>, pub terminal: Terminal<CrosstermBackend<Stdout>>,
pub cancellation_tx: broadcast::Sender<()>, pub cancellation_tx: broadcast::Sender<()>,
pub events: HashMap<Discriminant<EventData>, EventData>, pub events: HashMap<Discriminant<EventData>, EventData>,
@ -100,7 +99,7 @@ struct UiLoopState {
impl UiLoopState { impl UiLoopState {
fn new( fn new(
terminal: Terminal<CrosstermBackend<Stdout>>, terminal: Terminal<CrosstermBackend<Stdout>>,
log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, log_event_receiver: Receiver<(Level, String)>,
) -> Self { ) -> Self {
let (cancellation_tx, _) = broadcast::channel(1); let (cancellation_tx, _) = broadcast::channel(1);
let events = HashMap::new(); let events = HashMap::new();
@ -118,16 +117,16 @@ impl UiLoopState {
} }
pub struct TerminalUi { pub struct TerminalUi {
pub task_events: mpsc::UnboundedSender<UiEvent>, pub task_events: Sender<UiEvent>,
task_event_receiver: mpsc::UnboundedReceiver<UiEvent>, task_event_receiver: Receiver<UiEvent>,
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>, ui_event_rx: Receiver<TerminalEvent>,
} }
impl TerminalUi { impl TerminalUi {
pub fn init() -> Result<Self> { pub fn init() -> Result<Self> {
let (task_event_sender, task_event_receiver) = mpsc::unbounded_channel(); let (task_event_sender, task_event_receiver) = flume::unbounded();
let (ui_event_tx, ui_event_rx) = mpsc::unbounded_channel(); let (ui_event_tx, ui_event_rx) = flume::unbounded();
Ok(Self { Ok(Self {
task_events: task_event_sender, task_events: task_event_sender,
task_event_receiver, task_event_receiver,
@ -144,7 +143,7 @@ impl TerminalUi {
let backend = CrosstermBackend::new(stdout); let backend = CrosstermBackend::new(stdout);
let mut terminal = Terminal::new(backend)?; let mut terminal = Terminal::new(backend)?;
terminal.clear()?; terminal.clear()?;
let (log_event_sender, log_event_receiver) = mpsc::unbounded_channel(); let (log_event_sender, log_event_receiver) = flume::unbounded();
let initial_state = UiLoopState::new(terminal, log_event_receiver); let initial_state = UiLoopState::new(terminal, log_event_receiver);
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
@ -196,7 +195,7 @@ impl TerminalUi {
if let Some(timeout) = timeout { if let Some(timeout) = timeout {
let ui_event_tx = self.ui_event_tx.clone(); let ui_event_tx = self.ui_event_tx.clone();
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::delay_for(timeout).await; tokio::time::sleep(timeout).await;
let _ = ui_event_tx.send(TerminalEvent::Quit); let _ = ui_event_tx.send(TerminalEvent::Quit);
}); });
} }
@ -226,12 +225,12 @@ impl TerminalUi {
} }
async fn listen_telemetry_event( async fn listen_telemetry_event(
ui_event_tx: UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
mut cancellation_rx: broadcast::Receiver<()>, mut cancellation_rx: broadcast::Receiver<()>,
) -> Result<()> { ) -> Result<()> {
let mut rx = onefuzz_telemetry::subscribe_to_events(); let mut rx = onefuzz_telemetry::subscribe_to_events();
while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { while cancellation_rx.try_recv() == Err(broadcast::error::TryRecvError::Empty) {
match rx.try_recv() { match rx.try_recv() {
Ok((_event, data)) => { Ok((_event, data)) => {
let data = data let data = data
@ -240,7 +239,7 @@ impl TerminalUi {
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let _ = ui_event_tx.send(TerminalEvent::Telemetry(data)); let _ = ui_event_tx.send(TerminalEvent::Telemetry(data));
} }
Err(TryRecvError::Empty) => delay_for(EVENT_POLLING_PERIOD).await, Err(TryRecvError::Empty) => sleep(EVENT_POLLING_PERIOD).await,
Err(TryRecvError::Lagged(_)) => continue, Err(TryRecvError::Lagged(_)) => continue,
Err(TryRecvError::Closed) => break, Err(TryRecvError::Closed) => break,
} }
@ -249,11 +248,11 @@ impl TerminalUi {
} }
async fn ticking( async fn ticking(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
mut cancellation_rx: broadcast::Receiver<()>, mut cancellation_rx: broadcast::Receiver<()>,
) -> Result<()> { ) -> Result<()> {
let mut interval = tokio::time::interval(TICK_RATE); let mut interval = tokio::time::interval(TICK_RATE);
while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { while Err(broadcast::error::TryRecvError::Empty) == cancellation_rx.try_recv() {
interval.tick().await; interval.tick().await;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Tick) { if let Err(_err) = ui_event_tx.send(TerminalEvent::Tick) {
break; break;
@ -263,11 +262,11 @@ impl TerminalUi {
} }
fn read_keyboard_events( fn read_keyboard_events(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
mut cancellation_rx: broadcast::Receiver<()>, mut cancellation_rx: broadcast::Receiver<()>,
) -> JoinHandle<Result<()>> { ) -> JoinHandle<Result<()>> {
thread::spawn(move || { thread::spawn(move || {
while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { while Err(broadcast::error::TryRecvError::Empty) == cancellation_rx.try_recv() {
if event::poll(EVENT_POLLING_PERIOD)? { if event::poll(EVENT_POLLING_PERIOD)? {
let event = event::read()?; let event = event::read()?;
if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) { if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) {
@ -280,26 +279,26 @@ impl TerminalUi {
} }
async fn read_commands( async fn read_commands(
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
mut external_event_rx: mpsc::UnboundedReceiver<UiEvent>, external_event_rx: Receiver<UiEvent>,
mut cancellation_rx: broadcast::Receiver<()>, mut cancellation_rx: broadcast::Receiver<()>,
) -> Result<()> { ) -> Result<()> {
while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { while Err(broadcast::error::TryRecvError::Empty) == cancellation_rx.try_recv() {
match external_event_rx.try_recv() { match external_event_rx.try_recv() {
Ok(UiEvent::MonitorDir(dir)) => { Ok(UiEvent::MonitorDir(dir)) => {
if ui_event_tx.send(TerminalEvent::MonitorDir(dir)).is_err() { if ui_event_tx.send(TerminalEvent::MonitorDir(dir)).is_err() {
break; break;
} }
} }
Err(mpsc::error::TryRecvError::Empty) => delay_for(EVENT_POLLING_PERIOD).await, Err(flume::TryRecvError::Empty) => sleep(EVENT_POLLING_PERIOD).await,
Err(mpsc::error::TryRecvError::Closed) => break, Err(flume::TryRecvError::Disconnected) => break,
} }
} }
Ok(()) Ok(())
} }
fn take_available_logs<T>( fn take_available_logs<T>(
receiver: &mut mpsc::UnboundedReceiver<T>, receiver: &mut Receiver<T>,
size: usize, size: usize,
buffer: &mut ArrayDeque<[T; LOGS_BUFFER_SIZE], Wrapping>, buffer: &mut ArrayDeque<[T; LOGS_BUFFER_SIZE], Wrapping>,
) { ) {
@ -565,7 +564,7 @@ impl TerminalUi {
async fn on_monitor_dir( async fn on_monitor_dir(
ui_state: UiLoopState, ui_state: UiLoopState,
path: PathBuf, path: PathBuf,
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
cancellation_rx: broadcast::Receiver<()>, cancellation_rx: broadcast::Receiver<()>,
) -> Result<UiLoopState, UiLoopError> { ) -> Result<UiLoopState, UiLoopError> {
let mut file_monitors = ui_state.file_monitors; let mut file_monitors = ui_state.file_monitors;
@ -582,10 +581,11 @@ impl TerminalUi {
async fn ui_loop( async fn ui_loop(
initial_state: UiLoopState, initial_state: UiLoopState,
ui_event_rx: mpsc::UnboundedReceiver<TerminalEvent>, ui_event_rx: Receiver<TerminalEvent>,
ui_event_tx: mpsc::UnboundedSender<TerminalEvent>, ui_event_tx: Sender<TerminalEvent>,
) -> Result<()> { ) -> Result<()> {
let loop_result = ui_event_rx let loop_result = ui_event_rx
.stream()
.map(Ok) .map(Ok)
.try_fold(initial_state, |ui_state, event| async { .try_fold(initial_state, |ui_state, event| async {
let ui_event_tx = ui_event_tx.clone(); let ui_event_tx = ui_event_tx.clone();
@ -632,16 +632,16 @@ impl TerminalUi {
fn spawn_file_count_monitor( fn spawn_file_count_monitor(
dir: PathBuf, dir: PathBuf,
sender: mpsc::UnboundedSender<TerminalEvent>, sender: Sender<TerminalEvent>,
mut cancellation_rx: broadcast::Receiver<()>, mut cancellation_rx: broadcast::Receiver<()>,
) -> tokio::task::JoinHandle<Result<()>> { ) -> tokio::task::JoinHandle<Result<()>> {
tokio::spawn(async move { tokio::spawn(async move {
wait_for_dir(&dir).await?; wait_for_dir(&dir).await?;
while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { while cancellation_rx.try_recv() == Err(broadcast::error::TryRecvError::Empty) {
let mut rd = tokio::fs::read_dir(&dir).await?; let mut rd = tokio::fs::read_dir(&dir).await?;
let mut count: usize = 0; let mut count: usize = 0;
while let Some(Ok(entry)) = rd.next().await { while let Ok(Some(entry)) = rd.next_entry().await {
if entry.path().is_file() { if entry.path().is_file() {
count += 1; count += 1;
} }
@ -657,7 +657,7 @@ impl TerminalUi {
break; break;
} }
delay_for(FILE_MONITOR_POLLING_PERIOD).await; sleep(FILE_MONITOR_POLLING_PERIOD).await;
} }
Ok(()) Ok(())
}) })

2
src/agent/onefuzz-agent/src/main.rs
View File

@ -37,7 +37,7 @@ fn main() -> Result<()> {
let matches = app.get_matches(); let matches = app.get_matches();
let mut rt = tokio::runtime::Runtime::new()?; let rt = tokio::runtime::Runtime::new()?;
let result = rt.block_on(run(matches)); let result = rt.block_on(run(matches));
atexit::execute(); atexit::execute();
result result

4
src/agent/onefuzz-agent/src/tasks/analysis/generic.rs
View File

@ -5,7 +5,6 @@ use crate::tasks::{
config::CommonConfig, heartbeat::HeartbeatSender, report::crash_report::monitor_reports, config::CommonConfig, heartbeat::HeartbeatSender, report::crash_report::monitor_reports,
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use futures::stream::StreamExt;
use onefuzz::{az_copy, blob::url::BlobUrl}; use onefuzz::{az_copy, blob::url::BlobUrl};
use onefuzz::{ use onefuzz::{
expand::Expand, expand::Expand,
@ -116,9 +115,8 @@ async fn run_existing(config: &Config, reports_dir: &Option<PathBuf>) -> Result<
crashes.init_pull().await?; crashes.init_pull().await?;
let mut count: u64 = 0; let mut count: u64 = 0;
let mut read_dir = fs::read_dir(&crashes.path).await?; let mut read_dir = fs::read_dir(&crashes.path).await?;
while let Some(file) = read_dir.next().await { while let Some(file) = read_dir.next_entry().await? {
debug!("Processing file {:?}", file); debug!("Processing file {:?}", file);
let file = file?;
run_tool(file.path(), &config, &reports_dir).await?; run_tool(file.path(), &config, &reports_dir).await?;
count += 1; count += 1;
} }

8
src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs
View File

@ -38,7 +38,6 @@ use crate::tasks::{
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use async_trait::async_trait; use async_trait::async_trait;
use futures::stream::StreamExt;
use onefuzz::{fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir}; use onefuzz::{fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir};
use onefuzz_telemetry::{Event::coverage_data, EventData}; use onefuzz_telemetry::{Event::coverage_data, EventData};
use reqwest::Url; use reqwest::Url;
@ -159,9 +158,10 @@ impl CoverageTask {
})?; })?;
let mut seen_inputs = false; let mut seen_inputs = false;
while let Some(input) = corpus.next().await { loop {
let input = match input { let input = match corpus.next_entry().await {
Ok(input) => input, Ok(Some(input)) => input,
Ok(None) => break,
Err(err) => { Err(err) => {
error!("{}", err); error!("{}", err);
continue; continue;

5
src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs
View File

@ -7,7 +7,6 @@ use crate::tasks::{
utils::{self, default_bool_true}, utils::{self, default_bool_true},
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use futures::stream::StreamExt;
use onefuzz::{ use onefuzz::{
expand::Expand, expand::Expand,
fs::set_executable, fs::set_executable,
@ -122,9 +121,7 @@ impl GeneratorTask {
tester: &Tester<'_>, tester: &Tester<'_>,
) -> Result<()> { ) -> Result<()> {
let mut read_dir = fs::read_dir(generated_inputs).await?; let mut read_dir = fs::read_dir(generated_inputs).await?;
while let Some(file) = read_dir.next().await { while let Some(file) = read_dir.next_entry().await? {
let file = file?;
debug!("testing input: {:?}", file); debug!("testing input: {:?}", file);
let destination_file = if self.config.rename_output { let destination_file = if self.config.rename_output {

22
src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs
View File

@ -4,7 +4,7 @@
use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender, utils::default_bool_true}; use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender, utils::default_bool_true};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use arraydeque::{ArrayDeque, Wrapping}; use arraydeque::{ArrayDeque, Wrapping};
use futures::{future::try_join_all, stream::StreamExt}; use futures::future::try_join_all;
use onefuzz::{ use onefuzz::{
fs::list_files, fs::list_files,
libfuzzer::{LibFuzzer, LibFuzzerLine}, libfuzzer::{LibFuzzer, LibFuzzerLine},
@ -23,7 +23,7 @@ use tokio::{
io::{AsyncBufReadExt, BufReader}, io::{AsyncBufReadExt, BufReader},
sync::mpsc, sync::mpsc,
task, task,
time::{delay_for, Duration}, time::{sleep, Duration},
}; };
use uuid::Uuid; use uuid::Uuid;
@ -160,7 +160,7 @@ impl LibFuzzerFuzzTask {
.await?; .await?;
let mut entries = tokio::fs::read_dir(local_input_dir.path()).await?; let mut entries = tokio::fs::read_dir(local_input_dir.path()).await?;
while let Some(Ok(entry)) = entries.next().await { while let Ok(Some(entry)) = entries.next_entry().await {
let destination_path = self.config.inputs.path.clone().join(entry.file_name()); let destination_path = self.config.inputs.path.clone().join(entry.file_name());
tokio::fs::rename(&entry.path(), &destination_path) tokio::fs::rename(&entry.path(), &destination_path)
.await .await
@ -203,7 +203,11 @@ impl LibFuzzerFuzzTask {
let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?; let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?;
let running_id = running.id(); let running_id = running.id();
let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running_id)); let sys_info = task::spawn(report_fuzzer_sys_info(
worker_id,
run_id,
running_id.unwrap_or_default(),
));
// Splitting borrow. // Splitting borrow.
let stderr = running let stderr = running
@ -229,7 +233,7 @@ impl LibFuzzerFuzzTask {
libfuzzer_output.push_back(line); libfuzzer_output.push_back(line);
} }
let (exit_status, _) = tokio::join!(running, sys_info); let (exit_status, _) = tokio::join!(running.wait(), sys_info);
let exit_status: ExitStatus = exit_status?.into(); let exit_status: ExitStatus = exit_status?.into();
let files = list_files(crash_dir.path()).await?; let files = list_files(crash_dir.path()).await?;
@ -311,7 +315,7 @@ fn try_report_iter_update(
async fn report_fuzzer_sys_info(worker_id: usize, run_id: Uuid, fuzzer_pid: u32) -> Result<()> { async fn report_fuzzer_sys_info(worker_id: usize, run_id: Uuid, fuzzer_pid: u32) -> Result<()> {
// Allow for sampling CPU usage. // Allow for sampling CPU usage.
delay_for(PROC_INFO_COLLECTION_DELAY).await; sleep(PROC_INFO_COLLECTION_DELAY).await;
loop { loop {
// process doesn't exist // process doesn't exist
@ -335,7 +339,7 @@ async fn report_fuzzer_sys_info(worker_id: usize, run_id: Uuid, fuzzer_pid: u32)
break; break;
} }
delay_for(PROC_INFO_PERIOD).await; sleep(PROC_INFO_PERIOD).await;
} }
Ok(()) Ok(())
@ -396,7 +400,7 @@ impl Timer {
} }
async fn wait(&self) { async fn wait(&self) {
delay_for(self.interval).await; sleep(self.interval).await;
} }
} }
@ -424,7 +428,7 @@ async fn report_runtime_stats(
loop { loop {
tokio::select! { tokio::select! {
Some(stats) = stats_channel.next() => { Some(stats) = stats_channel.recv() => {
heartbeat_client.alive(); heartbeat_client.alive();
total.update(stats); total.update(stats);
total.report() total.report()

2
src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs
View File

@ -362,7 +362,7 @@ mod tests {
MAX_FUZZ_TIME_SECONDS MAX_FUZZ_TIME_SECONDS
); );
} }
tokio::time::delay_for(std::time::Duration::from_secs(1)).await; tokio::time::sleep(std::time::Duration::from_secs(1)).await;
} }
} }
} }

5
src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs
View File

@ -4,7 +4,6 @@
use std::{fmt, path::PathBuf}; use std::{fmt, path::PathBuf};
use anyhow::Result; use anyhow::Result;
use futures::stream::StreamExt;
use onefuzz::{blob::BlobUrl, jitter::delay_with_jitter, syncdir::SyncedDir}; use onefuzz::{blob::BlobUrl, jitter::delay_with_jitter, syncdir::SyncedDir};
use reqwest::Url; use reqwest::Url;
use tempfile::{tempdir, TempDir}; use tempfile::{tempdir, TempDir};
@ -133,8 +132,8 @@ impl<M> InputPoller<M> {
); );
let mut read_dir = fs::read_dir(&to_process.path).await?; let mut read_dir = fs::read_dir(&to_process.path).await?;
while let Some(file) = read_dir.next().await { while let Some(file) = read_dir.next_entry().await? {
let path = file?.path(); let path = file.path();
info!( info!(
"processing batch-downloaded input: {} - {}", "processing batch-downloaded input: {} - {}",
self.name, self.name,

6
src/agent/onefuzz-supervisor/Cargo.toml
View File

@ -12,15 +12,15 @@ appinsights = "0.1"
async-trait = "0.1" async-trait = "0.1"
downcast-rs = "1.2" downcast-rs = "1.2"
env_logger = "0.8" env_logger = "0.8"
futures = "0.3" futures = "0.3.14"
log = "0.4" log = "0.4"
onefuzz = { path = "../onefuzz" } onefuzz = { path = "../onefuzz" }
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
storage-queue = { path = "../storage-queue" } storage-queue = { path = "../storage-queue" }
structopt = "0.3" structopt = "0.3"
tokio = { version = "0.2", features = ["full"] } tokio = { version = "1.5.0", features = ["full"] }
url = { version = "2.2", features = ["serde"] } url = { version = "2.2", features = ["serde"] }
uuid = { version = "0.8", features = ["serde", "v4"] } uuid = { version = "0.8", features = ["serde", "v4"] }
clap = "2.33" clap = "2.33"

6
src/agent/onefuzz-supervisor/src/agent.rs
View File

@ -277,7 +277,7 @@ impl Agent {
async fn sleep(&mut self) { async fn sleep(&mut self) {
let delay = time::Duration::from_secs(30); let delay = time::Duration::from_secs(30);
time::delay_for(delay).await; time::sleep(delay).await;
} }
fn scheduler(&mut self) -> Result<&mut Scheduler> { fn scheduler(&mut self) -> Result<&mut Scheduler> {
@ -285,9 +285,9 @@ impl Agent {
} }
} }
fn command_delay() -> time::Delay { fn command_delay() -> time::Sleep {
let delay = time::Duration::from_secs(10); let delay = time::Duration::from_secs(10);
time::delay_for(delay) time::sleep(delay)
} }
// The agent owns a `Scheduler`, which it must consume when driving its state // The agent owns a `Scheduler`, which it must consume when driving its state

2
src/agent/onefuzz-supervisor/src/debug.rs
View File

@ -163,7 +163,7 @@ fn debug_run_worker(opt: RunWorkerOpt) -> Result<()> {
work_units: vec![work_unit], work_units: vec![work_unit],
}; };
let mut rt = tokio::runtime::Runtime::new()?; let rt = tokio::runtime::Runtime::new()?;
let events = rt.block_on(run_worker(work_set))?; let events = rt.block_on(run_worker(work_set))?;
for event in events { for event in events {

2
src/agent/onefuzz-supervisor/src/main.rs
View File

@ -104,7 +104,7 @@ fn run(opt: RunOpt) -> Result<()> {
let config = config?; let config = config?;
let mut rt = tokio::runtime::Runtime::new()?; let rt = tokio::runtime::Runtime::new()?;
let result = rt.block_on(run_agent(config)); let result = rt.block_on(run_agent(config));
if let Err(err) = &result { if let Err(err) = &result {

2
src/agent/onefuzz-supervisor/src/setup.rs
View File

@ -132,7 +132,7 @@ async fn create_setup_symlink(setup_dir: &Path, work_unit: &WorkUnit) -> Result<
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
async fn create_setup_symlink(setup_dir: &Path, work_unit: &WorkUnit) -> Result<()> { async fn create_setup_symlink(setup_dir: &Path, work_unit: &WorkUnit) -> Result<()> {
use tokio::fs::os::unix::symlink; use tokio::fs::symlink;
let working_dir = work_unit.working_dir()?; let working_dir = work_unit.working_dir()?;

2
src/agent/onefuzz-telemetry/Cargo.toml
View File

@ -17,7 +17,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
z3-sys = { version = "0.6", optional = true} z3-sys = { version = "0.6", optional = true}
iced-x86 = { version = "1.1", optional = true} iced-x86 = { version = "1.1", optional = true}
tokio = { version = "0.2" } tokio = { version = "1.5.0", features = ["full"] }
lazy_static = "1.4" lazy_static = "1.4"

12
src/agent/onefuzz/Cargo.toml
View File

@ -11,15 +11,16 @@ anyhow = "1.0"
appinsights = "0.1" appinsights = "0.1"
async-trait = "0.1" async-trait = "0.1"
base64 = "0.13" base64 = "0.13"
bytes = "0.5" bytes = "1.0.1"
dunce = "1.0" dunce = "1.0"
futures = "0.3" futures = "0.3.14"
futures-util = "0.3.14"
hex = "0.4" hex = "0.4"
lazy_static = "1.4" lazy_static = "1.4"
log = "0.4" log = "0.4"
notify = "4.0" notify = "4.0"
regex = "1.4" regex = "1.4"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
sha2 = "0.9" sha2 = "0.9"
url = { version = "2.2", features = ["serde"] } url = { version = "2.2", features = ["serde"] }
serde = "1.0" serde = "1.0"
@ -27,8 +28,9 @@ serde_json = "1.0"
rand = "0.7" rand = "0.7"
serde_derive = "1.0" serde_derive = "1.0"
sysinfo = "0.16" sysinfo = "0.16"
tokio = { version = "0.2", features = ["full"] } tokio = { version = "1.5.0", features = ["full"] }
tokio-util = { version = "0.3", features = ["full"] } tokio-stream = { version = "0.1.5", features = ["fs", "time", "tokio-util"] }
tokio-util = { version = "0.6.6", features = ["full"] }
uuid = { version = "0.8", features = ["serde", "v4"] } uuid = { version = "0.8", features = ["serde", "v4"] }
urlparse = "0.7" urlparse = "0.7"
ring = "0.16" ring = "0.16"

5
src/agent/onefuzz/src/asan.rs
View File

@ -3,7 +3,7 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use std::{collections::HashMap, hash::BuildHasher, path::Path}; use std::{collections::HashMap, hash::BuildHasher, path::Path};
use tokio::{fs, stream::StreamExt}; use tokio::fs;
pub use stacktrace_parser::CrashLog; pub use stacktrace_parser::CrashLog;
const ASAN_LOG_TRUNCATE_SIZE: usize = 4096; const ASAN_LOG_TRUNCATE_SIZE: usize = 4096;
@ -68,8 +68,7 @@ pub async fn check_asan_string(mut data: String) -> Result<Option<CrashLog>> {
pub async fn check_asan_path(asan_dir: &Path) -> Result<Option<CrashLog>> { pub async fn check_asan_path(asan_dir: &Path) -> Result<Option<CrashLog>> {
let mut entries = fs::read_dir(asan_dir).await?; let mut entries = fs::read_dir(asan_dir).await?;
// there should be only up to one file in asan_dir // there should be only up to one file in asan_dir
if let Some(file) = entries.next().await { if let Some(file) = entries.next_entry().await? {
let file = file?;
let mut asan_text = fs::read_to_string(file.path()).await?; let mut asan_text = fs::read_to_string(file.path()).await?;
let asan = CrashLog::parse(asan_text.clone()).with_context(|| { let asan = CrashLog::parse(asan_text.clone()).with_context(|| {

7
src/agent/onefuzz/src/fs.rs
View File

@ -7,6 +7,7 @@ use std::{
ffi::OsStr, ffi::OsStr,
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use tokio_stream::wrappers::ReadDirStream;
use std::process::Stdio; use std::process::Stdio;
use tokio::fs; use tokio::fs;
@ -41,7 +42,7 @@ pub async fn list_files(path: impl AsRef<Path>) -> Result<Vec<PathBuf>> {
.await .await
.with_context(|| format!("unable to list files: {}", path.display()))?; .with_context(|| format!("unable to list files: {}", path.display()))?;
let mut files = paths let mut files = ReadDirStream::new(paths)
.filter_map(|x| async { .filter_map(|x| async {
match x { match x {
Ok(x) => { Ok(x) => {
@ -298,7 +299,7 @@ pub async fn copy(src: SyncPath, dst: SyncPath, recur: bool) -> Result<()> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use tempfile::tempdir; use tempfile::tempdir;
use tokio::{fs, stream::StreamExt}; use tokio::fs;
use super::*; use super::*;
@ -306,7 +307,7 @@ mod tests {
let mut len = 0; let mut len = 0;
let mut entries = fs::read_dir(dir).await.unwrap(); let mut entries = fs::read_dir(dir).await.unwrap();
while entries.next().await.is_some() { while entries.next_entry().await.unwrap().is_some() {
len += 1; len += 1;
} }

2
src/agent/onefuzz/src/heartbeat.rs
View File

@ -35,7 +35,7 @@ where
T: Clone + Sync + Send, T: Clone + Sync + Send,
{ {
fn drop(&mut self) { fn drop(&mut self) {
self.context.cancelled.notify(); self.context.cancelled.notify_one();
} }
} }

6
src/agent/onefuzz/src/jitter.rs
View File

@ -3,7 +3,7 @@
use rand::prelude::*; use rand::prelude::*;
use std::time::Duration; use std::time::Duration;
use tokio::time::delay_for; use tokio::time::sleep;
pub fn jitter(value: Duration) -> Duration { pub fn jitter(value: Duration) -> Duration {
let random: u64 = thread_rng().gen_range(0, 10); let random: u64 = thread_rng().gen_range(0, 10);
@ -11,11 +11,11 @@ pub fn jitter(value: Duration) -> Duration {
} }
pub async fn delay_with_jitter(value: Duration) { pub async fn delay_with_jitter(value: Duration) {
delay_for(jitter(value)).await sleep(jitter(value)).await
} }
pub async fn random_delay(value: Duration) { pub async fn random_delay(value: Duration) {
let random: u64 = thread_rng().gen_range(0, value.as_secs()); let random: u64 = thread_rng().gen_range(0, value.as_secs());
let delay = Duration::new(random, 0); let delay = Duration::new(random, 0);
delay_for(delay).await sleep(delay).await
} }

2
src/agent/onefuzz/src/process.rs
View File

@ -165,7 +165,7 @@ async fn wait_process(context: &str, process: Child, stopped: Option<&Notify>) -
debug!("child exited. {}:{:?}", context, output.status); debug!("child exited. {}:{:?}", context, output.status);
if let Some(stopped) = stopped { if let Some(stopped) = stopped {
stopped.notify(); stopped.notify_one();
} }
if output.status.success() { if output.status.success() {

4
src/agent/onefuzz/src/uploader.rs
View File

@ -67,7 +67,9 @@ impl BlobUploader {
let file = fs::File::from_std(std::fs::File::open(file_path)?); let file = fs::File::from_std(std::fs::File::open(file_path)?);
let reader = io::BufReader::new(file); let reader = io::BufReader::new(file);
let codec = codec::BytesCodec::new(); let codec = codec::BytesCodec::new();
let file_stream = codec::FramedRead::new(reader, codec).map_ok(bytes::BytesMut::freeze); let file_stream = codec::FramedRead::new(reader, codec)
.map_ok(bytes::BytesMut::freeze)
.into_stream();
let request_builder = self let request_builder = self
.client .client

26
src/agent/onefuzz/src/utils.rs
View File

@ -46,7 +46,7 @@ mod test {
use anyhow::Result; use anyhow::Result;
use futures::*; use futures::*;
use std::sync::Arc; use std::sync::Arc;
use tokio::{spawn, sync::Notify, task::JoinHandle, time::delay_for}; use tokio::{spawn, sync::Notify, task::JoinHandle, time::sleep};
fn spawn_ok() -> (Arc<Notify>, JoinHandle<Result<()>>) { fn spawn_ok() -> (Arc<Notify>, JoinHandle<Result<()>>) {
let notify = Arc::new(Notify::new()); let notify = Arc::new(Notify::new());
@ -77,7 +77,7 @@ mod test {
let (_notify3, handle3) = spawn_ok(); let (_notify3, handle3) = spawn_ok();
let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]); let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]);
delay_for(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
assert!( assert!(
try_wait_handle.now_or_never().is_none(), try_wait_handle.now_or_never().is_none(),
"expected no result" "expected no result"
@ -92,9 +92,9 @@ mod test {
let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]); let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]);
notify1.notify(); notify1.notify_one();
notify2.notify(); notify2.notify_one();
delay_for(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
assert!( assert!(
try_wait_handle.now_or_never().is_none(), try_wait_handle.now_or_never().is_none(),
"expected no result" "expected no result"
@ -109,10 +109,10 @@ mod test {
let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]); let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]);
notify1.notify(); notify1.notify_one();
notify2.notify(); notify2.notify_one();
notify3.notify(); notify3.notify_one();
delay_for(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
if let Some(result) = try_wait_handle.now_or_never() { if let Some(result) = try_wait_handle.now_or_never() {
assert!(result.is_ok(), "expected Ok") assert!(result.is_ok(), "expected Ok")
} else { } else {
@ -128,8 +128,8 @@ mod test {
let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]); let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]);
notify1.notify(); notify1.notify_one();
delay_for(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
assert!( assert!(
try_wait_handle.now_or_never().is_none(), try_wait_handle.now_or_never().is_none(),
"expected no result" "expected no result"
@ -144,9 +144,9 @@ mod test {
let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]); let try_wait_handle = try_wait_all_join_handles(vec![handle1, handle2, handle3]);
notify2.notify(); notify2.notify_one();
delay_for(Duration::from_secs(1)).await; sleep(Duration::from_secs(1)).await;
if let Some(result) = try_wait_handle.now_or_never() { if let Some(result) = try_wait_handle.now_or_never() {
assert!(result.is_err(), "expected error") assert!(result.is_err(), "expected error")
} else { } else {

4
src/agent/reqwest-retry/Cargo.toml
View File

@ -8,10 +8,10 @@ license = "MIT"
[dependencies] [dependencies]
anyhow = "1.0" anyhow = "1.0"
async-trait = "0.1" async-trait = "0.1"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
backoff = { version = "0.3", features = ["async-std"] } backoff = { version = "0.3", features = ["async-std"] }
log = "0.4" log = "0.4"
onefuzz-telemetry = { path = "../onefuzz-telemetry" } onefuzz-telemetry = { path = "../onefuzz-telemetry" }
[dev-dependencies] [dev-dependencies]
tokio = { version = "0.2" , features=["macros"] } tokio = { version = "1.5.0", features = ["macros"] }

5
src/agent/storage-queue/Cargo.toml
View File

@ -12,15 +12,16 @@ backoff = { version = "0.3", features = ["async-std"] }
base64 = "0.13" base64 = "0.13"
bytes = "0.5" bytes = "0.5"
derivative = "2.2.0" derivative = "2.2.0"
flume = "0.10.4"
num_cpus = "1.13" num_cpus = "1.13"
regex = "1.4" regex = "1.4"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
reqwest-retry = { path = "../reqwest-retry" } reqwest-retry = { path = "../reqwest-retry" }
serde = { version = "1.0", features = ["derive"]} serde = { version = "1.0", features = ["derive"]}
serde_derive = "1.0" serde_derive = "1.0"
serde_json = "1.0" serde_json = "1.0"
serde-xml-rs = "0.4" serde-xml-rs = "0.4"
tokio = { version = "0.2", features = ["full"] } tokio = { version = "1.5.0" , features=["full"] }
queue-file = "1.1" queue-file = "1.1"
uuid = { version = "0.8", features = ["serde", "v4"] } uuid = { version = "0.8", features = ["serde", "v4"] }
yaque = "0.5.1" yaque = "0.5.1"

12
src/agent/storage-queue/src/local_queue.rs
View File

@ -103,22 +103,20 @@ impl FileQueueClient {
} }
} }
use tokio::sync::mpsc::{ use flume::{unbounded, Receiver, Sender, TryRecvError};
error::TryRecvError, unbounded_channel, UnboundedReceiver, UnboundedSender,
};
/// Queue based on mpsc channel /// Queue based on mpsc channel
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ChannelQueueClient { pub struct ChannelQueueClient {
sender: Arc<Mutex<UnboundedSender<Vec<u8>>>>, sender: Arc<Mutex<Sender<Vec<u8>>>>,
receiver: Arc<Mutex<UnboundedReceiver<Vec<u8>>>>, receiver: Arc<Mutex<Receiver<Vec<u8>>>>,
pub url: reqwest::Url, pub url: reqwest::Url,
low_resource: bool, low_resource: bool,
} }
impl ChannelQueueClient { impl ChannelQueueClient {
pub fn new() -> Result<Self> { pub fn new() -> Result<Self> {
let (sender, receiver) = unbounded_channel(); let (sender, receiver) = unbounded();
let cpus = num_cpus::get(); let cpus = num_cpus::get();
let low_resource = cpus < 4; let low_resource = cpus < 4;
Ok(ChannelQueueClient { Ok(ChannelQueueClient {
@ -152,7 +150,7 @@ impl ChannelQueueClient {
if self.low_resource { if self.low_resource {
tokio::task::yield_now().await; tokio::task::yield_now().await;
} }
let mut receiver = self let receiver = self
.receiver .receiver
.lock() .lock()
.map_err(|_| anyhow::anyhow!("unable to acquire lock"))?; .map_err(|_| anyhow::anyhow!("unable to acquire lock"))?;

342
src/proxy-manager/Cargo.lock generated
View File

@ -42,7 +42,7 @@ dependencies = [
"http", "http",
"log", "log",
"paste", "paste",
"reqwest", "reqwest 0.10.10",
"serde", "serde",
"serde_json", "serde_json",
"sm", "sm",
@ -478,6 +478,19 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "flume"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11fce69af4d4582ea989e6adfc5c9b81fd2071ff89234e5c14675c82a85217df"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"pin-project",
"spinning_top",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@ -546,9 +559,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -561,9 +574,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -571,15 +584,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -588,9 +601,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
[[package]] [[package]]
name = "futures-lite" name = "futures-lite"
@ -609,9 +622,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2 1.0.24", "proc-macro2 1.0.24",
@ -621,21 +634,21 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.13" version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -669,8 +682,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"js-sys",
"libc", "libc",
"wasi 0.10.2+wasi-snapshot-preview1", "wasi 0.10.2+wasi-snapshot-preview1",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -700,12 +715,31 @@ dependencies = [
"http", "http",
"indexmap", "indexmap",
"slab", "slab",
"tokio", "tokio 0.2.25",
"tokio-util", "tokio-util 0.3.1",
"tracing", "tracing",
"tracing-futures", "tracing-futures",
] ]
[[package]]
name = "h2"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00"
dependencies = [
"bytes 1.0.1",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http",
"indexmap",
"slab",
"tokio 1.5.0",
"tokio-util 0.6.6",
"tracing",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.9.1" version = "0.9.1"
@ -753,6 +787,17 @@ dependencies = [
"http", "http",
] ]
[[package]]
name = "http-body"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737"
dependencies = [
"bytes 1.0.1",
"http",
"pin-project-lite 0.2.5",
]
[[package]] [[package]]
name = "httparse" name = "httparse"
version = "1.3.5" version = "1.3.5"
@ -781,15 +826,39 @@ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-util", "futures-util",
"h2", "h2 0.2.7",
"http", "http",
"http-body", "http-body 0.3.1",
"httparse", "httparse",
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project", "pin-project",
"socket2", "socket2 0.3.19",
"tokio", "tokio 0.2.25",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1"
dependencies = [
"bytes 1.0.1",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.2",
"http",
"http-body 0.4.1",
"httparse",
"httpdate",
"itoa",
"pin-project",
"socket2 0.4.0",
"tokio 1.5.0",
"tower-service", "tower-service",
"tracing", "tracing",
"want", "want",
@ -802,12 +871,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 0.5.6",
"hyper", "hyper 0.13.10",
"native-tls", "native-tls",
"tokio", "tokio 0.2.25",
"tokio-tls", "tokio-tls",
] ]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes 1.0.1",
"hyper 0.14.5",
"native-tls",
"tokio 1.5.0",
"tokio-native-tls",
]
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.2" version = "0.2.2"
@ -919,6 +1001,15 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "265d751d31d6780a3f956bb5b8022feba2d94eeee5a84ba64f4212eedca42213" checksum = "265d751d31d6780a3f956bb5b8022feba2d94eeee5a84ba64f4212eedca42213"
[[package]]
name = "lock_api"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.14" version = "0.4.14"
@ -1010,29 +1101,6 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "mio-named-pipes"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656"
dependencies = [
"log",
"mio 0.6.23",
"miow 0.3.6",
"winapi 0.3.9",
]
[[package]]
name = "mio-uds"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
dependencies = [
"iovec",
"libc",
"mio 0.6.23",
]
[[package]] [[package]]
name = "miow" name = "miow"
version = "0.2.2" version = "0.2.2"
@ -1051,10 +1119,19 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
dependencies = [ dependencies = [
"socket2", "socket2 0.3.19",
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "nanorand"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1378b66f7c93a1c0f8464a19bf47df8795083842e5090f4b7305973d5a22d0"
dependencies = [
"getrandom 0.2.2",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.7" version = "0.2.7"
@ -1080,7 +1157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f" checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f"
dependencies = [ dependencies = [
"libc", "libc",
"socket2", "socket2 0.3.19",
] ]
[[package]] [[package]]
@ -1167,13 +1244,13 @@ dependencies = [
"futures", "futures",
"log", "log",
"onefuzz-telemetry", "onefuzz-telemetry",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"serde", "serde",
"serde_json", "serde_json",
"storage-queue", "storage-queue",
"thiserror", "thiserror",
"tokio", "tokio 1.5.0",
"url", "url",
"uuid", "uuid",
] ]
@ -1183,8 +1260,10 @@ name = "onefuzz-telemetry"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"appinsights", "appinsights",
"lazy_static",
"log", "log",
"serde", "serde",
"tokio 1.5.0",
"uuid", "uuid",
] ]
@ -1227,6 +1306,31 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
[[package]]
name = "parking_lot"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "paste" name = "paste"
version = "0.1.18" version = "0.1.18"
@ -1527,9 +1631,9 @@ dependencies = [
"futures-core", "futures-core",
"futures-util", "futures-util",
"http", "http",
"http-body", "http-body 0.3.1",
"hyper", "hyper 0.13.10",
"hyper-tls", "hyper-tls 0.4.3",
"ipnet", "ipnet",
"js-sys", "js-sys",
"lazy_static", "lazy_static",
@ -1542,7 +1646,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"tokio", "tokio 0.2.25",
"tokio-tls", "tokio-tls",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
@ -1551,6 +1655,41 @@ dependencies = [
"winreg", "winreg",
] ]
[[package]]
name = "reqwest"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
dependencies = [
"base64",
"bytes 1.0.1",
"encoding_rs",
"futures-core",
"futures-util",
"http",
"http-body 0.4.1",
"hyper 0.14.5",
"hyper-tls 0.5.0",
"ipnet",
"js-sys",
"lazy_static",
"log",
"mime",
"native-tls",
"percent-encoding",
"pin-project-lite 0.2.5",
"serde",
"serde_json",
"serde_urlencoded",
"tokio 1.5.0",
"tokio-native-tls",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
]
[[package]] [[package]]
name = "reqwest-retry" name = "reqwest-retry"
version = "0.1.0" version = "0.1.0"
@ -1560,7 +1699,7 @@ dependencies = [
"backoff", "backoff",
"log", "log",
"onefuzz-telemetry", "onefuzz-telemetry",
"reqwest", "reqwest 0.11.3",
] ]
[[package]] [[package]]
@ -1707,6 +1846,12 @@ dependencies = [
"syn 0.15.44", "syn 0.15.44",
] ]
[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]] [[package]]
name = "snafu" name = "snafu"
version = "0.6.10" version = "0.6.10"
@ -1739,6 +1884,25 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "socket2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]]
name = "spinning_top"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bd0ab6b8c375d2d963503b90d3770010d95bc3b5f98036f948dee24bf4e8879"
dependencies = [
"lock_api",
]
[[package]] [[package]]
name = "storage-queue" name = "storage-queue"
version = "0.1.0" version = "0.1.0"
@ -1749,15 +1913,17 @@ dependencies = [
"base64", "base64",
"bytes 0.5.6", "bytes 0.5.6",
"derivative", "derivative",
"flume",
"num_cpus",
"queue-file", "queue-file",
"regex", "regex",
"reqwest", "reqwest 0.11.3",
"reqwest-retry", "reqwest-retry",
"serde", "serde",
"serde-xml-rs", "serde-xml-rs",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"tokio", "tokio 1.5.0",
"uuid", "uuid",
"yaque", "yaque",
] ]
@ -1902,30 +2068,54 @@ dependencies = [
"futures-core", "futures-core",
"iovec", "iovec",
"lazy_static", "lazy_static",
"libc",
"memchr", "memchr",
"mio 0.6.23", "mio 0.6.23",
"mio-named-pipes",
"mio-uds",
"num_cpus", "num_cpus",
"pin-project-lite 0.1.12", "pin-project-lite 0.1.12",
"signal-hook-registry",
"slab", "slab",
]
[[package]]
name = "tokio"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5"
dependencies = [
"autocfg",
"bytes 1.0.1",
"libc",
"memchr",
"mio 0.7.9",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite 0.2.5",
"signal-hook-registry",
"tokio-macros", "tokio-macros",
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "0.2.6" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [ dependencies = [
"proc-macro2 1.0.24", "proc-macro2 1.0.24",
"quote 1.0.9", "quote 1.0.9",
"syn 1.0.60", "syn 1.0.60",
] ]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio 1.5.0",
]
[[package]] [[package]]
name = "tokio-tls" name = "tokio-tls"
version = "0.3.1" version = "0.3.1"
@ -1933,7 +2123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343"
dependencies = [ dependencies = [
"native-tls", "native-tls",
"tokio", "tokio 0.2.25",
] ]
[[package]] [[package]]
@ -1947,7 +2137,21 @@ dependencies = [
"futures-sink", "futures-sink",
"log", "log",
"pin-project-lite 0.1.12", "pin-project-lite 0.1.12",
"tokio", "tokio 0.2.25",
]
[[package]]
name = "tokio-util"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e"
dependencies = [
"bytes 1.0.1",
"futures-core",
"futures-sink",
"log",
"pin-project-lite 0.2.5",
"tokio 1.5.0",
] ]
[[package]] [[package]]

6
src/proxy-manager/Cargo.toml
View File

@ -10,13 +10,13 @@ license = "MIT"
anyhow = "1.0" anyhow = "1.0"
clap = "2.33" clap = "2.33"
env_logger = "0.8" env_logger = "0.8"
futures = "0.3" futures = "0.3.14"
reqwest = { version = "0.10", features = ["json", "stream"] } reqwest = { version = "0.11.3", features = ["json", "stream"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
storage-queue = { path = "../agent/storage-queue" } storage-queue = { path = "../agent/storage-queue" }
thiserror = "1.0" thiserror = "1.0"
tokio = { version = "0.2", features = ["macros", "rt-threaded", "fs", "process"] } tokio = { version = "1.5.0", features = ["macros", "rt-multi-thread", "fs", "process"] }
url = { version = "2.2", features = ["serde"] } url = { version = "2.2", features = ["serde"] }
reqwest-retry = { path = "../agent/reqwest-retry"} reqwest-retry = { path = "../agent/reqwest-retry"}
onefuzz-telemetry = { path = "../agent/onefuzz-telemetry" } onefuzz-telemetry = { path = "../agent/onefuzz-telemetry" }

6
src/proxy-manager/src/main.rs
View File

@ -20,7 +20,7 @@ use std::{
}; };
use tokio::{ use tokio::{
runtime::Runtime, runtime::Runtime,
time::{delay_for, Duration}, time::{sleep, Duration},
}; };
const MINIMUM_NOTIFY_INTERVAL: Duration = Duration::from_secs(120); const MINIMUM_NOTIFY_INTERVAL: Duration = Duration::from_secs(120);
@ -37,7 +37,7 @@ async fn run_loop(mut proxy_config: Config) -> Result<()> {
last_notified = Instant::now(); last_notified = Instant::now();
} }
delay_for(POLL_INTERVAL).await; sleep(POLL_INTERVAL).await;
} }
} }
@ -85,6 +85,6 @@ fn main() -> Result<()> {
let proxy = Config::from_file(config_path)?; let proxy = Config::from_file(config_path)?;
info!("parsed initial config"); info!("parsed initial config");
let mut rt = Runtime::new()?; let rt = Runtime::new()?;
rt.block_on(run(proxy)) rt.block_on(run(proxy))
} }

8
src/proxy-manager/src/proxy.rs
View File

@ -3,7 +3,6 @@
use crate::config::ConfigData; use crate::config::ConfigData;
use anyhow::Result; use anyhow::Result;
use futures::stream::StreamExt;
use std::{collections::HashMap, path::Path}; use std::{collections::HashMap, path::Path};
use tokio::process::Command; use tokio::process::Command;
@ -76,9 +75,10 @@ pub async fn update(data: &ConfigData) -> Result<()> {
let configs = build(data); let configs = build(data);
let mut config_dir = tokio::fs::read_dir(SYSTEMD_CONFIG_DIR).await?; let mut config_dir = tokio::fs::read_dir(SYSTEMD_CONFIG_DIR).await?;
while let Some(entry) = config_dir.next().await { loop {
let entry = match entry { let entry = match config_dir.next_entry().await {
Ok(entry) => entry, Ok(Some(entry)) => entry,
Ok(None) => break,
Err(err) => { Err(err) => {
error!("error listing files {}", err); error!("error listing files {}", err);
continue; continue;