From cbe6ef8e40f9d653ffe1ebe9a68796506c49b680 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Fri, 22 Oct 2021 07:00:46 -0700 Subject: [PATCH] Update NSGs after changes to instance config (#1385) * Refactor set_admins into configuration.py and update deployment params with nsg_config * Fixing arguments. * Param takes in network config json * Fixing Client in deploy * removing import * Adding onefuzztypes to reqs.txt * Reverting to single list * Removing imports. * Retriggering build * Setting specific pip version for local testing. * Removing imports? * More imports. * Fixing formatting. * Updating how to parse nsg param. * Removing old logging statements. * Fixing types. * REmoving bad log * Removing local pip version. * Removing comments * fixing * Formatting * Fixing .split() * Adding NSG rule checks and type. * Formatting. * Formatting. * Removing imports. * Fixing formatting. * Testing formatting. * Retrigger? * New InstanceConfigClient class. * Retrigger. * Cherry picked commit. * Reformatting. * Actually fixing formatting. * Fixing table_service call. * Fixing return statement and nsg_rule pass. * Full config. * Removing commented out code. * Fixing logic. * Adding wildcard check. * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Adding empty() function on request. * Removing empty function. * Fixing files for update. * Fixing nsg.py. * Fixing imports. * removing commented code. Co-authored-by: nharper285 Update configuration.py to check for 'block all' configuration. (#1394) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Build fix (#1374) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Delete NSG if no resources associated with it (#1358) Co-authored-by: stas * Update NSGs after changes to instance config (#1385) * Refactor set_admins into configuration.py and update deployment params with nsg_config * Fixing arguments. * Param takes in network config json * Fixing Client in deploy * removing import * Adding onefuzztypes to reqs.txt * Reverting to single list * Removing imports. * Retriggering build * Setting specific pip version for local testing. * Removing imports? * More imports. * Fixing formatting. * Updating how to parse nsg param. * Removing old logging statements. * Fixing types. * REmoving bad log * Removing local pip version. * Removing comments * fixing * Formatting * Fixing .split() * Adding NSG rule checks and type. * Formatting. * Formatting. * Removing imports. * Fixing formatting. * Testing formatting. * Retrigger? * New InstanceConfigClient class. * Retrigger. * Cherry picked commit. * Reformatting. * Actually fixing formatting. * Fixing table_service call. * Fixing return statement and nsg_rule pass. * Full config. * Removing commented out code. * Fixing logic. * Adding wildcard check. * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Adding empty() function on request. * Removing empty function. * Fixing files for update. * Fixing nsg.py. * Fixing imports. * removing commented code. Co-authored-by: nharper285 * Aligning feature branch with main (#1389) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/agent (#1368) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump crossterm from 0.21.0 to 0.22.1 in /src/agent (#1369) Bumps [crossterm](https://github.com/crossterm-rs/crossterm) from 0.21.0 to 0.22.1. - [Release notes](https://github.com/crossterm-rs/crossterm/releases) - [Changelog](https://github.com/crossterm-rs/crossterm/blob/master/CHANGELOG.md) - [Commits](https://github.com/crossterm-rs/crossterm/commits) --- updated-dependencies: - dependency-name: crossterm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Fix validation of `target_exe` blob name (#1371) * NSG Updated After CLI Update to Instance_Config (#1375) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Updating instanceconfig retrieval. * Fixing imports. * Adding empty() function on request. * Fixing name of function. * Removing empty function. Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: Joe Ranweiler * Revert "NSG Updated After CLI Update to Instance_Config (#1375)" (#1384) This reverts commit 357bc4fcadc0bae9886e53ba1a9a316365e41094. * Bump backtrace from 0.3.61 to 0.3.62 in /src/agent (#1382) Bumps [backtrace](https://github.com/rust-lang/backtrace-rs) from 0.3.61 to 0.3.62. - [Release notes](https://github.com/rust-lang/backtrace-rs/releases) - [Commits](https://github.com/rust-lang/backtrace-rs/compare/0.3.61...0.3.62) --- updated-dependencies: - dependency-name: backtrace dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Set compiler env vars to effect Win10 SDK downgrade (#1388) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler Co-authored-by: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Co-authored-by: nharper285 * Updating configuration.py to check for 'block all' config. * Fixing error message. * associate subnets with NSG (#1393) * associate subnets with NSG change NSG rule protocol to ANY * subnet wait * Improve NSG update logic Co-authored-by: stas Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler --- docs/webhook_events.md | 3 +- .../__app__/instance_config/__init__.py | 27 +++ .../__app__/onefuzzlib/azure/nsg.py | 19 ++- src/api-service/__app__/onefuzzlib/proxy.py | 4 +- src/api-service/__app__/onefuzzlib/repro.py | 5 +- src/deployment/configuration.py | 157 ++++++++++++++++++ src/deployment/deploy.py | 22 ++- src/deployment/requirements.txt | 2 +- src/deployment/set_admins.py | 74 +-------- src/pytypes/onefuzztypes/models.py | 2 +- 10 files changed, 233 insertions(+), 82 deletions(-) create mode 100644 src/deployment/configuration.py diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 774455f95..38946433b 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -650,7 +650,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "subnet": "10.0.0.0/16" }, "proxy_nsg_config": { - "allowed_ips": [] + "allowed_ips": [], + "allowed_service_tags": [] }, "proxy_vm_sku": "Standard_B2s" } diff --git a/src/api-service/__app__/instance_config/__init__.py b/src/api-service/__app__/instance_config/__init__.py index 800abe5f6..15c20a788 100644 --- a/src/api-service/__app__/instance_config/__init__.py +++ b/src/api-service/__app__/instance_config/__init__.py @@ -8,9 +8,11 @@ from onefuzztypes.enums import ErrorCode from onefuzztypes.models import Error from onefuzztypes.requests import InstanceConfigUpdate +from ..onefuzzlib.azure.nsg import set_allowed from ..onefuzzlib.config import InstanceConfig from ..onefuzzlib.endpoint_authorization import call_if_user, can_modify_config from ..onefuzzlib.request import not_ok, ok, parse_request +from ..onefuzzlib.workers.scalesets import Scaleset def get(req: func.HttpRequest) -> func.HttpResponse: @@ -30,8 +32,33 @@ def post(req: func.HttpRequest) -> func.HttpResponse: context="instance_config_update", ) + update_nsg = False + if request.config.proxy_nsg_config and config.proxy_nsg_config: + request_config = request.config.proxy_nsg_config + current_config = config.proxy_nsg_config + if set(request_config.allowed_service_tags) != set( + current_config.allowed_service_tags + ) or set(request_config.allowed_ips) != set(current_config.allowed_ips): + update_nsg = True + config.update(request.config) config.save() + + # Update All NSGs + if update_nsg: + scalesets = Scaleset.search() + regions = set(x.region for x in scalesets) + for region in regions: + result = set_allowed(region, request.config.proxy_nsg_config) + if isinstance(result, Error): + return not_ok( + Error( + code=ErrorCode.UNABLE_TO_CREATE, + errors=["Unable to update nsg %s due to %s" % (region, result)], + ), + context="instance_config_update", + ) + return ok(config) diff --git a/src/api-service/__app__/onefuzzlib/azure/nsg.py b/src/api-service/__app__/onefuzzlib/azure/nsg.py index 9d4e0bbc8..3a4becb54 100644 --- a/src/api-service/__app__/onefuzzlib/azure/nsg.py +++ b/src/api-service/__app__/onefuzzlib/azure/nsg.py @@ -17,7 +17,7 @@ from azure.mgmt.network.models import ( ) from msrestazure.azure_exceptions import CloudError from onefuzztypes.enums import ErrorCode -from onefuzztypes.models import Error +from onefuzztypes.models import Error, NetworkSecurityGroupConfig from onefuzztypes.primitives import Region from pydantic import BaseModel, validator @@ -127,7 +127,7 @@ def delete_nsg(name: str) -> bool: return False -def set_allowed(name: str, sources: List[str]) -> Union[None, Error]: +def set_allowed(name: str, sources: NetworkSecurityGroupConfig) -> Union[None, Error]: resource_group = get_base_resource_group() nsg = get_nsg(name) if not nsg: @@ -141,6 +141,7 @@ def set_allowed(name: str, sources: List[str]) -> Union[None, Error]: resource_group, name, ) + all_sources = sources.allowed_ips + sources.allowed_service_tags security_rules = [] # NSG security rule priority range defined here: # https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview @@ -148,17 +149,17 @@ def set_allowed(name: str, sources: List[str]) -> Union[None, Error]: # NSG rules per NSG limits: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits?toc=/azure/virtual-network/toc.json#networking-limits max_rule_count = 1000 - if len(sources) > max_rule_count: + if len(all_sources) > max_rule_count: return Error( code=ErrorCode.INVALID_REQUEST, errors=[ "too many rules provided %d. Max allowed: %d" - % ((len(sources)), max_rule_count), + % ((len(all_sources)), max_rule_count), ], ) priority = min_priority - for src in sources: + for src in all_sources: security_rules.append( SecurityRule( name="Allow" + str(priority), @@ -173,7 +174,7 @@ def set_allowed(name: str, sources: List[str]) -> Union[None, Error]: ) ) # Will not exceed `max_rule_count` or max NSG priority (4096) - # due to earlier check of `len(sources)`. + # due to earlier check of `len(all_sources)`. priority += 1 nsg.security_rules = security_rules @@ -181,7 +182,7 @@ def set_allowed(name: str, sources: List[str]) -> Union[None, Error]: def clear_all_rules(name: str) -> Union[None, Error]: - return set_allowed(name, []) + return set_allowed(name, NetworkSecurityGroupConfig()) def get_all_rules(name: str) -> Union[Error, List[SecurityRule]]: @@ -328,7 +329,9 @@ class NSG(BaseModel): def get(self) -> Optional[NetworkSecurityGroup]: return get_nsg(self.name) - def set_allowed_sources(self, sources: List[str]) -> Union[None, Error]: + def set_allowed_sources( + self, sources: NetworkSecurityGroupConfig + ) -> Union[None, Error]: return set_allowed(self.name, sources) def clear_all_rules(self) -> Union[None, Error]: diff --git a/src/api-service/__app__/onefuzzlib/proxy.py b/src/api-service/__app__/onefuzzlib/proxy.py index c07ef9868..78d82220c 100644 --- a/src/api-service/__app__/onefuzzlib/proxy.py +++ b/src/api-service/__app__/onefuzzlib/proxy.py @@ -101,7 +101,9 @@ class Proxy(ORMMixin): self.set_failed(result) return - result = nsg.set_allowed_sources(["*"]) + config = InstanceConfig.fetch() + nsg_config = config.proxy_nsg_config + result = nsg.set_allowed_sources(nsg_config) if isinstance(result, Error): self.set_failed(result) return diff --git a/src/api-service/__app__/onefuzzlib/repro.py b/src/api-service/__app__/onefuzzlib/repro.py index 12f9d5c90..937f252bd 100644 --- a/src/api-service/__app__/onefuzzlib/repro.py +++ b/src/api-service/__app__/onefuzzlib/repro.py @@ -21,6 +21,7 @@ from .azure.ip import get_public_ip from .azure.nsg import NSG from .azure.storage import StorageType from .azure.vm import VM +from .config import InstanceConfig from .extension import repro_extensions from .orm import ORMMixin, QueryFilter from .reports import get_report @@ -95,7 +96,9 @@ class Repro(BASE_REPRO, ORMMixin): self.set_failed(result) return - result = nsg.set_allowed_sources(["*"]) + config = InstanceConfig.fetch() + nsg_config = config.proxy_nsg_config + result = nsg.set_allowed_sources(nsg_config) if isinstance(result, Error): self.set_failed(result) return diff --git a/src/deployment/configuration.py b/src/deployment/configuration.py new file mode 100644 index 000000000..b3e1b5030 --- /dev/null +++ b/src/deployment/configuration.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import ipaddress +import json +import logging +from typing import List, Optional +from uuid import UUID + +from azure.cosmosdb.table.tableservice import TableService + +storage_client_logger = logging.getLogger("azure.cosmosdb.table.common.storageclient") +TABLE_NAME = "InstanceConfig" + +logger = logging.getLogger("deploy") + + +class InstanceConfigClient: + + table_service: TableService + resource_group: str + + def __init__(self, table_service: TableService, resource_group: str): + self.resource_group = resource_group + self.table_service = table_service + self.create_if_missing(table_service) + + ## Disable logging from storageclient. This module displays an error message + ## when a resource is not found even if the exception is raised and handled internally. + ## This happen when a table does not exist. An error message is displayed but the exception is + ## handled by the library. + def disable_storage_client_logging(self) -> None: + if storage_client_logger: + storage_client_logger.disabled = True + + def enable_storage_client_logging(self) -> None: + if storage_client_logger: + storage_client_logger.disabled = False + + def create_if_missing(self, table_service: TableService) -> None: + try: + self.disable_storage_client_logging() + + if not table_service.exists(TABLE_NAME): + table_service.create_table(TABLE_NAME) + finally: + self.enable_storage_client_logging() + + +class NsgRule: + + rule: str + is_tag: bool + + def __init__(self, rule: str): + try: + self.is_tag = False + self.check_rule(rule) + self.rule = rule + except Exception: + raise ValueError( + "Invalid rule. Please provide a valid rule or supply the wild card *." + ) + + def check_rule(self, value: str) -> None: + if value is None: + raise ValueError( + "Please provide a valid rule or supply the empty string '' to block all sources or the wild card * to allow all sources." + ) + # Check block all + if len(value.strip()) == 0: + return + # Check Wild Card + if value == "*": + return + # Check if IP Address + try: + ipaddress.ip_address(value) + return + except ValueError: + pass + # Check if IP Range + try: + ipaddress.ip_network(value) + return + except ValueError: + pass + + self.is_tag = True + + +def update_allowed_aad_tenants( + config_client: InstanceConfigClient, tenants: List[UUID] +) -> None: + as_str = [str(x) for x in tenants] + config_client.table_service.insert_or_merge_entity( + TABLE_NAME, + { + "PartitionKey": config_client.resource_group, + "RowKey": config_client.resource_group, + "allowed_aad_tenants": json.dumps(as_str), + }, + ) + + +def update_admins(config_client: InstanceConfigClient, admins: List[UUID]) -> None: + admins_as_str: Optional[List[str]] = None + if admins: + admins_as_str = [str(x) for x in admins] + + config_client.table_service.insert_or_merge_entity( + TABLE_NAME, + { + "PartitionKey": config_client.resource_group, + "RowKey": config_client.resource_group, + "admins": json.dumps(admins_as_str), + }, + ) + + +def parse_rules(rules_str: str) -> List[NsgRule]: + rules_list = rules_str.split(",") + + nsg_rules = [] + for rule in rules_list: + try: + nsg_rule = NsgRule(rule) + nsg_rules.append(nsg_rule) + except Exception: + raise ValueError( + "One or more input rules was invalid. Please enter a comma-separted list if valid sources." + ) + return nsg_rules + + +def update_nsg( + config_client: InstanceConfigClient, + allowed_rules: List[NsgRule], +) -> None: + tags_as_str = [x.rule for x in allowed_rules if x.is_tag] + ips_as_str = [x.rule for x in allowed_rules if not x.is_tag] + nsg_config = {"allowed_service_tags": tags_as_str, "allowed_ips": ips_as_str} + # create class initialized by table service/resource group outside function that's checked in deploy.py + config_client.table_service.insert_or_merge_entity( + TABLE_NAME, + { + "PartitionKey": config_client.resource_group, + "RowKey": config_client.resource_group, + "proxy_nsg_config": json.dumps(nsg_config), + }, + ) + + +if __name__ == "__main__": + pass diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 44b2bf614..62b4f8e18 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -47,6 +47,13 @@ from azure.storage.blob import ( ) from msrest.serialization import TZ_UTC +from configuration import ( + InstanceConfigClient, + parse_rules, + update_admins, + update_allowed_aad_tenants, + update_nsg, +) from data_migration import migrate from registration import ( GraphQueryError, @@ -61,7 +68,6 @@ from registration import ( set_app_audience, update_pool_registration, ) -from set_admins import update_admins, update_allowed_aad_tenants # Found by manually assigning the User.Read permission to application # registration in the admin portal. The values are in the manifest under @@ -104,6 +110,7 @@ class Client: location: str, application_name: str, owner: str, + nsg_config: str, client_id: Optional[str], client_secret: Optional[str], app_zip: str, @@ -127,6 +134,7 @@ class Client: self.location = location self.application_name = application_name self.owner = owner + self.nsg_config = nsg_config self.app_zip = app_zip self.tools = tools self.instance_specific = instance_specific @@ -608,13 +616,19 @@ class Client: tenant = UUID(self.results["deploy"]["tenant_id"]["value"]) table_service = TableService(account_name=name, account_key=key) + config_client = InstanceConfigClient(table_service, self.application_name) + + if self.nsg_config: + rules = parse_rules(self.nsg_config) + update_nsg(config_client, rules) + if self.admins: - update_admins(table_service, self.application_name, self.admins) + update_admins(config_client, self.admins) tenants = self.allowed_aad_tenants if tenant not in tenants: tenants.append(tenant) - update_allowed_aad_tenants(table_service, self.application_name, tenants) + update_allowed_aad_tenants(config_client, tenants) def create_eventgrid(self) -> None: logger.info("creating eventgrid subscription") @@ -972,6 +986,7 @@ def main() -> None: parser.add_argument("resource_group") parser.add_argument("application_name") parser.add_argument("owner") + parser.add_argument("nsg_config") parser.add_argument( "--arm-template", type=arg_file, @@ -1079,6 +1094,7 @@ def main() -> None: location=args.location, application_name=args.application_name, owner=args.owner, + nsg_config=args.nsg_config, client_id=args.client_id, client_secret=args.client_secret, app_zip=args.app_zip, diff --git a/src/deployment/requirements.txt b/src/deployment/requirements.txt index 3c514df03..e8fa658f0 100644 --- a/src/deployment/requirements.txt +++ b/src/deployment/requirements.txt @@ -8,4 +8,4 @@ azure-storage-blob==12.8.1 pyfunctional==1.4.3 pyopenssl==19.1.0 adal~=1.2.5 -idna<3,>=2.5 +idna<3,>=2.5 \ No newline at end of file diff --git a/src/deployment/set_admins.py b/src/deployment/set_admins.py index 91e9cd2e4..20eea27d4 100644 --- a/src/deployment/set_admins.py +++ b/src/deployment/set_admins.py @@ -4,74 +4,17 @@ # Licensed under the MIT License. import argparse -import json -import logging -from typing import List, Optional from uuid import UUID from azure.common.client_factory import get_client_from_cli_profile from azure.cosmosdb.table.tableservice import TableService from azure.mgmt.storage import StorageManagementClient -storage_client_logger = logging.getLogger("azure.cosmosdb.table.common.storageclient") -TABLE_NAME = "InstanceConfig" - - -## Disable logging from storageclient. This module displays an error message -## when a resource is not found even if the exception is raised and handled internally. -## This happen when a table does not exist. An error message is displayed but the exception is -## handled by the library. -def disable_storage_client_logging() -> None: - if storage_client_logger: - storage_client_logger.disabled = True - - -def enable_storage_client_logging() -> None: - if storage_client_logger: - storage_client_logger.disabled = False - - -def create_if_missing(table_service: TableService) -> None: - try: - disable_storage_client_logging() - - if not table_service.exists(TABLE_NAME): - table_service.create_table(TABLE_NAME) - finally: - enable_storage_client_logging() - - -def update_allowed_aad_tenants( - table_service: TableService, resource_group: str, tenants: List[UUID] -) -> None: - create_if_missing(table_service) - as_str = [str(x) for x in tenants] - table_service.insert_or_merge_entity( - TABLE_NAME, - { - "PartitionKey": resource_group, - "RowKey": resource_group, - "allowed_aad_tenants": json.dumps(as_str), - }, - ) - - -def update_admins( - table_service: TableService, resource_group: str, admins: List[UUID] -) -> None: - create_if_missing(table_service) - admins_as_str: Optional[List[str]] = None - if admins: - admins_as_str = [str(x) for x in admins] - - table_service.insert_or_merge_entity( - TABLE_NAME, - { - "PartitionKey": resource_group, - "RowKey": resource_group, - "admins": json.dumps(admins_as_str), - }, - ) +from configuration import ( + InstanceConfigClient, + update_admins, + update_allowed_aad_tenants, +) def main() -> None: @@ -90,12 +33,11 @@ def main() -> None: table_service = TableService( account_name=args.storage_account, account_key=storage_keys.keys[0].value ) + config_client = InstanceConfigClient(table_service, args.resource_group) if args.admins: - update_admins(table_service, args.resource_group, args.admins) + update_admins(config_client, args.admins) if args.allowed_aad_tenants: - update_allowed_aad_tenants( - table_service, args.resource_group, args.allowed_aad_tenants - ) + update_allowed_aad_tenants(config_client, args.allowed_aad_tenants) if __name__ == "__main__": diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 2508b03cd..f3a47b32f 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -800,7 +800,7 @@ class NetworkConfig(BaseModel): class NetworkSecurityGroupConfig(BaseModel): - allowed_service_tags: Optional[List[str]] + allowed_service_tags: List[str] = Field(default_factory=list) allowed_ips: List[str] = Field(default_factory=list)