bump azure-cli and azure-cli-core to 2.31.0 (#1557)

* bump azure-cli and azure-cli-core to 2.31.0

and all required dependencies

* Update src/cli/onefuzz/cred_wrapper.py

Co-authored-by: Joe Ranweiler <joe@lemma.co>

* updating credential wrapper

* Update src/cli/onefuzz/azure_identity_credential_adapter.py

Co-authored-by: Joe Ranweiler <joe@lemma.co>

* updating credential wrapper

* .

Co-authored-by: stas <statis@microsoft.com>
Co-authored-by: Joe Ranweiler <joe@lemma.co>

src/cli/onefuzz/azure_identity_credential_adapter.py

@@ -0,0 +1,66 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+
+# Adapt credentials from azure-identity to be compatible with SDK that needs msrestazure or azure.common.credentials
+# Need msrest >= 0.6.0
+# See also https://pypi.org/project/azure-identity/
+
+# Source: https://github.com/jongio/azidext/blob/8374293bd80648f764237ddfc5f5223e7e98472b/python/azure_identity_credential_adapter.py
+
+from typing import Any
+
+from azure.core.pipeline import PipelineContext, PipelineRequest
+from azure.core.pipeline.policies import BearerTokenCredentialPolicy
+from azure.core.pipeline.transport import HttpRequest
+from azure.identity import DefaultAzureCredential
+from msrest.authentication import BasicTokenAuthentication
+
+
+class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
+    def __init__(
+        self,
+        credential: Any = None,
+        resource_id: Any = "https://management.azure.com/.default",
+        **kwargs: Any
+    ):
+        """Adapt any azure-identity credential to work with SDK that needs azure.common.credentials or msrestazure.
+
+        Default resource is ARM (syntax of endpoint v2)
+
+        :param credential: Any azure-identity credential (DefaultAzureCredential by default)
+        :param str resource_id: The scope to use to get the token (default ARM)
+        """
+        super(AzureIdentityCredentialAdapter, self).__init__({})
+        if credential is None:
+            credential = DefaultAzureCredential()
+        self._policy = BearerTokenCredentialPolicy(credential, resource_id, **kwargs)
+
+    def _make_request(self) -> Any:
+        return PipelineRequest(
+            HttpRequest(
+                "AzureIdentityCredentialAdapter",
+                # This URL is not actually used. We just create a phony request to get credentials using only public APIs.
+                # Use a standard Microsoft-controlled example URL anyway.
+                "https://contoso.com",
+            ),
+            PipelineContext(None),
+        )
+
+    def set_token(self) -> Any:
+        """Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
+
+        Using the policy gives us for free the caching system of azure-core.
+        We could make this code simpler by using private method, but by definition
+        I can't assure they will be there forever, so mocking a fake call to the policy
+        to extract the token, using 100% public API."""
+        request = self._make_request()
+        self._policy.on_request(request)
+        # Read Authorization, and get the second part after Bearer
+        token = request.http_request.headers["Authorization"].split(" ", 1)[1]
+        self.token = {"access_token": token}
+
+    def signed_session(self, session: Any = None) -> Any:
+        self.set_token()
+        return super(AzureIdentityCredentialAdapter, self).signed_session(session)

src/cli/onefuzz/debug.py

@@ -15,13 +15,14 @@ from uuid import UUID
 import jmespath
 from azure.applicationinsights import ApplicationInsightsDataClient
 from azure.applicationinsights.models import QueryBody
-from azure.common.client_factory import get_azure_cli_credentials
+from azure.identity import AzureCliCredential
 from onefuzztypes.enums import ContainerType, TaskType
 from onefuzztypes.models import BlobRef, Job, NodeAssignment, Report, Task, TaskConfig
 from onefuzztypes.primitives import Container, Directory, PoolName
 
 from onefuzz.api import UUID_EXPANSION, Command, Onefuzz
 
+from .azure_identity_credential_adapter import AzureIdentityCredentialAdapter
 from .backend import wait
 from .rdp import rdp_connect
 from .ssh import ssh_connect
@@ -455,8 +456,8 @@ class DebugLog(Command):
             raise Exception("instance does not have an insights_appid")
         if self._client is None:
 
-            creds, _ = get_azure_cli_credentials(
+            creds = AzureIdentityCredentialAdapter(
-                resource="https://api.applicationinsights.io"
+                AzureCliCredential(), resource_id="https://api.applicationinsights.io"
             )
             self._client = ApplicationInsightsDataClient(creds)
 

src/cli/requirements.txt

@@ -1,4 +1,4 @@
-msal~=1.14.0
+msal~=1.16.0
 requests~=2.25.1
 jmespath~=0.10.0
 semver~=2.13.0
@@ -11,7 +11,8 @@ azure-storage-blob~=12.8
 azure-applicationinsights==0.1.0
 tenacity==8.0.1
 docstring_parser==0.8.1
-azure-cli-core==2.27.2
+azure-identity==1.7.1
+azure-cli-core==2.31.0
 # packaging is required but not specified by azure-cli-core
 packaging==20.9
 # urllib3[secure] needs to be specifically stated for azure-cli-core

src/deployment/deploy.py

@@ -19,9 +19,9 @@ from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple, Union, cast
 from uuid import UUID
 
-from azure.common.client_factory import get_client_from_cli_profile
 from azure.common.credentials import get_cli_profile
 from azure.cosmosdb.table.tableservice import TableService
+from azure.identity import AzureCliCredential
 from azure.mgmt.applicationinsights import ApplicationInsightsManagementClient
 from azure.mgmt.applicationinsights.models import (
     ApplicationInsightsComponentExportRequest,
@@ -190,8 +190,9 @@ class Client:
         return self.subscription_id
 
     def get_location_display_name(self) -> str:
-        location_client = get_client_from_cli_profile(
+        credential = AzureCliCredential()
-            SubscriptionClient, subscription_id=self.get_subscription_id()
+        location_client = SubscriptionClient(
+            credential, subscription_id=self.get_subscription_id()
         )
         locations = location_client.subscriptions.list_locations(
             self.get_subscription_id()
@@ -211,8 +212,9 @@ class Client:
         with open(self.arm_template, "r") as handle:
             arm = json.load(handle)
 
-        client = get_client_from_cli_profile(
+        credential = AzureCliCredential()
-            ResourceManagementClient, subscription_id=self.get_subscription_id()
+        client = ResourceManagementClient(
+            credential, subscription_id=self.get_subscription_id()
         )
         providers = {x.namespace: x for x in client.providers.list()}
 
@@ -524,8 +526,9 @@ class Client:
         with open(self.arm_template, "r") as template_handle:
             template = json.load(template_handle)
 
-        client = get_client_from_cli_profile(
+        credential = AzureCliCredential()
-            ResourceManagementClient, subscription_id=self.get_subscription_id()
+        client = ResourceManagementClient(
+            credential, subscription_id=self.get_subscription_id()
         )
         client.resource_groups.create_or_update(
             self.resource_group, {"location": self.location}
@@ -695,8 +698,10 @@ class Client:
         logger.info("creating eventgrid subscription")
         src_resource_id = self.results["deploy"]["fuzz-storage"]["value"]
         dst_resource_id = self.results["deploy"]["func-storage"]["value"]
-        client = get_client_from_cli_profile(
+
-            StorageManagementClient, subscription_id=self.get_subscription_id()
+        credential = AzureCliCredential()
+        client = StorageManagementClient(
+            credential, subscription_id=self.get_subscription_id()
         )
         event_subscription_info = EventSubscription(
             destination=StorageQueueEventSubscriptionDestination(
@@ -714,8 +719,8 @@ class Client:
             ),
         )
 
-        client = get_client_from_cli_profile(
+        client = EventGridManagementClient(
-            EventGridManagementClient, subscription_id=self.get_subscription_id()
+            credential, subscription_id=self.get_subscription_id()
         )
         result = client.event_subscriptions.begin_create_or_update(
             src_resource_id, "onefuzz1", event_subscription_info
@@ -789,8 +794,9 @@ class Client:
             destination_address=url,
         )
 
-        app_insight_client = get_client_from_cli_profile(
+        credential = AzureCliCredential()
-            ApplicationInsightsManagementClient,
+        app_insight_client = ApplicationInsightsManagementClient(
+            credential,
             subscription_id=self.get_subscription_id(),
         )
 

src/deployment/deploylib/data_migration.py

@@ -8,9 +8,10 @@ import json
 from typing import Callable, Dict, List
 from uuid import UUID
 
-from azure.common.client_factory import get_client_from_cli_profile
 from azure.cosmosdb.table.tablebatch import TableBatch
 from azure.cosmosdb.table.tableservice import TableService
+from azure.identity import AzureCliCredential
+from azure.mgmt.resource import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
 
 
@@ -91,7 +92,8 @@ def main() -> None:
     parser.add_argument("migration", choices=migrations.keys(), nargs="+")
     args = parser.parse_args()
 
-    client = get_client_from_cli_profile(StorageManagementClient)
+    credential = AzureCliCredential()
+    client = StorageManagementClient(credential)
     storage_keys = client.storage_accounts.list_keys(
         args.resource_group, args.storage_account
     )

src/deployment/deploylib/set_admins.py

@@ -6,8 +6,9 @@
 import argparse
 from uuid import UUID
 
-from azure.common.client_factory import get_client_from_cli_profile
 from azure.cosmosdb.table.tableservice import TableService
+from azure.identity import AzureCliCredential
+from azure.mgmt.resource import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
 
 from deploylib.configuration import (
@@ -26,7 +27,8 @@ def main() -> None:
     parser.add_argument("--allowed_aad_tenants", type=UUID, nargs="*")
     args = parser.parse_args()
 
-    client = get_client_from_cli_profile(StorageManagementClient)
+    credential = AzureCliCredential()
+    client = StorageManagementClient(credential)
     storage_keys = client.storage_accounts.list_keys(
         args.resource_group, args.storage_account
     )

src/deployment/requirements.txt

@@ -1,10 +1,10 @@
-azure-cli-core==2.27.2
+azure-cli-core==2.31.0
-azure-cli==2.27.2
+azure-cli==2.31.0
 azure-cosmosdb-table==1.0.6
 azure-mgmt-eventgrid==9.0.0
-azure-mgmt-resource==18.0.0
+azure-mgmt-resource==20.0.0
-azure-mgmt-storage==18.0.0
+azure-mgmt-storage==19.0.0
-azure-storage-blob==12.8.1
+azure-storage-blob==12.9.0
 pyfunctional==1.4.3
 pyopenssl==19.1.0
 adal~=1.2.5

src/utils/add-corpus-storage-accounts/add-corpus-storage-account.py

@@ -7,9 +7,10 @@ import argparse
 import json
 import uuid
 
-from azure.common.client_factory import get_client_from_cli_profile
+from azure.identity import AzureCliCredential
 from azure.mgmt.eventgrid import EventGridManagementClient
 from azure.mgmt.eventgrid.models import EventSubscription
+from azure.mgmt.resource import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
 from azure.mgmt.storage.models import (
     AccessTier,
@@ -42,7 +43,8 @@ def get_base_event(
 
 
 def add_event_grid(src_account_id: str, resource_group: str, location: str) -> None:
-    client = get_client_from_cli_profile(EventGridManagementClient)
+    credential = AzureCliCredential()
+    client = EventGridManagementClient(credential)
     base = get_base_event(client, resource_group, location)
 
     event_subscription_info = EventSubscription(
@@ -74,7 +76,8 @@ def create_storage(resource_group: str, account_name: str, location: str) -> str
         minimum_tls_version="TLS1_2",
     )
 
-    client = get_client_from_cli_profile(StorageManagementClient)
+    credential = AzureCliCredential()
+    client = StorageManagementClient(credential)
     account = client.storage_accounts.begin_create(
         resource_group, account_name, params
     ).result()

src/utils/add-corpus-storage-accounts/requirements.txt

@@ -1,3 +1,3 @@
-azure-mgmt-storage~=18.0.0
+azure-mgmt-storage~=19.0.0
-azure-cli-core==2.27.2
+azure-cli-core==2.31.0
 azure-mgmt-eventgrid==3.0.0rc9

src/utils/check-pr/requirements.txt

@@ -1,5 +1,5 @@
 azure-common~=1.1.25
-azure-identity==1.7.0
+azure-identity==1.7.1
 PyGithub==1.55
-azure-cli-core==2.27.2
+azure-cli-core==2.31.0
 msgraph-core==0.2.2