mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-15 03:18:07 +00:00
Support retry during function deploy (#330)
Starting earlier today, I saw roughly 1 in 3 deployments fail with the error `Azure.Functions.Cli.Common.CliException: Timed out waiting for SCM to update the Environment Settings`. Redeploying the application resolves the issue. New builds and past releases alike hit this exception. According to https://github.com/Azure/azure-functions-core-tools/issues/1863, function app deployments may fail due to timeouts related to cold-start. This PR executes the deploy in a loop with a delay in the case of failure.
This commit is contained in:
@ -16,6 +16,7 @@ import time
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from azure.common.client_factory import get_client_from_cli_profile
|
||||
from azure.common.credentials import get_cli_profile
|
||||
@ -658,19 +659,34 @@ class Client:
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
with zipfile.ZipFile(self.app_zip, "r") as zip_ref:
|
||||
zip_ref.extractall(tmpdirname)
|
||||
subprocess.check_output(
|
||||
[
|
||||
shutil.which("func"),
|
||||
"azure",
|
||||
"functionapp",
|
||||
"publish",
|
||||
self.application_name,
|
||||
"--python",
|
||||
"--no-build",
|
||||
],
|
||||
env=dict(os.environ, CLI_DEBUG="1"),
|
||||
cwd=tmpdirname,
|
||||
)
|
||||
error: Optional[subprocess.CalledProcessError] = None
|
||||
max_tries = 5
|
||||
for i in range(max_tries):
|
||||
try:
|
||||
subprocess.check_output(
|
||||
[
|
||||
shutil.which("func"),
|
||||
"azure",
|
||||
"functionapp",
|
||||
"publish",
|
||||
self.application_name,
|
||||
"--python",
|
||||
"--no-build",
|
||||
],
|
||||
env=dict(os.environ, CLI_DEBUG="1"),
|
||||
cwd=tmpdirname,
|
||||
)
|
||||
return
|
||||
except subprocess.CalledProcessError as err:
|
||||
error = err
|
||||
if i + 1 < max_tries:
|
||||
logger.debug("func failure error: %s", err)
|
||||
logger.warning(
|
||||
"function failed to deploy, waiting 60 seconds and trying again"
|
||||
)
|
||||
time.sleep(60)
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
def update_registration(self):
|
||||
if not self.create_registration:
|
||||
|
Reference in New Issue
Block a user