mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-16 11:58:09 +00:00
Support retry during function deploy (#330)
Starting earlier today, I saw roughly 1 in 3 deployments fail with the error `Azure.Functions.Cli.Common.CliException: Timed out waiting for SCM to update the Environment Settings`. Redeploying the application resolves the issue. New builds and past releases alike hit this exception. According to https://github.com/Azure/azure-functions-core-tools/issues/1863, function app deployments may fail due to timeouts related to cold-start. This PR executes the deploy in a loop with a delay in the case of failure.
This commit is contained in:
@ -16,6 +16,7 @@ import time
|
|||||||
import uuid
|
import uuid
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
from azure.common.client_factory import get_client_from_cli_profile
|
from azure.common.client_factory import get_client_from_cli_profile
|
||||||
from azure.common.credentials import get_cli_profile
|
from azure.common.credentials import get_cli_profile
|
||||||
@ -658,6 +659,10 @@ class Client:
|
|||||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||||
with zipfile.ZipFile(self.app_zip, "r") as zip_ref:
|
with zipfile.ZipFile(self.app_zip, "r") as zip_ref:
|
||||||
zip_ref.extractall(tmpdirname)
|
zip_ref.extractall(tmpdirname)
|
||||||
|
error: Optional[subprocess.CalledProcessError] = None
|
||||||
|
max_tries = 5
|
||||||
|
for i in range(max_tries):
|
||||||
|
try:
|
||||||
subprocess.check_output(
|
subprocess.check_output(
|
||||||
[
|
[
|
||||||
shutil.which("func"),
|
shutil.which("func"),
|
||||||
@ -671,6 +676,17 @@ class Client:
|
|||||||
env=dict(os.environ, CLI_DEBUG="1"),
|
env=dict(os.environ, CLI_DEBUG="1"),
|
||||||
cwd=tmpdirname,
|
cwd=tmpdirname,
|
||||||
)
|
)
|
||||||
|
return
|
||||||
|
except subprocess.CalledProcessError as err:
|
||||||
|
error = err
|
||||||
|
if i + 1 < max_tries:
|
||||||
|
logger.debug("func failure error: %s", err)
|
||||||
|
logger.warning(
|
||||||
|
"function failed to deploy, waiting 60 seconds and trying again"
|
||||||
|
)
|
||||||
|
time.sleep(60)
|
||||||
|
if error is not None:
|
||||||
|
raise error
|
||||||
|
|
||||||
def update_registration(self):
|
def update_registration(self):
|
||||||
if not self.create_registration:
|
if not self.create_registration:
|
||||||
|
Reference in New Issue
Block a user