From 15d92cad5276f534f551c1c7377f1bc93f6740de Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 15 Aug 2022 09:24:26 -0700 Subject: [PATCH] =?UTF-8?q?Retry=20on=20"No=20connection=20could=20be=20ma?= =?UTF-8?q?de=20because=20target=20machine=20actively=E2=80=A6=20(#2252)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Retry on "No connection could be made because target machine actively refused it" when connecting debugger using create_and_connect command * limit number of retries Co-authored-by: stas --- src/cli/onefuzz/api.py | 68 ++++++++++++++++++++++++++++-------------- src/cli/onefuzz/ssh.py | 12 +++++++- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index e1505ed9d..8b36b85cb 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -9,6 +9,7 @@ import os import pkgutil import re import subprocess # nosec +import time import uuid from enum import Enum from shutil import which @@ -553,7 +554,9 @@ class Repro(Endpoint): return None def _dbg_windows( - self, repro: models.Repro, debug_command: Optional[str] + self, + repro: models.Repro, + debug_command: Optional[str], ) -> Optional[str]: """Setup an SSH tunnel, then connect via CDB over SSH tunnel""" @@ -564,33 +567,54 @@ class Repro(Endpoint): ): raise Exception("vm setup failed: %s" % repro.state) + NUM_RETRIES = 10 bind_all = which("wslpath") is not None and repro.os == enums.OS.windows proxy = "*:" + REPRO_SSH_FORWARD if bind_all else REPRO_SSH_FORWARD with ssh_connect(repro.ip, repro.auth.private_key, proxy=proxy): dbg = ["cdb.exe", "-remote", "tcp:port=1337,server=localhost"] - if debug_command: - dbg_script = [debug_command, "qq"] - with temp_file("db.script", "\r\n".join(dbg_script)) as dbg_script_path: - dbg += ["-cf", _wsl_path(dbg_script_path)] + while NUM_RETRIES > 0: + NUM_RETRIES = NUM_RETRIES - 1 + if debug_command: + dbg_script = [debug_command, "qq"] + with temp_file( + "db.script", "\r\n".join(dbg_script) + ) as dbg_script_path: + dbg += ["-cf", _wsl_path(dbg_script_path)] + logging.debug("launching: %s", dbg) + try: + # security note: dbg is built from content coming from the server, + # which is trusted in this context. + return subprocess.run( # nosec + dbg, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ).stdout.decode(errors="ignore") + except subprocess.CalledProcessError as err: + if err.returncode == 0x8007274D: + self.logger.info( + "failed to connect to debug-server trying again in 10 seconds..." + ) + time.sleep(10.0) + else: + self.logger.error( + "debug failed: %s", + err.output.decode(errors="ignore"), + ) + raise err + else: logging.debug("launching: %s", dbg) + # security note: dbg is built from content coming from the + # server, which is trusted in this context. try: - # security note: dbg is built from content coming from the server, - # which is trusted in this context. - return subprocess.run( # nosec - dbg, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ).stdout.decode(errors="ignore") + subprocess.check_call(dbg) # nosec + break except subprocess.CalledProcessError as err: - self.logger.error( - "debug failed: %s", err.output.decode(errors="ignore") - ) - raise err - else: - logging.debug("launching: %s", dbg) - # security note: dbg is built from content coming from the - # server, which is trusted in this context. - subprocess.call(dbg) # nosec - + if err.returncode == 0x8007274D: + self.logger.info( + "failed to connect to debug-server trying again in 10 seconds..." + ) + time.sleep(10.0) + else: + break return None def connect( @@ -635,9 +659,9 @@ class Repro(Endpoint): ) repro = wait(func) - + # give time for debug server to initialize + time.sleep(30.0) result: Optional[str] = None - if repro.os == enums.OS.windows: result = self._dbg_windows(repro, debug_command) elif repro.os == enums.OS.linux: diff --git a/src/cli/onefuzz/ssh.py b/src/cli/onefuzz/ssh.py index 9ed2f30fb..6301a2dba 100644 --- a/src/cli/onefuzz/ssh.py +++ b/src/cli/onefuzz/ssh.py @@ -60,6 +60,7 @@ def build_ssh_command( proxy: Optional[str] = None, port: Optional[int] = None, command: Optional[str] = None, + num_auth_retries: Optional[int] = None, ) -> Generator: with temp_file("id_rsa", private_key, set_owner_only=True) as ssh_key: cmd = [ @@ -82,6 +83,9 @@ def build_ssh_command( if log_level <= logging.DEBUG: cmd += ["-v"] + if num_auth_retries: + cmd += ["authentication-retries", str(num_auth_retries)] + if command: cmd += [command] @@ -97,9 +101,15 @@ def ssh_connect( call: bool = False, port: Optional[int] = None, command: Optional[str] = None, + num_auth_retries: Optional[int] = None, ) -> Generator: with build_ssh_command( - ip, private_key, proxy=proxy, port=port, command=command + ip, + private_key, + proxy=proxy, + port=port, + command=command, + num_auth_retries=num_auth_retries, ) as cmd: logging.info("launching ssh: %s", " ".join(cmd))