From 9c8ac5812852146df2236dd5fbfc6506b57e35df Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 13 Jan 2017 13:09:57 -0300 Subject: [PATCH] Issue #378: Increase delta timeouts (to avoid ESOCKETTIMEOUT errors), make them configurable, and provide nicer message when the delta server times out Current delta timeouts are too limiting, so we increase the request timeout to 30 minutes which is big enough that the server will time out first and we can provide a nice message letting the user know we'll retry; and we increase the total timeout to 24 hours to account for really big deltas over slower connections (the rsync calls will time out anyways if something else goes wrong, as they have a 5 minute I/O timeout). The timeouts are now configurable with the RESIN_SUPERVISOR_DELTA_REQUEST_TIMEOUT and RESIN_SUPERVISOR_DELTA_TOTAL_TIMEOUT configuration variables. Change-Type: minor Signed-off-by: Pablo Carranza Velez --- src/application.coffee | 4 +++- src/docker-utils.coffee | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/application.coffee b/src/application.coffee index ac468366..c143c305 100644 --- a/src/application.coffee +++ b/src/application.coffee @@ -196,7 +196,9 @@ fetch = (app, setDeviceUpdateState = true) -> if conf['RESIN_SUPERVISOR_DELTA'] == '1' logSystemEvent(logTypes.downloadAppDelta, app) - dockerUtils.rsyncImageWithProgress(app.imageId, onProgress) + requestTimeout = conf['RESIN_SUPERVISOR_DELTA_REQUEST_TIMEOUT'] ? 30 * 60 * 1000 + totalTimeout = conf['RESIN_SUPERVISOR_DELTA_TOTAL_TIMEOUT'] ? 24 * 60 * 60 * 1000 + dockerUtils.rsyncImageWithProgress(app.imageId, { requestTimeout, totalTimeout }, onProgress) else logSystemEvent(logTypes.downloadApp, app) dockerUtils.fetchImageWithProgress(app.imageId, onProgress) diff --git a/src/docker-utils.coffee b/src/docker-utils.coffee index 102162fd..af73dff4 100644 --- a/src/docker-utils.coffee +++ b/src/docker-utils.coffee @@ -43,9 +43,6 @@ findSimilarImage = (repoTag) -> # Otherwise we start from scratch return 'resin/scratch' -DELTA_REQUEST_TIMEOUT = 15 * 60 * 1000 -DELTA_TOTAL_TIMEOUT = 30 * 60 * 1000 - getRepoAndTag = (image) -> docker.getRegistryAndName(image) .then ({ registry, imageName, tagName }) -> @@ -65,7 +62,7 @@ do -> .disposer (release) -> release() - exports.rsyncImageWithProgress = (imgDest, onProgress, startFromEmpty = false) -> + exports.rsyncImageWithProgress = (imgDest, { requestTimeout, totalTimeout, startFromEmpty = false }, onProgress) -> Promise.using readLockImages(), -> Promise.try -> if startFromEmpty @@ -73,14 +70,16 @@ do -> findSimilarImage(imgDest) .then (imgSrc) -> new Promise (resolve, reject) -> - progress request.get("#{config.deltaHost}/api/v2/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: DELTA_REQUEST_TIMEOUT) + progress request.get("#{config.deltaHost}/api/v2/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: requestTimeout) .on 'progress', (progress) -> # In request-progress ^2.0.1, "percentage" is a ratio from 0 to 1 onProgress(percentage: progress.percentage * 100) .on 'end', -> onProgress(percentage: 100) .on 'response', (res) -> - if res.statusCode isnt 200 + if res.statusCode is 504 + reject(new Error('Delta server is still processing the delta, will retry')) + else if res.statusCode isnt 200 reject(new Error("Got #{res.statusCode} when requesting image from delta server.")) else if imgSrc is 'resin/scratch' @@ -91,14 +90,14 @@ do -> .on('id', resolve) .on('error', reject) .on 'error', reject - .timeout(DELTA_TOTAL_TIMEOUT) + .timeout(totalTimeout) .then (id) -> getRepoAndTag(imgDest) .then ({ repo, tag }) -> docker.getImage(id).tagAsync({ repo, tag, force: true }) .catch dockerDelta.OutOfSyncError, (err) -> console.log('Falling back to delta-from-empty') - exports.rsyncImageWithProgress(imgDest, onProgress, true) + exports.rsyncImageWithProgress(imgDest, { requestTimeout, totalTimeout, startFromEmpty: true }, onProgress) exports.fetchImageWithProgress = (image, onProgress) -> Promise.using readLockImages(), ->