ExternalVendorCode/balena-supervisor
1
0
Fork 0
mirror of https://github.com/balena-os/balena-supervisor.git synced 2025-04-10 04:39:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for Balena deltas

Browse Source 
Resin’s delta server supports Balena deltas as version 3 deltas. This commit adds support for triggering delta generation for Balena deltas, and applying them locally to the device via a simple pull.

The delta version to use when updating has been abstracted away as an env var that is user-defined. The default value is still instructing use of rsync deltas (v2).

Change-Type: minor
This commit is contained in:
Akis Kesoglou 2018-04-25 14:22:50 +03:00
parent 56ecd845f7
commit 8479801674
6 changed files with 50 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -34,7 +34,7 @@
"coffee-script": "~1.11.0",
"copy-webpack-plugin": "^4.2.3",
"dbus-native": "^0.2.5",
"docker-delta": "^2.0.4",
"docker-delta": "^2.1.0",
"docker-progress": "^2.7.2",
"docker-toolbelt": "^3.2.1",
"duration-js": "^4.0.0",

2
src/compose/images.coffee
View File

@ -64,7 +64,7 @@ module.exports = class Images extends EventEmitter
@inspectByName(opts.deltaSource)
.then (srcImage) =>
opts.deltaSourceId = srcImage.Id
@docker.rsyncImageWithProgress(imageName, opts, onProgress)
@docker.fetchDeltaWithProgress(imageName, opts, onProgress)
.tap (id) =>
if !hasDigest(imageName)
@docker.getRepoAndTag(imageName)

3
src/config.coffee
View File

@ -94,6 +94,7 @@ module.exports = class Config extends EventEmitter
'deltaApplyTimeout'
'deltaRetryCount'
'deltaRetryInterval'
'deltaVersion'
])
@schema = {
@ -131,6 +132,7 @@ module.exports = class Config extends EventEmitter
extendedEnvOptions: { source: 'func' }
fetchOptions: { source: 'func' }
# NOTE: all 'db' values are stored and loaded as *strings*
apiSecret: { source: 'db', mutable: true }
logsChannelSecret: { source: 'db', mutable: true }
name: { source: 'db', mutable: true }
@ -145,6 +147,7 @@ module.exports = class Config extends EventEmitter
deltaApplyTimeout: { source: 'db', mutable: true, default: '' }
deltaRetryCount: { source: 'db', mutable: true, default: '30' }
deltaRetryInterval: { source: 'db', mutable: true, default: '10000' }
deltaVersion: { source: 'db', mutable: true, default: '2' }
lockOverride: { source: 'db', mutable: true, default: 'false' }
legacyAppsPresent: { source: 'db', mutable: true, default: 'false' }
nativeLogger: { source: 'db', mutable: true, default: 'true' }

1
src/device-config.coffee
View File

@ -52,6 +52,7 @@ module.exports = class DeviceConfig
deltaApplyTimeout: { envVarName: 'RESIN_SUPERVISOR_DELTA_APPLY_TIMEOUT', varType: 'int', defaultValue: '' }
deltaRetryCount: { envVarName: 'RESIN_SUPERVISOR_DELTA_RETRY_COUNT', varType: 'int', defaultValue: '30' }
deltaRetryInterval: { envVarName: 'RESIN_SUPERVISOR_DELTA_RETRY_INTERVAL', varType: 'int', defaultValue: '10000' }
deltaVersion: { envVarName: 'RESIN_SUPERVISOR_DELTA_VERSION', varType: 'int', defaultValue: '2' }
lockOverride: { envVarName: 'RESIN_SUPERVISOR_OVERRIDE_LOCK', varType: 'bool', defaultValue: 'false' }
nativeLogger: { envVarName: 'RESIN_SUPERVISOR_NATIVE_LOGGER', varType: 'bool', defaultValue: 'true' }
}

60
src/lib/docker-utils.coffee
View File

@ -9,8 +9,8 @@ _ = require 'lodash'
{ envArrayToObject } = require './conversions'
{ checkInt } = require './validation'
applyDelta = (imgSrc, deltaUrl, applyTimeout, opts, onProgress, log) ->
log('Applying delta...')
applyRsyncDelta = (imgSrc, deltaUrl, applyTimeout, opts, onProgress, log) ->
log('Applying rsync delta...')
new Promise (resolve, reject) ->
req = resumable(Object.assign({ url: deltaUrl }, opts))
.on('progress', onProgress)
@ -22,7 +22,7 @@ applyDelta = (imgSrc, deltaUrl, applyTimeout, opts, onProgress, log) ->
else if parseInt(res.headers['content-length']) is 0
reject(new Error('Invalid delta URL.'))
else
deltaStream = dockerDelta.applyDelta(imgSrc, timeout: applyTimeout)
deltaStream = dockerDelta.applyDelta(imgSrc, { log, timeout: applyTimeout })
res.pipe(deltaStream)
.on('id', (id) -> resolve('sha256:' + id))
.on 'error', (err) ->
@ -30,6 +30,15 @@ applyDelta = (imgSrc, deltaUrl, applyTimeout, opts, onProgress, log) ->
req.abort(err)
reject(err)
applyBalenaDelta = (docker, deltaImg, token, onProgress, log) ->
log('Applying balena delta...')
if token?
log('Using registry auth token')
auth = { authconfig: registrytoken: token }
docker.dockerProgress.pull(deltaImg, onProgress, auth)
.then =>
docker.getImage(deltaImg).inspect().get('Id')
module.exports = class DockerUtils extends DockerToolbelt
constructor: (opts) ->
super(opts)
@ -48,23 +57,29 @@ module.exports = class DockerUtils extends DockerToolbelt
repoName = imageName
return { repo: repoName, tag: tagName }
# TODO: somehow fix this to work with image names having repo digests instead of tags
rsyncImageWithProgress: (imgDest, fullDeltaOpts, onProgress) =>
fetchDeltaWithProgress: (imgDest, fullDeltaOpts, onProgress) =>
{
deltaRequestTimeout, deltaApplyTimeout, deltaRetryCount, deltaRetryInterval,
uuid, currentApiKey, deltaEndpoint, resinApiEndpoint,
deltaSource, deltaSourceId, startFromEmpty = false
deltaSource, deltaSourceId, deltaVersion, startFromEmpty = false
} = fullDeltaOpts
retryCount = checkInt(deltaRetryCount)
retryInterval = checkInt(deltaRetryInterval)
requestTimeout = checkInt(deltaRequestTimeout)
applyTimeout = checkInt(deltaApplyTimeout)
version = checkInt(deltaVersion)
deltaSource = 'resin/scratch' if startFromEmpty or !deltaSource?
deltaSourceId ?= deltaSource
log = (str) ->
console.log("delta(#{deltaSource}): #{str}")
if not (version in [ 2, 3 ])
log("Unsupported delta version: #{version}. Falling back to regular pull")
return @fetchImageWithProgress(imgDest, fullDeltaOpts, onProgress)
docker = this
log("Starting delta to #{imgDest}")
Promise.join @getRegistryAndName(imgDest), @getRegistryAndName(deltaSource), (dstInfo, srcInfo) ->
tokenEndpoint = "#{resinApiEndpoint}/auth/v1/token"
@ -79,31 +94,38 @@ module.exports = class DockerUtils extends DockerToolbelt
request.getAsync(url, opts)
.get(1)
.then (responseBody) ->
token = responseBody?.token
opts =
followRedirect: false
timeout: requestTimeout
if responseBody?.token?
if token?
opts.auth =
bearer: responseBody.token
bearer: token
sendImmediately: true
new Promise (resolve, reject) ->
request.get("#{deltaEndpoint}/api/v2/delta?src=#{deltaSource}&dest=#{imgDest}", opts)
.on 'response', (res) ->
res.resume() # discard response body -- we only care about response headers
if res.statusCode in [ 502, 504 ]
reject(new Error('Delta server is still processing the delta, will retry'))
else if not (300 <= res.statusCode < 400 and res.headers['location']?)
reject(new Error("Got #{res.statusCode} when requesting image from delta server."))
else
request.getAsync("#{deltaEndpoint}/api/v#{version}/delta?src=#{deltaSource}&dest=#{imgDest}", opts)
.spread (res, data) ->
if res.statusCode in [ 502, 504 ]
throw new Error('Delta server is still processing the delta, will retry')
switch version
when 2
if not (300 <= res.statusCode < 400 and res.headers['location']?)
throw new Error("Got #{res.statusCode} when requesting image from delta server.")
deltaUrl = res.headers['location']
if deltaSource is 'resin/scratch'
deltaSrc = null
else
deltaSrc = deltaSourceId
resumeOpts = { timeout: requestTimeout, maxRetries: retryCount, retryInterval }
resolve(applyDelta(deltaSrc, deltaUrl, applyTimeout, resumeOpts, onProgress, log))
.on 'error', reject
applyRsyncDelta(deltaSrc, deltaUrl, applyTimeout, resumeOpts, onProgress, log)
when 3
if res.statusCode isnt 200
throw new Error("Got #{res.statusCode} when requesting image from delta server.")
name = JSON.parse(data).name
applyBalenaDelta(docker, name, token, onProgress, log)
else
# we guard against arbitrary versions above, so this can't really happen
throw new Error("Unsupported delta version: #{version}")
.catch dockerDelta.OutOfSyncError, (err) =>
log('Falling back to regular pull')
@fetchImageWithProgress(imgDest, fullDeltaOpts, onProgress)

3
test/05-device-state.spec.coffee
View File

@ -19,6 +19,7 @@ mockedInitialConfig = {
'RESIN_SUPERVISOR_DELTA_REQUEST_TIMEOUT': '30000'
'RESIN_SUPERVISOR_DELTA_RETRY_COUNT': '30'
'RESIN_SUPERVISOR_DELTA_RETRY_INTERVAL': '10000'
'RESIN_SUPERVISOR_DELTA_VERSION': '2'
'RESIN_SUPERVISOR_LOCAL_MODE': 'false'
'RESIN_SUPERVISOR_LOG_CONTROL': 'true'
'RESIN_SUPERVISOR_OVERRIDE_LOCK': 'false'
@ -37,6 +38,7 @@ testTarget1 = {
'RESIN_SUPERVISOR_DELTA_REQUEST_TIMEOUT': '30000'
'RESIN_SUPERVISOR_DELTA_RETRY_COUNT': '30'
'RESIN_SUPERVISOR_DELTA_RETRY_INTERVAL': '10000'
'RESIN_SUPERVISOR_DELTA_VERSION': '2'
'RESIN_SUPERVISOR_LOCAL_MODE': 'false'
'RESIN_SUPERVISOR_LOG_CONTROL': 'true'
'RESIN_SUPERVISOR_NATIVE_LOGGER': 'true'
@ -118,6 +120,7 @@ testTargetWithDefaults2 = {
'RESIN_SUPERVISOR_DELTA_REQUEST_TIMEOUT': '30000'
'RESIN_SUPERVISOR_DELTA_RETRY_COUNT': '30'
'RESIN_SUPERVISOR_DELTA_RETRY_INTERVAL': '10000'
'RESIN_SUPERVISOR_DELTA_VERSION': '2'
'RESIN_SUPERVISOR_LOCAL_MODE': 'false'
'RESIN_SUPERVISOR_LOG_CONTROL': 'true'
'RESIN_SUPERVISOR_NATIVE_LOGGER': 'true'