import { expect } from 'chai'; import * as sinon from 'sinon'; import * as Docker from 'dockerode'; import * as applicationManager from '~/src/compose/application-manager'; import * as imageManager from '~/src/compose/images'; import * as serviceManager from '~/src/compose/service-manager'; import Network from '~/src/compose/network'; import * as networkManager from '~/src/compose/network-manager'; import Volume from '~/src/compose/volume'; import * as config from '~/src/config'; import { createDockerImage } from '~/test-lib/docker-helper'; import { createService, createImage, createApps, createCurrentState, DEFAULT_NETWORK, } from '~/test-lib/state-helper'; // TODO: application manager inferNextSteps still queries some stuff from // the engine instead of receiving that information as parameter. Refactoring // the method to be more of a pure function would allow us to move a lot of these tests // to unit tests, leaving the need of integration tests just for more complex stuff that // the application-manager also does and that is not currently tested. // TODO: also, there is some redundancy between what is tested here and what is tested in // the app spec, remove that redundancy to simplify the tests describe('compose/application-manager', () => { before(async () => { // Service.fromComposeObject gets api keys from the database // which also depend on the local mode. This ensures the database // is initialized. This can be removed when ApplicationManager and Service // a refactored to work as pure functions await config.initialized(); }); beforeEach(async () => { // Set up network by default await networkManager.ensureSupervisorNetwork(); }); afterEach(async () => { // Delete any created networks const docker = new Docker(); const allNetworks = await docker.listNetworks(); await Promise.all( allNetworks // exclude docker default networks from the cleanup .filter(({ Name }) => !['bridge', 'host', 'none'].includes(Name)) .map(({ Name }) => docker.getNetwork(Name).remove()), ); }); // TODO: we don't test application manager initialization as it sets up a bunch of timers // and listeners that may affect other tests. This is a bad pattern and it needs to be purged // from the codebase it.skip('should init', async () => { await applicationManager.initialized(); }); // TODO: missing tests for getCurrentApps it('should not infer a start step when all that changes is a running state', async () => { const targetApps = createApps( { services: [await createService({ running: true, appId: 1 })], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [await createService({ running: false, appId: 1 })], networks: [DEFAULT_NETWORK], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // There should be no steps since the engine manages restart policy for stopped containers expect(steps.length).to.equal(0); }); it('infers a kill step when a service has to be removed', async () => { const targetApps = createApps( { services: [], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [await createService()], networks: [DEFAULT_NETWORK], }); const [killStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(killStep).to.have.property('action').that.equals('kill'); expect(killStep) .to.have.property('current') .that.deep.includes({ serviceName: 'main' }); }); it('infers a fetch step when a service has to be updated', async () => { const targetApps = createApps( { services: [await createService({ image: 'image-new', appId: 1 })], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [await createService({ appId: 1 })], networks: [DEFAULT_NETWORK], images: [], }); const [fetchStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(fetchStep).to.have.property('action').that.equals('fetch'); expect(fetchStep) .to.have.property('image') .that.deep.includes({ name: 'image-new' }); }); it('does not infer a fetch step when the download is already in progress', async () => { const targetApps = createApps( { services: [await createService({ image: 'image-new', appId: 1 })], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [await createService({ appId: 1 })], networks: [DEFAULT_NETWORK], downloading: ['image-new'], }); const [noopStep, ...nextSteps] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(noopStep).to.have.property('action').that.equals('noop'); expect(nextSteps).to.have.lengthOf(0); }); it('infers a kill step when a service has to be updated but the strategy is kill-then-download', async () => { const labels = { 'io.balena.update.strategy': 'kill-then-download', }; const targetApps = createApps( { services: [ await createService({ image: 'image-new', labels, appId: 1, commit: 'new-release', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'image-old', labels, appId: 1, commit: 'old-release', }, { options: { imageInfo: { Id: 'sha256:image-old-id' } } }, ), ], networks: [DEFAULT_NETWORK], }); const [killStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(killStep).to.have.property('action').that.equals('kill'); expect(killStep) .to.have.property('current') .that.deep.includes({ serviceName: 'main' }); }); it('infers a kill step when a service has to be updated but the strategy is delete-then-download', async () => { const labels = { 'io.balena.update.strategy': 'delete-then-download', }; const targetApps = createApps( { services: [ await createService({ image: 'image-new', labels, appId: 1, commit: 'new-release', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'image-old', labels, appId: 1, commit: 'old-release', }, { options: { imageInfo: { Id: 'sha256:image-old-id' } } }, ), ], networks: [DEFAULT_NETWORK], }); const [killStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(killStep).to.have.property('action').that.equals('kill'); expect(killStep) .to.have.property('current') .that.deep.includes({ serviceName: 'main' }); }); it('infers a remove step when the current service has stopped and the strategy is delete-then-download', async () => { const labels = { 'io.balena.update.strategy': 'delete-then-download', }; const targetApps = createApps( { services: [ await createService({ image: 'image-new', labels, appId: 1, serviceName: 'main', commit: 'new-release', }), ], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], images: [ createImage({ appId: 1, name: 'image-old', serviceName: 'main', dockerImageId: 'image-old-id', }), ], networks: [DEFAULT_NETWORK], }); const [removeImage] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // First we should see a kill expect(removeImage).to.have.property('action').that.equals('removeImage'); expect(removeImage) .to.have.property('image') .that.deep.includes({ name: 'image-old' }); }); it('does not infer to kill a service with default strategy if a dependency is not downloaded', async () => { const targetApps = createApps( { services: [ await createService({ image: 'main-image', appId: 1, commit: 'new-release', serviceName: 'main', composition: { depends_on: ['dep'], }, }), await createService({ image: 'dep-image', appId: 1, commit: 'new-release', serviceName: 'dep', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService({ appId: 1, commit: 'old-release', serviceName: 'main', composition: { depends_on: ['dep'], }, }), await createService({ appId: 1, commit: 'old-release', serviceName: 'dep', }), ], networks: [DEFAULT_NETWORK], downloading: ['dep-image'], // dep-image is still being downloaded images: [ // main-image was already downloaded createImage({ appId: 1, name: 'main-image', serviceName: 'main', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // Only noop steps should be seen at this point expect(steps.filter((s) => s.action !== 'noop')).to.have.lengthOf(0); }); it('infers to kill several services as long as there is no unmet dependency', async () => { const targetApps = createApps( { services: [ await createService({ image: 'main-image', appId: 1, appUuid: 'appuuid', commit: 'new-release', serviceName: 'main', composition: { depends_on: ['dep'], }, }), await createService({ image: 'dep-image', appId: 1, appUuid: 'appuuid', commit: 'new-release', serviceName: 'dep', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService({ appId: 1, appUuid: 'appuuid', commit: 'old-release', serviceName: 'main', composition: { depends_on: ['dep'], }, }), await createService({ appId: 1, appUuid: 'appuuid', commit: 'old-release', serviceName: 'dep', }), ], networks: [DEFAULT_NETWORK], images: [ // Both images have been downloaded createImage({ appId: 1, appUuid: 'appuuid', name: 'main-image', serviceName: 'main', commit: 'new-release', }), createImage({ appId: 1, appUuid: 'appuuid', name: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // We should see kill steps for both currently running services expect( steps.filter( (s: any) => s.action === 'kill' && s.current.serviceName === 'dep', ), ).to.have.lengthOf(1); expect( steps.filter( (s: any) => s.action === 'kill' && s.current.serviceName === 'main', ), ).to.have.lengthOf(1); }); it('infers to start the dependency first', async () => { const targetApps = createApps( { services: [ await createService({ image: 'main-image', serviceName: 'main', commit: 'new-release', composition: { depends_on: ['dep'], }, }), await createService({ image: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], images: [ // Both images have been downloaded createImage({ name: 'main-image', serviceName: 'main', commit: 'new-release', }), createImage({ name: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], }); const [startStep, ...nextSteps] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step should happen for the depended service first expect(startStep).to.have.property('action').that.equals('start'); expect(startStep) .to.have.property('target') .that.deep.includes({ serviceName: 'dep' }); // No more steps until the first container has been started expect(nextSteps).to.have.lengthOf(0); }); it('infers to start a service once its dependency has been met', async () => { const targetApps = createApps( { services: [ await createService({ image: 'main-image', serviceName: 'main', commit: 'new-release', composition: { depends_on: ['dep'], }, }), await createService({ image: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService({ image: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], networks: [DEFAULT_NETWORK], images: [ // Both images have been downloaded createImage({ name: 'main-image', serviceName: 'main', commit: 'new-release', }), createImage({ name: 'dep-image', serviceName: 'dep', commit: 'new-release', }), ], }); const [startStep, ...nextSteps] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step shoud happen for the depended service first expect(startStep).to.have.property('action').that.equals('start'); expect(startStep) .to.have.property('target') .that.deep.includes({ serviceName: 'main' }); expect(nextSteps).to.have.lengthOf(0); }); it('infers to remove spurious containers', async () => { const targetApps = createApps( { services: [await createService({ image: 'main-image' })], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService({ appId: 5, serviceName: 'old-service' }), ], networks: [DEFAULT_NETWORK], images: [ // Image has been downloaded createImage({ name: 'main-image', serviceName: 'main', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // Start the new service expect( steps.filter( (s: any) => s.action === 'start' && s.target.serviceName === 'main', ), ).to.have.lengthOf(1); // Remove the leftover service expect( steps.filter( (s: any) => s.action === 'kill' && s.current.serviceName === 'old-service', ), ).to.have.lengthOf(1); }); it('should not remove an app volumes when they are no longer referenced', async () => { const targetApps = createApps({ networks: [DEFAULT_NETWORK] }, true); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], volumes: [Volume.fromComposeObject('test-volume', 1, 'deadbeef')], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(steps.filter((s) => s.action === 'removeVolume')).to.be.empty; }); it('should remove volumes from previous applications', async () => { const targetApps = createApps({ networks: [DEFAULT_NETWORK] }, true); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [], // Volume with different id volumes: [Volume.fromComposeObject('test-volume', 2, 'deadbeef')], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(steps.filter((s) => s.action === 'removeVolume')).to.not.be.empty; }); it('should remove volumes from previous applications except if keepVolumes is set', async () => { const targetApps = createApps({ networks: [DEFAULT_NETWORK] }, true); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [], // Volume with different id volumes: [Volume.fromComposeObject('test-volume', 2, 'deadbeef')], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { keepVolumes: true, downloading, availableImages, containerIdsByAppId, }, ); expect(steps.filter((s) => s.action === 'removeVolume')).to.be.empty; }); it('should infer that we need to create the supervisor network if it does not exist', async () => { const docker = new Docker(); await docker.getNetwork('supervisor0').remove(); const targetApps = createApps( { services: [await createService()], networks: [DEFAULT_NETWORK] }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], }); const [ensureNetworkStep, ...nextSteps] = await applicationManager.inferNextSteps(currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }); expect(ensureNetworkStep).to.deep.include({ action: 'ensureSupervisorNetwork', }); expect(nextSteps).to.have.lengthOf(0); }); it('should kill a service which depends on the supervisor network, if we need to create the network', async () => { const docker = new Docker(); await docker.getNetwork('supervisor0').remove(); const labels = { 'io.balena.features.supervisor-api': 'true' }; const targetApps = createApps( { services: [ await createService({ labels }, { options: { listenPort: '48484' } }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService({ labels }, { options: { listenPort: '48484' } }), ], networks: [DEFAULT_NETWORK], }); const [killStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step shoud happen for the depended service first expect(killStep).to.have.property('action').that.equals('kill'); expect(killStep) .to.have.property('current') .that.deep.includes({ serviceName: 'main' }); }); it('should infer a cleanup step when a cleanup is required', async () => { // Create a dangling image; this is done by building an image again with // some slightly different metadata, leaving the old image with no metadata. const docker = new Docker(); const dockerImageIdOne = await createDockerImage( 'some-image:some-tag', ['io.balena.testing=1'], docker, ); const dockerImageIdTwo = await createDockerImage( 'some-image:some-tag', ['io.balena.testing=2'], docker, ); // Remove the tagged image, leaving only the dangling image await docker.getImage(dockerImageIdTwo).remove(); const targetApps = createApps( { services: [await createService()], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [await createService()], networks: [DEFAULT_NETWORK], }); const [cleanupStep, ...nextSteps] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // Cleanup needs to happen first expect(cleanupStep).to.deep.include({ action: 'cleanup', }); expect(nextSteps).to.have.lengthOf(0); await docker.getImage(dockerImageIdOne).remove(); }); it('should infer that an image should be removed if it is no longer referenced in current or target state (only target)', async () => { const targetApps = createApps( { services: [ await createService( { image: 'main-image' }, // Target has a matching image already { options: { imageInfo: { Id: 'sha256:bbbb' } } }, ), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], images: [ // An image for a service that no longer exists createImage({ name: 'old-image', appId: 5, serviceName: 'old-service', dockerImageId: 'sha256:aaaa', }), createImage({ name: 'main-image', appId: 1, serviceName: 'main', dockerImageId: 'sha256:bbbb', }), ], }); const [removeImageStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step shoud happen for the depended service first expect(removeImageStep) .to.have.property('action') .that.equals('removeImage'); expect(removeImageStep) .to.have.property('image') .that.deep.includes({ name: 'old-image' }); }); it('should infer that an image should be removed if it is no longer referenced in current or target state (only target) unless keepImages is true', async () => { const targetApps = createApps( { services: [ await createService( { image: 'main-image' }, // Target has a matching image already { options: { imageInfo: { Id: 'sha256:bbbb' } } }, ), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], images: [ // An image for a service that no longer exists createImage({ name: 'old-image', appId: 5, serviceName: 'old-service', dockerImageId: 'sha256:aaaa', }), createImage({ name: 'main-image', appId: 1, serviceName: 'main', dockerImageId: 'sha256:bbbb', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { keepImages: true, downloading, availableImages, containerIdsByAppId, }, ); expect(steps.filter((s) => s.action === 'removeImage')).to.be.empty; }); it('should infer that an image should be removed if it is no longer referenced in current or target state (only current)', async () => { const targetApps = createApps( { services: [], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'main-image' }, // Target has a matching image already { options: { imageInfo: { Id: 'sha256:bbbb' } } }, ), ], networks: [DEFAULT_NETWORK], images: [ // An image for a service that no longer exists createImage({ name: 'old-image', appId: 5, serviceName: 'old-service', dockerImageId: 'sha256:aaaa', }), createImage({ name: 'main-image', appId: 1, serviceName: 'main', dockerImageId: 'sha256:bbbb', }), ], }); const [removeImageStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step shoud happen for the depended service first expect(removeImageStep) .to.have.property('action') .that.equals('removeImage'); expect(removeImageStep) .to.have.property('image') .that.deep.includes({ name: 'old-image' }); }); it('should infer that an image should be removed if it is no longer referenced in current or target state (only current) unless keepImages is true', async () => { const targetApps = createApps( { services: [], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'main-image' }, // Target has a matching image already { options: { imageInfo: { Id: 'sha256:bbbb' } } }, ), ], networks: [DEFAULT_NETWORK], images: [ // An image for a service that no longer exists createImage({ name: 'old-image', appId: 5, serviceName: 'old-service', dockerImageId: 'sha256:aaaa', }), createImage({ name: 'main-image', appId: 1, serviceName: 'main', dockerImageId: 'sha256:bbbb', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { keepImages: true, downloading, availableImages, containerIdsByAppId, }, ); expect(steps.filter((s) => s.action === 'removeImage')).to.be.empty; }); it('should infer that an image should be saved if it is not in the available image list but it can be found on disk', async () => { const targetApps = createApps( { services: [ await createService( { image: 'main-image' }, // Target has image info { options: { imageInfo: { Id: 'sha256:bbbb' } } }, ), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [DEFAULT_NETWORK], images: [], // no available images exist }); const [saveImageStep] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // A start step shoud happen for the depended service first expect(saveImageStep).to.have.property('action').that.equals('saveImage'); expect(saveImageStep) .to.have.property('image') .that.deep.includes({ name: 'main-image' }); }); it('should correctly generate steps for multiple apps', async () => { const targetApps = createApps( { services: [ await createService({ running: true, image: 'main-image-1', appId: 1, appUuid: 'app-one', commit: 'commit-for-app-1', }), await createService({ running: true, image: 'main-image-2', appId: 2, appUuid: 'app-two', commit: 'commit-for-app-2', }), ], networks: [ // Default networks for two apps Network.fromComposeObject('default', 1, 'app-one', {}), Network.fromComposeObject('default', 2, 'app-two', {}), ], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [], networks: [ // Default networks for two apps Network.fromComposeObject('default', 1, 'app-one', {}), Network.fromComposeObject('default', 2, 'app-two', {}), ], images: [ createImage({ name: 'main-image-1', appId: 1, appUuid: 'app-one', serviceName: 'main', commit: 'commit-for-app-1', }), createImage({ name: 'main-image-2', appId: 2, appUuid: 'app-two', serviceName: 'main', commit: 'commit-for-app-2', }), ], }); const steps = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); // Expect a start step for both apps expect( steps.filter( (s: any) => s.action === 'start' && s.target.appId === 1 && s.target.serviceName === 'main', ), ).to.have.lengthOf(1); expect( steps.filter( (s: any) => s.action === 'start' && s.target.appId === 2 && s.target.serviceName === 'main', ), ).to.have.lengthOf(1); }); describe("getting application's current state", () => { let getImagesState: sinon.SinonStub; let getServicesState: sinon.SinonStub; before(() => { getImagesState = sinon.stub(imageManager, 'getState'); getServicesState = sinon.stub(serviceManager, 'getState'); }); afterEach(() => { getImagesState.reset(); getServicesState.reset(); }); after(() => { getImagesState.restore(); getServicesState.restore(); }); it('reports the state of images if no service is available', async () => { getImagesState.resolves([ { name: 'ubuntu:latest', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'ubuntu', status: 'Downloaded', }, { name: 'alpine:latest', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Downloading', downloadProgress: 50, }, { name: 'fedora:latest', commit: 'newrelease', appUuid: 'fedora', serviceName: 'fedora', status: 'Downloading', downloadProgress: 75, }, { name: 'fedora:older', commit: 'oldrelease', appUuid: 'fedora', serviceName: 'fedora', status: 'Downloaded', }, ]); getServicesState.resolves([]); expect(await applicationManager.getState()).to.deep.equal({ myapp: { releases: { latestrelease: { services: { ubuntu: { image: 'ubuntu:latest', status: 'Downloaded', }, alpine: { image: 'alpine:latest', status: 'Downloading', download_progress: 50, }, }, }, }, }, fedora: { releases: { oldrelease: { services: { fedora: { image: 'fedora:older', status: 'Downloaded', }, }, }, newrelease: { services: { fedora: { image: 'fedora:latest', status: 'Downloading', download_progress: 75, }, }, }, }, }, }); }); it('augments the service data with image data', async () => { getImagesState.resolves([ { name: 'ubuntu:latest', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'ubuntu', status: 'Downloaded', }, { name: 'node:latest', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'node', status: 'Downloading', downloadProgress: 0, }, { name: 'alpine:latest', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Downloading', downloadProgress: 50, }, { name: 'fedora:older', commit: 'oldrelease', appUuid: 'fedora', serviceName: 'fedora', status: 'Downloaded', }, ]); getServicesState.resolves([ { commit: 'latestrelease', appUuid: 'myapp', serviceName: 'ubuntu', status: 'Running', createdAt: new Date('2021-09-01T13:00:00'), }, { commit: 'oldrelease', serviceName: 'fedora', status: 'Stopped', createdAt: new Date('2021-09-01T12:00:00'), }, { // Service without an image should not show on the final state appUuid: 'debian', commit: 'otherrelease', serviceName: 'debian', status: 'Stopped', createdAt: new Date('2021-09-01T12:00:00'), }, ]); expect(await applicationManager.getState()).to.deep.equal({ myapp: { releases: { latestrelease: { services: { ubuntu: { image: 'ubuntu:latest', status: 'Running', }, alpine: { image: 'alpine:latest', status: 'Downloading', download_progress: 50, }, node: { image: 'node:latest', status: 'Downloading', download_progress: 0, }, }, }, }, }, fedora: { releases: { oldrelease: { services: { fedora: { image: 'fedora:older', status: 'Stopped', }, }, }, }, }, }); }); it('reports handover state if multiple services are running for the same app', async () => { getImagesState.resolves([ { name: 'alpine:3.13', commit: 'latestrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Downloaded', }, { name: 'alpine:3.12', commit: 'oldrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Downloaded', }, ]); getServicesState.resolves([ { commit: 'latestrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Running', createdAt: new Date('2021-09-01T13:00:00'), }, { commit: 'oldrelease', appUuid: 'myapp', serviceName: 'alpine', status: 'Running', createdAt: new Date('2021-09-01T12:00:00'), }, ]); expect(await applicationManager.getState()).to.deep.equal({ myapp: { releases: { latestrelease: { services: { alpine: { image: 'alpine:3.13', status: 'Awaiting handover', }, }, }, oldrelease: { services: { alpine: { image: 'alpine:3.12', status: 'Handing over', }, }, }, }, }, }); }); }); // In the case where a container requires a host resource such as a network interface that is not created by the time the Engine // comes up, the Engine will not attempt to restart the container which seems to be Docker's implemented behavior (if not the correct behavior). // An example of a host resource would be a port binding such as 192.168.88.1:3000:3000, where the IP is an interface delayed in creation by host. // In this case, the Supervisor needs to wait a grace period for the Engine to start the container, and if this does not occur, the Supervisor // deduces the existence of this race condition and generates another start step after a delay (SECONDS_TO_WAIT_FOR_START). describe('handling Engine restart policy inaction when host resource required by container is delayed in creation', () => { // Time 61 seconds ago const date61SecondsAgo = new Date(); date61SecondsAgo.setSeconds(date61SecondsAgo.getSeconds() - 61); // Time 59 seconds ago const date50SecondsAgo = new Date(); date50SecondsAgo.setSeconds(date50SecondsAgo.getSeconds() - 50); // TODO: We need to be able to start a service with restart policy "no" if that service did not start at all due to // the host resource race condition described above. However, this is harder to parse as the containers do not include // the proper metadata for this. The last resort would be parsing the error message that caused the container to exit. it('should not infer any steps for a service with a status of "exited" if restart policy is "no" or "on-failure"', async () => { // Conditions: // - restart: "no" || "on-failure" // - status: "exited" const targetApps = createApps( { services: [ await createService({ image: 'image-1', serviceName: 'one', composition: { restart: 'no', }, }), await createService({ image: 'image-2', serviceName: 'two', composition: { restart: 'no', }, }), await createService({ image: 'image-3', serviceName: 'three', composition: { restart: 'on-failure', }, }), await createService({ image: 'image-4', serviceName: 'four', composition: { restart: 'on-failure', }, }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'image-1', serviceName: 'one', composition: { restart: 'no', }, }, { state: { status: 'exited', // Should not generate noop if exited within SECONDS_TO_WAIT_FOR_START createdAt: date50SecondsAgo, }, }, ), await createService( { image: 'image-2', serviceName: 'two', composition: { restart: 'no', }, }, { state: { status: 'exited', // Should not generate start if exited more than SECONDS_TO_WAIT_FOR_START ago createdAt: date61SecondsAgo, }, }, ), await createService( { image: 'image-3', serviceName: 'three', composition: { restart: 'on-failure', }, }, { state: { status: 'exited', // Should not generate noop if exited within SECONDS_TO_WAIT_FOR_START createdAt: date50SecondsAgo, }, }, ), await createService( { image: 'image-4', serviceName: 'four', composition: { restart: 'on-failure', }, }, { state: { status: 'exited', // Should not generate start if exited more than SECONDS_TO_WAIT_FOR_START ago createdAt: date61SecondsAgo, }, }, ), ], networks: [DEFAULT_NETWORK], images: [ createImage({ name: 'image-1', serviceName: 'one', }), createImage({ name: 'image-2', serviceName: 'two', }), createImage({ name: 'image-3', serviceName: 'three', }), createImage({ name: 'image-4', serviceName: 'four', }), ], }); const [...steps] = await applicationManager.inferNextSteps( currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }, ); expect(steps).to.have.lengthOf(0); }); it('should infer a noop step for a service that was created <= SECONDS_TO_WAIT_FOR_START ago with status of "exited" if restart policy is "always" or "unless-stopped"', async () => { // Conditions: // - restart: "always" || "unless-stopped" // - status: "exited" // - createdAt: <= SECONDS_TO_WAIT_FOR_START ago const targetApps = createApps( { services: [ await createService({ image: 'image-1', serviceName: 'one', composition: { restart: 'always', }, }), await createService({ image: 'image-2', serviceName: 'two', composition: { restart: 'unless-stopped', }, }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'image-1', serviceName: 'one', running: false, composition: { restart: 'always', }, }, { state: { status: 'exited', createdAt: date50SecondsAgo, }, }, ), await createService( { image: 'image-2', serviceName: 'two', running: false, composition: { restart: 'unless-stopped', }, }, { state: { status: 'exited', createdAt: date50SecondsAgo, }, }, ), ], networks: [DEFAULT_NETWORK], images: [ createImage({ name: 'image-1', serviceName: 'one', }), createImage({ name: 'image-2', serviceName: 'two', }), ], }); const [noopStep1, noopStep2, ...nextSteps] = await applicationManager.inferNextSteps(currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }); expect(noopStep1).to.have.property('action').that.equals('noop'); expect(noopStep2).to.have.property('action').that.equals('noop'); expect(nextSteps).to.have.lengthOf(0); }); it('should infer a start step for a service that was created > SECONDS_TO_WAIT_FOR_START ago with status of "exited" if restart policy is "always" or "unless-stopped"', async () => { // Conditions: // - restart: "always" || "unless-stopped" // - status: "exited" // - createdAt: > SECONDS_TO_WAIT_FOR_START ago const targetApps = createApps( { services: [ await createService({ image: 'image-1', serviceName: 'one', composition: { restart: 'always', }, }), await createService({ image: 'image-2', serviceName: 'two', composition: { restart: 'unless-stopped', }, }), ], networks: [DEFAULT_NETWORK], }, true, ); const { currentApps, availableImages, downloading, containerIdsByAppId } = createCurrentState({ services: [ await createService( { image: 'image-1', serviceName: 'one', running: false, composition: { restart: 'always', }, }, { state: { status: 'exited', createdAt: date61SecondsAgo, }, }, ), await createService( { image: 'image-2', serviceName: 'two', running: false, composition: { restart: 'unless-stopped', }, }, { state: { status: 'exited', createdAt: date61SecondsAgo, }, }, ), ], networks: [DEFAULT_NETWORK], images: [ createImage({ name: 'image-1', serviceName: 'one', }), createImage({ name: 'image-2', serviceName: 'two', }), ], }); const [startStep1, startStep2, ...nextSteps] = await applicationManager.inferNextSteps(currentApps, targetApps, { downloading, availableImages, containerIdsByAppId, }); [startStep1, startStep2].forEach((step) => { expect(step).to.have.property('action').that.equals('start'); expect(step) .to.have.property('target') .that.has.property('serviceName') .that.is.oneOf(['one', 'two']); }); expect(nextSteps).to.have.lengthOf(0); }); }); });