Support for retention policies on containers (#3501)

- [x] ability to specify a retention period on a container, which applies to newly-created blobs
- [x] specify default retention periods in templates from CLI side 

There's a small breaking change to the Python JobHelper class.

.github/workflows/ci.yml

@@ -123,7 +123,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: src/ci/check-check-pr.sh
@@ -137,7 +137,7 @@ jobs:
         shell: bash
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - uses: actions/download-artifact@v3
         with:
           name: artifact-onefuzztypes
@@ -190,7 +190,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: |
@@ -208,7 +208,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: |
@@ -224,7 +224,7 @@ jobs:
       - run: src/ci/set-versions.sh
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - run: src/ci/onefuzztypes.sh
       - uses: actions/upload-artifact@v3
         with:
@@ -481,7 +481,7 @@ jobs:
           path: artifacts
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - name: Lint
         shell: bash
         run: |

src/ApiService/ApiService/FeatureFlags.cs

@@ -8,4 +8,5 @@ public static class FeatureFlagConstants {
     public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy";
     public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
     public const string EnableWorkItemCreation = "EnableWorkItemCreation";
+    public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies";
 }

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -1,5 +1,6 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using System.Threading.Tasks;
 using Azure.Core;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
@@ -54,6 +55,8 @@ public class QueueFileChanges {
             return;
         }
 
+        var storageAccount = new ResourceIdentifier(topicElement.GetString()!);
+
         try {
             // Setting isLastRetryAttempt to false will rethrow any exceptions
             // With the intention that the azure functions runtime will handle requeing
@@ -61,7 +64,7 @@ public class QueueFileChanges {
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(storageAccount, fileChangeEvent, isLastRetryAttempt: false);
             if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
                 await RequeueMessage(msg, TimeSpan.FromDays(1));
             }
@@ -71,16 +74,47 @@ public class QueueFileChanges {
         }
     }
 
-    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storageAccount, JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
 
-        var container = parts[0];
+        var container = Container.Parse(parts[0]);
         var path = string.Join('/', parts.Skip(1));
 
-        _log.LogInformation("file added : {Container} - {Path}", container, path);
-        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+        _log.LogInformation("file added : {Container} - {Path}", container.String, path);
+
+        var (_, result) = await (
+            ApplyRetentionPolicy(storageAccount, container, path),
+            _notificationOperations.NewFiles(container, path, isLastRetryAttempt));
+
+        return result;
+    }
+
+    private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
+        if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) {
+            // default retention period can be applied to the container
+            // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
+            var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
+            var containerClient = account.GetBlobContainerClient(container.String);
+            var containerProps = await containerClient.GetPropertiesAsync();
+            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata);
+            if (!retentionPeriod.IsOk) {
+                _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV);
+            } else if (retentionPeriod.OkV is TimeSpan period) {
+                var blobClient = containerClient.GetBlobClient(path);
+                var tags = (await blobClient.GetTagsAsync()).Value.Tags;
+                var expiryDate = DateTime.UtcNow + period;
+                var tag = RetentionPolicyUtils.CreateExpiryDateTag(DateOnly.FromDateTime(expiryDate));
+                if (tags.TryAdd(tag.Key, tag.Value)) {
+                    _ = await blobClient.SetTagsAsync(tags);
+                    _log.LogInformation("applied container retention policy ({Policy}) to {Path}", period, path);
+                    return true;
+                }
+            }
+        }
+
+        return false;
     }
 
     private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -50,6 +50,7 @@ public enum ErrorCode {
     ADO_WORKITEM_PROCESSING_DISABLED = 494,
     ADO_VALIDATION_INVALID_PATH = 495,
     ADO_VALIDATION_INVALID_PROJECT = 496,
+    INVALID_RETENTION_PERIOD = 497,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -22,13 +22,12 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
 
     }
     public async Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt) {
-        var result = OneFuzzResultVoid.Ok;
-
         // We don't want to store file added events for the events container because that causes an infinite loop
         if (container == WellKnownContainers.Events) {
-            return result;
+            return Result.Ok();
         }
 
+        var result = OneFuzzResultVoid.Ok;
         var notifications = GetNotifications(container);
         var hasNotifications = await notifications.AnyAsync();
         var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);

src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs

@@ -1,24 +0,0 @@
-namespace Microsoft.OneFuzz.Service;
-
-
-public interface IRetentionPolicy {
-    DateOnly GetExpiryDate();
-}
-
-public class RetentionPolicyUtils {
-    public const string EXPIRY_TAG = "Expiry";
-    public static KeyValuePair<string, string> CreateExpiryDateTag(DateOnly expiryDate) =>
-        new(EXPIRY_TAG, expiryDate.ToString());
-
-    public static DateOnly? GetExpiryDateTagFromTags(IDictionary<string, string>? blobTags) {
-        if (blobTags != null &&
-            blobTags.TryGetValue(EXPIRY_TAG, out var expiryTag) &&
-            !string.IsNullOrWhiteSpace(expiryTag) &&
-            DateOnly.TryParse(expiryTag, out var expiryDate)) {
-            return expiryDate;
-        }
-        return null;
-    }
-
-    public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'";
-}

src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs

@@ -0,0 +1,43 @@
+using System.Xml;
+
+namespace Microsoft.OneFuzz.Service;
+
+
+public interface IRetentionPolicy {
+    DateOnly GetExpiryDate();
+}
+
+public class RetentionPolicyUtils {
+    public const string EXPIRY_TAG = "Expiry";
+    public static KeyValuePair<string, string> CreateExpiryDateTag(DateOnly expiryDate) =>
+        new(EXPIRY_TAG, expiryDate.ToString());
+
+    public static DateOnly? GetExpiryDateTagFromTags(IDictionary<string, string>? blobTags) {
+        if (blobTags != null &&
+            blobTags.TryGetValue(EXPIRY_TAG, out var expiryTag) &&
+            !string.IsNullOrWhiteSpace(expiryTag) &&
+            DateOnly.TryParse(expiryTag, out var expiryDate)) {
+            return expiryDate;
+        }
+        return null;
+    }
+
+    public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'";
+
+    // NB: this must match the value used on the CLI side
+    public const string CONTAINER_RETENTION_KEY = "onefuzz_retentionperiod";
+
+    public static OneFuzzResult<TimeSpan?> GetContainerRetentionPeriodFromMetadata(IDictionary<string, string>? containerMetadata) {
+        if (containerMetadata is not null &&
+            containerMetadata.TryGetValue(CONTAINER_RETENTION_KEY, out var retentionString) &&
+            !string.IsNullOrWhiteSpace(retentionString)) {
+            try {
+                return Result.Ok<TimeSpan?>(XmlConvert.ToTimeSpan(retentionString));
+            } catch (Exception ex) {
+                return Error.Create(ErrorCode.INVALID_RETENTION_PERIOD, ex.Message);
+            }
+        }
+
+        return Result.Ok<TimeSpan?>(null);
+    }
+}

src/cli/examples/domato.py

@@ -67,7 +67,7 @@ def upload_to_fuzzer_container(of: Onefuzz, fuzzer_name: str, fuzzer_url: str) -
 
 
 def upload_to_setup_container(of: Onefuzz, helper: JobHelper, setup_dir: str) -> None:
-    setup_sas = of.containers.get(helper.containers[ContainerType.setup]).sas_url
+    setup_sas = of.containers.get(helper.container_name(ContainerType.setup)).sas_url
     if AZCOPY_PATH is None:
         raise Exception("missing azcopy")
     command = [AZCOPY_PATH, "sync", setup_dir, setup_sas]
@@ -143,13 +143,16 @@ def main() -> None:
     helper.create_containers()
     helper.setup_notifications(notification_config)
     upload_to_setup_container(of, helper, args.setup_dir)
-    add_setup_script(of, helper.containers[ContainerType.setup])
+    add_setup_script(of, helper.container_name(ContainerType.setup))
 
     containers = [
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-        (ContainerType.reports, helper.containers[ContainerType.reports]),
-        (ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+        (ContainerType.reports, helper.container_name(ContainerType.reports)),
+        (
+            ContainerType.unique_reports,
+            helper.container_name(ContainerType.unique_reports),
+        ),
     ]
 
     of.logger.info("Creating generic_crash_report task")
@@ -164,11 +167,11 @@ def main() -> None:
 
     containers = [
         (ContainerType.tools, Container(FUZZER_NAME)),
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.crashes, helper.containers[ContainerType.crashes]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
         (
             ContainerType.readonly_inputs,
-            helper.containers[ContainerType.readonly_inputs],
+            helper.container_name(ContainerType.readonly_inputs),
         ),
     ]
 

src/cli/examples/honggfuzz.py

@@ -88,13 +88,16 @@ def main() -> None:
     if args.inputs:
         helper.upload_inputs(args.inputs)
 
-    add_setup_script(of, helper.containers[ContainerType.setup])
+    add_setup_script(of, helper.container_name(ContainerType.setup))
 
     containers = [
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-        (ContainerType.reports, helper.containers[ContainerType.reports]),
-        (ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+        (ContainerType.reports, helper.container_name(ContainerType.reports)),
+        (
+            ContainerType.unique_reports,
+            helper.container_name(ContainerType.unique_reports),
+        ),
     ]
 
     of.logger.info("Creating generic_crash_report task")
@@ -109,11 +112,11 @@ def main() -> None:
 
     containers = [
         (ContainerType.tools, Container("honggfuzz")),
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.crashes, helper.containers[ContainerType.crashes]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
         (
             ContainerType.inputs,
-            helper.containers[ContainerType.inputs],
+            helper.container_name(ContainerType.inputs),
         ),
     ]
 

src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py

@@ -74,15 +74,15 @@ def main() -> None:
     helper.create_containers()
 
     of.containers.files.upload_file(
-        helper.containers[ContainerType.tools], f"{args.tools}/source-coverage.sh"
+        helper.container_name(ContainerType.tools), f"{args.tools}/source-coverage.sh"
     )
 
     containers = [
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.analysis, helper.containers[ContainerType.analysis]),
-        (ContainerType.tools, helper.containers[ContainerType.tools]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.analysis, helper.container_name(ContainerType.analysis)),
+        (ContainerType.tools, helper.container_name(ContainerType.tools)),
         # note, analysis is typically for crashes, but this is analyzing inputs
-        (ContainerType.crashes, helper.containers[ContainerType.inputs]),
+        (ContainerType.crashes, helper.container_name(ContainerType.inputs)),
     ]
 
     of.logger.info("Creating generic_analysis task")

src/cli/examples/llvm-source-coverage/source-coverage.py

@@ -61,15 +61,15 @@ def main() -> None:
         helper.upload_inputs(args.inputs)
 
     of.containers.files.upload_file(
-        helper.containers[ContainerType.tools], f"{args.tools}/source-coverage.sh"
+        helper.container_name(ContainerType.tools), f"{args.tools}/source-coverage.sh"
     )
 
     containers = [
-        (ContainerType.setup, helper.containers[ContainerType.setup]),
-        (ContainerType.analysis, helper.containers[ContainerType.analysis]),
-        (ContainerType.tools, helper.containers[ContainerType.tools]),
+        (ContainerType.setup, helper.container_name(ContainerType.setup)),
+        (ContainerType.analysis, helper.container_name(ContainerType.analysis)),
+        (ContainerType.tools, helper.container_name(ContainerType.tools)),
         # note, analysis is typically for crashes, but this is analyzing inputs
-        (ContainerType.crashes, helper.containers[ContainerType.inputs]),
+        (ContainerType.crashes, helper.container_name(ContainerType.inputs)),
     ]
 
     of.logger.info("Creating generic_analysis task")

src/cli/onefuzz/templates/__init__.py

@@ -6,6 +6,7 @@
 import os
 import tempfile
 import zipfile
+from datetime import timedelta
 from typing import Any, Dict, List, Optional, Tuple
 from uuid import uuid4
 
@@ -22,6 +23,30 @@ class StoppedEarly(Exception):
     pass
 
 
+class ContainerTemplate:
+    def __init__(
+        self,
+        name: Container,
+        exists: bool,
+        *,
+        retention_period: Optional[timedelta] = None,
+    ):
+        self.name = name
+        self.retention_period = retention_period
+        # TODO: exists is not yet used/checked
+        self.exists = exists
+
+    @staticmethod
+    def existing(name: Container) -> "ContainerTemplate":
+        return ContainerTemplate(name, True)
+
+    @staticmethod
+    def fresh(
+        name: Container, *, retention_period: Optional[timedelta] = None
+    ) -> "ContainerTemplate":
+        return ContainerTemplate(name, False, retention_period=retention_period)
+
+
 class JobHelper:
     def __init__(
         self,
@@ -59,7 +84,7 @@ class JobHelper:
 
         self.wait_for_running: bool = False
         self.wait_for_stopped: bool = False
-        self.containers: Dict[ContainerType, Container] = {}
+        self.containers: Dict[ContainerType, ContainerTemplate] = {}
         self.tags: Dict[str, str] = {"project": project, "name": name, "build": build}
         if job is None:
             self.onefuzz.versions.check()
@@ -71,6 +96,20 @@ class JobHelper:
         else:
             self.job = job
 
+    def add_existing_container(
+        self, container_type: ContainerType, container: Container
+    ) -> None:
+        self.containers[container_type] = ContainerTemplate.existing(container)
+
+    def container_name(self, container_type: ContainerType) -> Container:
+        return self.containers[container_type].name
+
+    def container_names(self) -> Dict[ContainerType, Container]:
+        return {
+            container_type: container.name
+            for (container_type, container) in self.containers.items()
+        }
+
     def define_containers(self, *types: ContainerType) -> None:
         """
         Define default container set based on provided types
@@ -79,13 +118,23 @@ class JobHelper:
         """
 
         for container_type in types:
-            self.containers[container_type] = self.onefuzz.utils.build_container_name(
+            container_name = self.onefuzz.utils.build_container_name(
                 container_type=container_type,
                 project=self.project,
                 name=self.name,
                 build=self.build,
                 platform=self.platform,
             )
+            self.containers[container_type] = ContainerTemplate.fresh(
+                container_name,
+                retention_period=JobHelper._default_retention_period(container_type),
+            )
+
+    @staticmethod
+    def _default_retention_period(container_type: ContainerType) -> Optional[timedelta]:
+        if container_type == ContainerType.crashdumps:
+            return timedelta(days=90)
+        return None
 
     def get_unique_container_name(self, container_type: ContainerType) -> Container:
         return Container(
@@ -97,11 +146,17 @@ class JobHelper:
         )
 
     def create_containers(self) -> None:
-        for container_type, container_name in self.containers.items():
-            self.logger.info("using container: %s", container_name)
-            self.onefuzz.containers.create(
-                container_name, metadata={"container_type": container_type.name}
-            )
+        for container_type, container in self.containers.items():
+            self.logger.info("using container: %s", container.name)
+            metadata = {"container_type": container_type.name}
+            if container.retention_period is not None:
+                # format as ISO8601 period
+                # NB: this must match the value used on the server side
+                metadata[
+                    "onefuzz_retentionperiod"
+                ] = f"P{container.retention_period.days}D"
+
+            self.onefuzz.containers.create(container.name, metadata=metadata)
 
     def delete_container(self, container_name: Container) -> None:
         self.onefuzz.containers.delete(container_name)
@@ -112,12 +167,12 @@ class JobHelper:
 
         containers: List[Container] = []
         if ContainerType.unique_reports in self.containers:
-            containers.append(self.containers[ContainerType.unique_reports])
+            containers.append(self.container_name(ContainerType.unique_reports))
         else:
-            containers.append(self.containers[ContainerType.reports])
+            containers.append(self.container_name(ContainerType.reports))
 
         if ContainerType.regression_reports in self.containers:
-            containers.append(self.containers[ContainerType.regression_reports])
+            containers.append(self.container_name(ContainerType.regression_reports))
 
         for container in containers:
             self.logger.info("creating notification config for %s", container)
@@ -141,25 +196,25 @@ class JobHelper:
 
             self.logger.info("uploading setup dir `%s`" % setup_dir)
             self.onefuzz.containers.files.upload_dir(
-                self.containers[ContainerType.setup], setup_dir
+                self.container_name(ContainerType.setup), setup_dir
             )
         else:
             self.logger.info("uploading target exe `%s`" % target_exe)
             self.onefuzz.containers.files.upload_file(
-                self.containers[ContainerType.setup], target_exe
+                self.container_name(ContainerType.setup), target_exe
             )
 
             pdb_path = os.path.splitext(target_exe)[0] + ".pdb"
             if os.path.exists(pdb_path):
                 pdb_name = os.path.basename(pdb_path)
                 self.onefuzz.containers.files.upload_file(
-                    self.containers[ContainerType.setup], pdb_path, pdb_name
+                    self.container_name(ContainerType.setup), pdb_path, pdb_name
                 )
         if setup_files:
             for filename in setup_files:
                 self.logger.info("uploading %s", filename)
                 self.onefuzz.containers.files.upload_file(
-                    self.containers[ContainerType.setup], filename
+                    self.container_name(ContainerType.setup), filename
                 )
 
     def upload_inputs(self, path: Directory, read_only: bool = False) -> None:
@@ -167,7 +222,9 @@ class JobHelper:
         container_type = ContainerType.inputs
         if read_only:
             container_type = ContainerType.readonly_inputs
-        self.onefuzz.containers.files.upload_dir(self.containers[container_type], path)
+        self.onefuzz.containers.files.upload_dir(
+            self.container_name(container_type), path
+        )
 
     def upload_inputs_zip(self, path: File) -> None:
         with tempfile.TemporaryDirectory() as tmp_dir:
@@ -176,7 +233,7 @@ class JobHelper:
 
             self.logger.info("uploading inputs from zip: `%s`" % path)
             self.onefuzz.containers.files.upload_dir(
-                self.containers[ContainerType.inputs], Directory(tmp_dir)
+                self.container_name(ContainerType.inputs), Directory(tmp_dir)
             )
 
     @classmethod
@@ -195,8 +252,8 @@ class JobHelper:
             wait_for_files = []
 
         self.to_monitor = {
-            self.containers[x]: len(
-                self.onefuzz.containers.files.list(self.containers[x]).files
+            self.container_name(x): len(
+                self.onefuzz.containers.files.list(self.container_name(x)).files
             )
             for x in wait_for_files
         }

src/cli/onefuzz/templates/afl.py

@@ -98,7 +98,7 @@ class AFL(Command):
 
         if existing_inputs:
             self.onefuzz.containers.get(existing_inputs)
-            helper.containers[ContainerType.inputs] = existing_inputs
+            helper.add_existing_container(ContainerType.inputs, existing_inputs)
         else:
             helper.define_containers(ContainerType.inputs)
 
@@ -112,7 +112,7 @@ class AFL(Command):
         if (
             len(
                 self.onefuzz.containers.files.list(
-                    helper.containers[ContainerType.inputs]
+                    helper.containers[ContainerType.inputs].name
                 ).files
             )
             == 0
@@ -131,16 +131,16 @@ class AFL(Command):
 
         containers = [
             (ContainerType.tools, afl_container),
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.inputs, helper.containers[ContainerType.inputs]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.inputs, helper.container_name(ContainerType.inputs)),
         ]
 
         if extra_setup_container is not None:
             containers.append(
                 (
                     ContainerType.extra_setup,
-                    helper.containers[ContainerType.extra_setup],
+                    extra_setup_container,
                 )
             )
 
@@ -169,12 +169,12 @@ class AFL(Command):
         )
 
         report_containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.reports, helper.containers[ContainerType.reports]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.reports, helper.container_name(ContainerType.reports)),
             (
                 ContainerType.unique_reports,
-                helper.containers[ContainerType.unique_reports],
+                helper.container_name(ContainerType.unique_reports),
             ),
         ]
 
@@ -182,7 +182,7 @@ class AFL(Command):
             report_containers.append(
                 (
                     ContainerType.extra_setup,
-                    helper.containers[ContainerType.extra_setup],
+                    helper.container_name(ContainerType.extra_setup),
                 )
             )
 

src/cli/onefuzz/templates/libfuzzer.py

@@ -85,7 +85,7 @@ class Libfuzzer(Command):
         task_env: Optional[Dict[str, str]] = None,
     ) -> None:
         target_options = target_options or []
-        regression_containers = [
+        regression_containers: List[Tuple[ContainerType, Container]] = [
             (ContainerType.setup, containers[ContainerType.setup]),
             (ContainerType.crashes, containers[ContainerType.crashes]),
             (ContainerType.unique_reports, containers[ContainerType.unique_reports]),
@@ -129,7 +129,7 @@ class Libfuzzer(Command):
             task_env=task_env,
         )
 
-        fuzzer_containers = [
+        fuzzer_containers: List[Tuple[ContainerType, Container]] = [
             (ContainerType.setup, containers[ContainerType.setup]),
             (ContainerType.crashes, containers[ContainerType.crashes]),
             (ContainerType.crashdumps, containers[ContainerType.crashdumps]),
@@ -184,7 +184,7 @@ class Libfuzzer(Command):
 
         prereq_tasks = [fuzzer_task.task_id, regression_task.task_id]
 
-        coverage_containers = [
+        coverage_containers: List[Tuple[ContainerType, Container]] = [
             (ContainerType.setup, containers[ContainerType.setup]),
             (ContainerType.coverage, containers[ContainerType.coverage]),
             (ContainerType.readonly_inputs, containers[ContainerType.inputs]),
@@ -245,7 +245,7 @@ class Libfuzzer(Command):
             task_env=task_env,
         )
 
-        report_containers = [
+        report_containers: List[Tuple[ContainerType, Container]] = [
             (ContainerType.setup, containers[ContainerType.setup]),
             (ContainerType.crashes, containers[ContainerType.crashes]),
             (ContainerType.reports, containers[ContainerType.reports]),
@@ -285,7 +285,7 @@ class Libfuzzer(Command):
         if analyzer_exe is not None:
             self.logger.info("creating custom analysis")
 
-            analysis_containers = [
+            analysis_containers: List[Tuple[ContainerType, Container]] = [
                 (ContainerType.setup, containers[ContainerType.setup]),
                 (ContainerType.analysis, containers[ContainerType.analysis]),
                 (ContainerType.crashes, containers[ContainerType.crashes]),
@@ -428,15 +428,17 @@ class Libfuzzer(Command):
         )
 
         if existing_inputs:
-            helper.containers[ContainerType.inputs] = existing_inputs
+            helper.add_existing_container(ContainerType.inputs, existing_inputs)
         else:
             helper.define_containers(ContainerType.inputs)
 
         if readonly_inputs:
-            helper.containers[ContainerType.readonly_inputs] = readonly_inputs
+            helper.add_existing_container(
+                ContainerType.readonly_inputs, readonly_inputs
+            )
 
         if crashes:
-            helper.containers[ContainerType.crashes] = crashes
+            helper.add_existing_container(ContainerType.crashes, crashes)
 
         if analyzer_exe is not None:
             helper.define_containers(ContainerType.analysis)
@@ -465,17 +467,19 @@ class Libfuzzer(Command):
         else:
             source_allowlist_blob_name = None
 
-        containers = helper.containers
-
         if extra_setup_container is not None:
-            containers[ContainerType.extra_setup] = extra_setup_container
+            helper.add_existing_container(
+                ContainerType.extra_setup, extra_setup_container
+            )
 
         if extra_output_container is not None:
-            containers[ContainerType.extra_output] = extra_output_container
+            helper.add_existing_container(
+                ContainerType.extra_output, extra_output_container
+            )
 
         self._create_tasks(
             job=helper.job,
-            containers=containers,
+            containers=helper.container_names(),
             pool_name=pool_name,
             target_exe=target_exe_blob_name,
             vm_count=vm_count,
@@ -600,19 +604,35 @@ class Libfuzzer(Command):
         target_exe_blob_name = helper.setup_relative_blob_name(target_exe, setup_dir)
 
         merge_containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (
-                ContainerType.unique_inputs,
-                output_container or helper.containers[ContainerType.unique_inputs],
-            ),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
         ]
 
+        if output_container:
+            merge_containers.append(
+                (
+                    ContainerType.unique_inputs,
+                    output_container,
+                )
+            )
+        else:
+            merge_containers.append(
+                (
+                    ContainerType.unique_inputs,
+                    helper.container_name(ContainerType.unique_inputs),
+                )
+            )
+
         if extra_setup_container is not None:
-            merge_containers.append((ContainerType.extra_setup, extra_setup_container))
+            merge_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         if inputs:
             merge_containers.append(
-                (ContainerType.inputs, helper.containers[ContainerType.inputs])
+                (ContainerType.inputs, helper.container_name(ContainerType.inputs))
             )
         if existing_inputs:
             for existing_container in existing_inputs:
@@ -735,18 +755,18 @@ class Libfuzzer(Command):
             ContainerType.no_repro,
         )
 
-        containers = helper.containers
-
         if existing_inputs:
-            helper.containers[ContainerType.inputs] = existing_inputs
+            helper.add_existing_container(ContainerType.inputs, existing_inputs)
         else:
             helper.define_containers(ContainerType.inputs)
 
         if readonly_inputs:
-            helper.containers[ContainerType.readonly_inputs] = readonly_inputs
+            helper.add_existing_container(
+                ContainerType.readonly_inputs, readonly_inputs
+            )
 
         if crashes:
-            helper.containers[ContainerType.crashes] = crashes
+            helper.add_existing_container(ContainerType.crashes, crashes)
 
         # Assumes that `libfuzzer-dotnet` and supporting tools were uploaded upon deployment.
         fuzzer_tools_container = Container(
@@ -754,15 +774,20 @@ class Libfuzzer(Command):
         )
 
         fuzzer_containers = [
-            (ContainerType.setup, containers[ContainerType.setup]),
-            (ContainerType.crashes, containers[ContainerType.crashes]),
-            (ContainerType.crashdumps, containers[ContainerType.crashdumps]),
-            (ContainerType.inputs, containers[ContainerType.inputs]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.crashdumps, helper.container_name(ContainerType.crashdumps)),
+            (ContainerType.inputs, helper.container_name(ContainerType.inputs)),
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
         if extra_setup_container is not None:
-            fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container))
+            fuzzer_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         helper.create_containers()
         helper.setup_notifications(notification_config)
@@ -814,15 +839,21 @@ class Libfuzzer(Command):
         libfuzzer_dotnet_loader_dll = LIBFUZZER_DOTNET_LOADER_PATH
 
         coverage_containers = [
-            (ContainerType.setup, containers[ContainerType.setup]),
-            (ContainerType.coverage, containers[ContainerType.coverage]),
-            (ContainerType.readonly_inputs, containers[ContainerType.inputs]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.coverage, helper.container_name(ContainerType.coverage)),
+            (
+                ContainerType.readonly_inputs,
+                helper.container_name(ContainerType.inputs),
+            ),
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
         if extra_setup_container is not None:
             coverage_containers.append(
-                (ContainerType.extra_setup, extra_setup_container)
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
             )
 
         self.logger.info("creating `dotnet_coverage` task")
@@ -846,16 +877,24 @@ class Libfuzzer(Command):
         )
 
         report_containers = [
-            (ContainerType.setup, containers[ContainerType.setup]),
-            (ContainerType.crashes, containers[ContainerType.crashes]),
-            (ContainerType.reports, containers[ContainerType.reports]),
-            (ContainerType.unique_reports, containers[ContainerType.unique_reports]),
-            (ContainerType.no_repro, containers[ContainerType.no_repro]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.reports, helper.container_name(ContainerType.reports)),
+            (
+                ContainerType.unique_reports,
+                helper.container_name(ContainerType.unique_reports),
+            ),
+            (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)),
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
         if extra_setup_container is not None:
-            report_containers.append((ContainerType.extra_setup, extra_setup_container))
+            report_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         self.logger.info("creating `dotnet_crash_report` task")
         self.onefuzz.tasks.create(
@@ -972,27 +1011,37 @@ class Libfuzzer(Command):
 
         if existing_inputs:
             self.onefuzz.containers.get(existing_inputs)  # ensure it exists
-            helper.containers[ContainerType.inputs] = existing_inputs
+            helper.add_existing_container(ContainerType.inputs, existing_inputs)
         else:
             helper.define_containers(ContainerType.inputs)
 
         if crashes:
             self.onefuzz.containers.get(crashes)
-            helper.containers[ContainerType.crashes] = crashes
+            helper.add_existing_container(ContainerType.crashes, crashes)
 
         fuzzer_containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.crashdumps, helper.containers[ContainerType.crashdumps]),
-            (ContainerType.inputs, helper.containers[ContainerType.inputs]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.crashdumps, helper.container_name(ContainerType.crashdumps)),
+            (ContainerType.inputs, helper.container_name(ContainerType.inputs)),
         ]
 
         if extra_setup_container is not None:
-            fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container))
+            fuzzer_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         if readonly_inputs is not None:
             self.onefuzz.containers.get(readonly_inputs)  # ensure it exists
-            fuzzer_containers.append((ContainerType.readonly_inputs, readonly_inputs))
+            fuzzer_containers.append(
+                (
+                    ContainerType.readonly_inputs,
+                    readonly_inputs,
+                )
+            )
 
         helper.create_containers()
 
@@ -1079,18 +1128,23 @@ class Libfuzzer(Command):
         )
 
         report_containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.reports, helper.containers[ContainerType.reports]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.reports, helper.container_name(ContainerType.reports)),
             (
                 ContainerType.unique_reports,
-                helper.containers[ContainerType.unique_reports],
+                helper.container_name(ContainerType.unique_reports),
             ),
-            (ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
+            (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)),
         ]
 
         if extra_setup_container is not None:
-            report_containers.append((ContainerType.extra_setup, extra_setup_container))
+            report_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         self.logger.info("creating libfuzzer_crash_report task")
         self.onefuzz.tasks.create(

src/cli/onefuzz/templates/ossfuzz.py

@@ -215,13 +215,15 @@ class OssFuzz(Command):
             )
 
             if extra_setup_container is not None:
-                helper.containers[ContainerType.extra_setup] = extra_setup_container
+                helper.add_existing_container(
+                    ContainerType.extra_setup, extra_setup_container
+                )
 
             helper.create_containers()
             helper.setup_notifications(notification_config)
 
             dst_sas = self.onefuzz.containers.get(
-                helper.containers[ContainerType.setup]
+                helper.containers[ContainerType.setup].name
             ).sas_url
             self._copy_exe(container_sas["build"], dst_sas, File(fuzzer))
             self._copy_all(container_sas["base"], dst_sas)
@@ -245,7 +247,7 @@ class OssFuzz(Command):
 
             self.onefuzz.template.libfuzzer._create_tasks(
                 job=base_helper.job,
-                containers=helper.containers,
+                containers=helper.container_names(),
                 pool_name=pool_name,
                 target_exe=fuzzer_blob_name,
                 vm_count=VM_COUNT,

src/cli/onefuzz/templates/radamsa.py

@@ -3,7 +3,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 
 from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag, TaskType
 from onefuzztypes.models import Job, NotificationConfig
@@ -94,7 +94,9 @@ class Radamsa(Command):
 
         if existing_inputs:
             self.onefuzz.containers.get(existing_inputs)
-            helper.containers[ContainerType.readonly_inputs] = existing_inputs
+            helper.add_existing_container(
+                ContainerType.readonly_inputs, existing_inputs
+            )
         else:
             helper.define_containers(ContainerType.readonly_inputs)
         helper.create_containers()
@@ -108,7 +110,7 @@ class Radamsa(Command):
         if (
             len(
                 self.onefuzz.containers.files.list(
-                    helper.containers[ContainerType.readonly_inputs]
+                    helper.containers[ContainerType.readonly_inputs].name
                 ).files
             )
             == 0
@@ -147,18 +149,23 @@ class Radamsa(Command):
 
         self.logger.info("creating radamsa task")
 
-        containers = [
+        containers: List[Tuple[ContainerType, Container]] = [
             (ContainerType.tools, tools),
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
             (
                 ContainerType.readonly_inputs,
-                helper.containers[ContainerType.readonly_inputs],
+                helper.container_name(ContainerType.readonly_inputs),
             ),
         ]
 
         if extra_setup_container is not None:
-            containers.append((ContainerType.extra_setup, extra_setup_container))
+            containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         fuzzer_task = self.onefuzz.tasks.create(
             helper.job.job_id,
@@ -183,18 +190,23 @@ class Radamsa(Command):
         )
 
         report_containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.reports, helper.containers[ContainerType.reports]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.reports, helper.container_name(ContainerType.reports)),
             (
                 ContainerType.unique_reports,
-                helper.containers[ContainerType.unique_reports],
+                helper.container_name(ContainerType.unique_reports),
             ),
-            (ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
+            (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)),
         ]
 
         if extra_setup_container is not None:
-            report_containers.append((ContainerType.extra_setup, extra_setup_container))
+            report_containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         self.logger.info("creating generic_crash_report task")
         self.onefuzz.tasks.create(
@@ -233,15 +245,18 @@ class Radamsa(Command):
             self.logger.info("creating custom analysis")
 
             analysis_containers = [
-                (ContainerType.setup, helper.containers[ContainerType.setup]),
+                (ContainerType.setup, helper.container_name(ContainerType.setup)),
                 (ContainerType.tools, tools),
-                (ContainerType.analysis, helper.containers[ContainerType.analysis]),
-                (ContainerType.crashes, helper.containers[ContainerType.crashes]),
+                (ContainerType.analysis, helper.container_name(ContainerType.analysis)),
+                (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
             ]
 
             if extra_setup_container is not None:
                 analysis_containers.append(
-                    (ContainerType.extra_setup, extra_setup_container)
+                    (
+                        ContainerType.extra_setup,
+                        extra_setup_container,
+                    )
                 )
 
             self.onefuzz.tasks.create(

src/cli/onefuzz/templates/regression.py

@@ -12,7 +12,7 @@ from onefuzztypes.primitives import Container, Directory, File, PoolName
 
 from onefuzz.api import Command
 
-from . import JobHelper
+from . import ContainerTemplate, JobHelper
 
 
 class Regression(Command):
@@ -207,31 +207,36 @@ class Regression(Command):
         )
 
         containers = [
-            (ContainerType.setup, helper.containers[ContainerType.setup]),
-            (ContainerType.crashes, helper.containers[ContainerType.crashes]),
-            (ContainerType.reports, helper.containers[ContainerType.reports]),
-            (ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
+            (ContainerType.setup, helper.container_name(ContainerType.setup)),
+            (ContainerType.crashes, helper.container_name(ContainerType.crashes)),
+            (ContainerType.reports, helper.container_name(ContainerType.reports)),
+            (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)),
             (
                 ContainerType.unique_reports,
-                helper.containers[ContainerType.unique_reports],
+                helper.container_name(ContainerType.unique_reports),
             ),
             (
                 ContainerType.regression_reports,
-                helper.containers[ContainerType.regression_reports],
+                helper.container_name(ContainerType.regression_reports),
             ),
         ]
 
         if extra_setup_container:
-            containers.append((ContainerType.extra_setup, extra_setup_container))
+            containers.append(
+                (
+                    ContainerType.extra_setup,
+                    extra_setup_container,
+                )
+            )
 
         if crashes:
-            helper.containers[
-                ContainerType.readonly_inputs
-            ] = helper.get_unique_container_name(ContainerType.readonly_inputs)
+            helper.containers[ContainerType.readonly_inputs] = ContainerTemplate.fresh(
+                helper.get_unique_container_name(ContainerType.readonly_inputs)
+            )
             containers.append(
                 (
                     ContainerType.readonly_inputs,
-                    helper.containers[ContainerType.readonly_inputs],
+                    helper.container_name(ContainerType.readonly_inputs),
                 )
             )
 
@@ -239,7 +244,7 @@ class Regression(Command):
         if crashes:
             for file in crashes:
                 self.onefuzz.containers.files.upload_file(
-                    helper.containers[ContainerType.readonly_inputs], file
+                    helper.container_name(ContainerType.readonly_inputs), file
                 )
 
         helper.setup_notifications(notification_config)
@@ -276,7 +281,7 @@ class Regression(Command):
             if task.error:
                 raise Exception("task failed: %s", task.error)
 
-            container = helper.containers[ContainerType.regression_reports]
+            container = helper.containers[ContainerType.regression_reports].name
             for filename in self.onefuzz.containers.files.list(container).files:
                 self.logger.info("checking file: %s", filename)
                 if self._check_regression(container, File(filename)):
@@ -287,4 +292,6 @@ class Regression(Command):
             delete_input_container
             and ContainerType.readonly_inputs in helper.containers
         ):
-            helper.delete_container(helper.containers[ContainerType.readonly_inputs])
+            helper.delete_container(
+                helper.containers[ContainerType.readonly_inputs].name
+            )

src/deployment/bicep-templates/feature-flags.bicep

@@ -89,4 +89,17 @@ resource enableWorkItemCreation 'Microsoft.AppConfiguration/configurationStores/
   }
 }
 
+resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = {
+  parent: featureFlags
+  name: '.appconfig.featureflag~2FEnableContainerRetentionPolicies'
+  properties: {
+    value: string({
+      id: 'EnableContainerRetentionPolicies'
+      description: 'Enable retention policies on containers'
+      enabled: true
+    })
+    contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8'
+  }
+}
+
 output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io'

src/pytypes/onefuzztypes/enums.py

@@ -304,6 +304,7 @@ class ErrorCode(Enum):
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492
     ADO_VALIDATION_INVALID_PATH = 495
     ADO_VALIDATION_INVALID_PROJECT = 496
+    INVALID_RETENTION_PERIOD = 497
     # NB: if you update this enum, also update Enums.cs