ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-14 02:58:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework use of storage clients (#2302)

Browse Source 
- Reuse Storage clients by caching them against account name.
  - This follows recommendations of Storage documentation: https://devblogs.microsoft.com/azure-sdk/lifetime-management-and-thread-safety-guarantees-of-azure-sdk-net-clients/
- Use strongly-typed `ResourceIdentifier` when passing around Account IDs, to prevent mixing them up with Account Names. 
- Simplify & centralize SAS generation in one place.
This commit is contained in:
George Pollard
2022-09-07 10:11:10 +12:00
committed by GitHub
parent 318e568cab
commit 84b2cc9992
14 changed files with 270 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/ApiService/ApiService/onefuzzlib/Containers.cs
View File

@ -1,8 +1,6 @@
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.ResourceManager;
using Azure.Storage;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Sas;
@ -34,19 +32,15 @@ public interface IContainers {
}
public class Containers : IContainers {
private ILogTracer _log;
private IStorage _storage;
private ICreds _creds;
private ArmClient _armClient;
private readonly ILogTracer _log;
private readonly IStorage _storage;
private readonly IServiceConfig _config;
static TimeSpan CONTAINER_SAS_DEFAULT_DURATION = TimeSpan.FromDays(30);
public Containers(ILogTracer log, IStorage storage, ICreds creds, IServiceConfig config) {
public Containers(ILogTracer log, IStorage storage, IServiceConfig config) {
_log = log;
_storage = storage;
_creds = creds;
_armClient = creds.ArmClient;
_config = config;
_getInstanceId = new Lazy<Async.Task<Guid>>(async () => {
@ -64,7 +58,7 @@ public class Containers : IContainers {
if (client is null)
return null;
return new Uri($"{_storage.GetBlobEndpoint(client.AccountName)}{container}/{name}");
return client.GetBlobClient(name).Uri;
}
public async Async.Task<BinaryData?> GetBlob(Container container, string name, StorageType storageType) {
@ -131,33 +125,22 @@ public class Containers : IContainers {
var containerName = _config.OneFuzzStoragePrefix + container.ContainerName;
var containers =
_storage.GetAccounts(storageType)
.Reverse()
.Select(async account => (await _storage.GetBlobServiceClientForAccount(account)).GetBlobContainerClient(containerName));
foreach (var c in containers) {
var client = await c;
if ((await client.ExistsAsync()).Value) {
return client;
foreach (var account in _storage.GetAccounts(storageType).Reverse()) {
var accountClient = await _storage.GetBlobServiceClientForAccount(account);
var containerClient = accountClient.GetBlobContainerClient(containerName);
if (await containerClient.ExistsAsync()) {
return containerClient;
}
}
return null;
}
public async Async.Task<Uri> GetFileSasUrl(Container container, string name, StorageType storageType, BlobSasPermissions permissions, TimeSpan? duration = null) {
var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}");
var (startTime, endTime) = SasTimeWindow(duration ?? TimeSpan.FromDays(30));
var sasBuilder = new BlobSasBuilder(permissions, endTime) {
StartsOn = startTime,
BlobContainerName = _config.OneFuzzStoragePrefix + container.ContainerName,
BlobName = name
};
var sasUrl = client.GetBlobClient(name).GenerateSasUri(sasBuilder);
return sasUrl;
var blobClient = client.GetBlobClient(name);
var timeWindow = SasTimeWindow(duration ?? TimeSpan.FromDays(30));
return _storage.GenerateBlobSasUri(permissions, blobClient, timeWindow);
}
public static (DateTimeOffset, DateTimeOffset) SasTimeWindow(TimeSpan timeSpan) {
@ -184,14 +167,12 @@ public class Containers : IContainers {
public virtual Async.Task<Guid> GetInstanceId() => _getInstanceId.Value;
private readonly Lazy<Async.Task<Guid>> _getInstanceId;
public static Uri? GetContainerSasUrlService(
public Uri GetContainerSasUrlService(
BlobContainerClient client,
BlobContainerSasPermissions permissions,
bool tag = false,
TimeSpan? timeSpan = null) {
var (start, expiry) = SasTimeWindow(timeSpan ?? TimeSpan.FromDays(30.0));
var sasBuilder = new BlobSasBuilder(permissions, expiry) { StartsOn = start };
return client.GenerateSasUri(sasBuilder);
var timeWindow = SasTimeWindow(timeSpan ?? TimeSpan.FromDays(30.0));
return _storage.GenerateBlobContainerSasUri(permissions, client, timeWindow);
}
public async Async.Task<Uri> AddContainerSasUrl(Uri uri, TimeSpan? duration = null) {
@ -199,32 +180,21 @@ public class Containers : IContainers {
return uri;
}
var (startTime, endTime) = SasTimeWindow(duration ?? CONTAINER_SAS_DEFAULT_DURATION);
var blobUriBuilder = new BlobUriBuilder(uri);
var accountKey = await _storage.GetStorageAccountNameKeyByName(blobUriBuilder.AccountName);
var sasBuilder = new BlobSasBuilder(
BlobContainerSasPermissions.Read | BlobContainerSasPermissions.Write | BlobContainerSasPermissions.Delete | BlobContainerSasPermissions.List,
endTime) {
BlobContainerName = blobUriBuilder.BlobContainerName,
StartsOn = startTime
};
var serviceClient = await _storage.GetBlobServiceClientForAccountName(blobUriBuilder.AccountName);
var containerClient = serviceClient.GetBlobContainerClient(blobUriBuilder.BlobContainerName);
var sas = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(blobUriBuilder.AccountName, accountKey)).ToString();
return new UriBuilder(uri) {
Query = sas
}.Uri;
var permissions = BlobContainerSasPermissions.Read | BlobContainerSasPermissions.Write | BlobContainerSasPermissions.Delete | BlobContainerSasPermissions.List;
var timeWindow = SasTimeWindow(duration ?? CONTAINER_SAS_DEFAULT_DURATION);
return _storage.GenerateBlobContainerSasUri(permissions, containerClient, timeWindow);
}
public async Async.Task<Uri> GetContainerSasUrl(Container container, StorageType storageType, BlobContainerSasPermissions permissions, TimeSpan? duration = null) {
public async Task<Uri> GetContainerSasUrl(Container container, StorageType storageType, BlobContainerSasPermissions permissions, TimeSpan? duration = null) {
var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}");
var (startTime, endTime) = SasTimeWindow(duration ?? CONTAINER_SAS_DEFAULT_DURATION);
var sasBuilder = new BlobSasBuilder(permissions, endTime) {
StartsOn = startTime,
BlobContainerName = _config.OneFuzzStoragePrefix + container.ContainerName,
};
var sasUrl = client.GenerateSasUri(sasBuilder);
return sasUrl;
var timeWindow = SasTimeWindow(duration ?? CONTAINER_SAS_DEFAULT_DURATION);
return _storage.GenerateBlobContainerSasUri(permissions, client, timeWindow);
}
public async Async.Task<bool> BlobExists(Container container, string name, StorageType storageType) {
@ -237,10 +207,6 @@ public class Containers : IContainers {
IEnumerable<IEnumerable<KeyValuePair<string, IDictionary<string, string>>>> data =
await Async.Task.WhenAll(accounts.Select(async acc => {
var service = await _storage.GetBlobServiceClientForAccount(acc);
if (service is null) {
throw new InvalidOperationException($"unable to get blob service for account {acc}");
}
return await service.GetBlobContainersAsync(BlobContainerTraits.Metadata).Select(container =>
KeyValuePair.Create(container.Name, container.Properties.Metadata)).ToListAsync();
}));