support multiple corpus accounts (#334)

Add support for sharding across multiple storage accounts for blob containers used for corpus management.

Things to note:

1. Additional storage accounts must be in the same resource group, support the "blob" endpoint, and have the tag `storage_type` with the value `corpus`.  A utility is provided (`src/utils/add-corpus-storage-accounts`), which adds storage accounts. 
2. If any secondary storage accounts exist, they are used by default for containers.
3. Storage account names are cached in memory the Azure Function instance forever.   Upon adding new storage accounts, the app needs to be restarted to pick up the new accounts.
This commit is contained in:
bmc-msft
2021-01-06 18:11:39 -05:00
committed by GitHub
parent f345bd239d
commit 3b26ffef65
29 changed files with 496 additions and 179 deletions

View File

@ -16,14 +16,15 @@ from onefuzztypes.models import (
ProxyConfig,
ProxyHeartbeat,
)
from onefuzztypes.primitives import Region
from onefuzztypes.primitives import Container, Region
from pydantic import Field
from .__version__ import __version__
from .azure.auth import build_auth
from .azure.containers import StorageType, get_file_sas_url, save_blob
from .azure.containers import get_file_sas_url, save_blob
from .azure.ip import get_public_ip
from .azure.queue import get_queue_sas
from .azure.storage import StorageType
from .azure.vm import VM
from .extension import proxy_manager_extensions
from .orm import MappingIntStrAny, ORMMixin, QueryFilter
@ -188,7 +189,7 @@ class Proxy(ORMMixin):
forwards = self.get_forwards()
proxy_config = ProxyConfig(
url=get_file_sas_url(
"proxy-configs",
Container("proxy-configs"),
"%s/config.json" % self.region,
StorageType.config,
read=True,
@ -203,7 +204,7 @@ class Proxy(ORMMixin):
)
save_blob(
"proxy-configs",
Container("proxy-configs"),
"%s/config.json" % self.region,
proxy_config.json(),
StorageType.config,