support multiple corpus accounts (#334)

Add support for sharding across multiple storage accounts for blob containers used for corpus management.

Things to note:

1. Additional storage accounts must be in the same resource group, support the "blob" endpoint, and have the tag `storage_type` with the value `corpus`.  A utility is provided (`src/utils/add-corpus-storage-accounts`), which adds storage accounts. 
2. If any secondary storage accounts exist, they are used by default for containers.
3. Storage account names are cached in memory the Azure Function instance forever.   Upon adding new storage accounts, the app needs to be restarted to pick up the new accounts.
This commit is contained in:
bmc-msft
2021-01-06 18:11:39 -05:00
committed by GitHub
parent f345bd239d
commit 3b26ffef65
29 changed files with 496 additions and 179 deletions

View File

@ -10,15 +10,12 @@ from uuid import UUID
from onefuzztypes.enums import Compare, ContainerPermission, ContainerType, TaskFeature
from onefuzztypes.models import TaskConfig, TaskDefinition, TaskUnitConfig
from onefuzztypes.primitives import Container
from ..azure.containers import (
StorageType,
blob_exists,
container_exists,
get_container_sas_url,
)
from ..azure.containers import blob_exists, container_exists, get_container_sas_url
from ..azure.creds import get_instance_id, get_instance_url
from ..azure.queue import get_queue_sas
from ..azure.storage import StorageType
from .defs import TASK_DEFINITIONS
LOGGER = logging.getLogger("onefuzz")
@ -334,7 +331,7 @@ def build_task_config(
return config
def get_setup_container(config: TaskConfig) -> str:
def get_setup_container(config: TaskConfig) -> Container:
for container in config.containers:
if container.type == ContainerType.setup:
return container.name

View File

@ -18,9 +18,9 @@ from onefuzztypes.webhooks import (
WebhookEventTaskStopped,
)
from ..azure.containers import StorageType
from ..azure.image import get_os
from ..azure.queue import create_queue, delete_queue
from ..azure.storage import StorageType
from ..orm import MappingIntStrAny, ORMMixin, QueryFilter
from ..pools import Node, Pool, Scaleset
from ..proxy_forward import ProxyForward

View File

@ -11,7 +11,8 @@ from onefuzztypes.enums import OS, PoolState, TaskState
from onefuzztypes.models import WorkSet, WorkUnit
from pydantic import BaseModel
from ..azure.containers import StorageType, blob_exists, get_container_sas_url
from ..azure.containers import blob_exists, get_container_sas_url
from ..azure.storage import StorageType
from ..pools import Pool
from .config import build_task_config, get_setup_container
from .main import Task