support multiple corpus accounts (#334)

Add support for sharding across multiple storage accounts for blob containers used for corpus management.

Things to note:

1. Additional storage accounts must be in the same resource group, support the "blob" endpoint, and have the tag `storage_type` with the value `corpus`.  A utility is provided (`src/utils/add-corpus-storage-accounts`), which adds storage accounts. 
2. If any secondary storage accounts exist, they are used by default for containers.
3. Storage account names are cached in memory the Azure Function instance forever.   Upon adding new storage accounts, the app needs to be restarted to pick up the new accounts.
This commit is contained in:
bmc-msft
2021-01-06 18:11:39 -05:00
committed by GitHub
parent f345bd239d
commit 3b26ffef65
29 changed files with 496 additions and 179 deletions

View File

@ -25,6 +25,7 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
)
from memoization import cached
from onefuzztypes.models import ADOTemplate, Report
from onefuzztypes.primitives import Container
from .common import Render, fail_task
@ -49,7 +50,7 @@ def get_valid_fields(
class ADO:
def __init__(
self, container: str, filename: str, config: ADOTemplate, report: Report
self, container: Container, filename: str, config: ADOTemplate, report: Report
):
self.config = config
self.renderer = Render(container, filename, report)
@ -200,7 +201,7 @@ class ADO:
def notify_ado(
config: ADOTemplate, container: str, filename: str, report: Report
config: ADOTemplate, container: Container, filename: str, report: Report
) -> None:
logging.info(
"notify ado: job_id:%s task_id:%s container:%s filename:%s",

View File

@ -9,6 +9,7 @@ from typing import Optional
from jinja2.sandbox import SandboxedEnvironment
from onefuzztypes.enums import ErrorCode
from onefuzztypes.models import Error, Report
from onefuzztypes.primitives import Container
from ..azure.containers import auth_download_url
from ..azure.creds import get_instance_url
@ -33,7 +34,7 @@ def fail_task(report: Report, error: Exception) -> None:
class Render:
def __init__(self, container: str, filename: str, report: Report):
def __init__(self, container: Container, filename: str, report: Report):
self.report = report
self.container = container
self.filename = filename

View File

@ -11,13 +11,18 @@ from github3.exceptions import GitHubException
from github3.issues import Issue
from onefuzztypes.enums import GithubIssueSearchMatch
from onefuzztypes.models import GithubIssueTemplate, Report
from onefuzztypes.primitives import Container
from .common import Render, fail_task
class GithubIssue:
def __init__(
self, config: GithubIssueTemplate, container: str, filename: str, report: Report
self,
config: GithubIssueTemplate,
container: Container,
filename: str,
report: Report,
):
self.config = config
self.report = report
@ -95,7 +100,10 @@ class GithubIssue:
def github_issue(
config: GithubIssueTemplate, container: str, filename: str, report: Optional[Report]
config: GithubIssueTemplate,
container: Container,
filename: str,
report: Optional[Report],
) -> None:
if report is None:
return

View File

@ -21,12 +21,12 @@ from onefuzztypes.models import (
from onefuzztypes.primitives import Container, Event
from ..azure.containers import (
StorageType,
container_exists,
get_container_metadata,
get_file_sas_url,
)
from ..azure.queue import send_message
from ..azure.storage import StorageType
from ..dashboard import add_event
from ..orm import ORMMixin
from ..reports import get_report

View File

@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
import requests
from onefuzztypes.models import Report, TeamsTemplate
from onefuzztypes.primitives import Container
from ..azure.containers import auth_download_url
from ..tasks.config import get_setup_container
@ -51,7 +52,7 @@ def send_teams_webhook(
def notify_teams(
config: TeamsTemplate, container: str, filename: str, report: Optional[Report]
config: TeamsTemplate, container: Container, filename: str, report: Optional[Report]
) -> None:
text = None
facts: List[Dict[str, str]] = []