mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-15 03:18:07 +00:00
support multiple corpus accounts (#334)
Add support for sharding across multiple storage accounts for blob containers used for corpus management. Things to note: 1. Additional storage accounts must be in the same resource group, support the "blob" endpoint, and have the tag `storage_type` with the value `corpus`. A utility is provided (`src/utils/add-corpus-storage-accounts`), which adds storage accounts. 2. If any secondary storage accounts exist, they are used by default for containers. 3. Storage account names are cached in memory the Azure Function instance forever. Upon adding new storage accounts, the app needs to be restarted to pick up the new accounts.
This commit is contained in:
@ -25,6 +25,7 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
|
||||
)
|
||||
from memoization import cached
|
||||
from onefuzztypes.models import ADOTemplate, Report
|
||||
from onefuzztypes.primitives import Container
|
||||
|
||||
from .common import Render, fail_task
|
||||
|
||||
@ -49,7 +50,7 @@ def get_valid_fields(
|
||||
|
||||
class ADO:
|
||||
def __init__(
|
||||
self, container: str, filename: str, config: ADOTemplate, report: Report
|
||||
self, container: Container, filename: str, config: ADOTemplate, report: Report
|
||||
):
|
||||
self.config = config
|
||||
self.renderer = Render(container, filename, report)
|
||||
@ -200,7 +201,7 @@ class ADO:
|
||||
|
||||
|
||||
def notify_ado(
|
||||
config: ADOTemplate, container: str, filename: str, report: Report
|
||||
config: ADOTemplate, container: Container, filename: str, report: Report
|
||||
) -> None:
|
||||
logging.info(
|
||||
"notify ado: job_id:%s task_id:%s container:%s filename:%s",
|
||||
|
@ -9,6 +9,7 @@ from typing import Optional
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
from onefuzztypes.enums import ErrorCode
|
||||
from onefuzztypes.models import Error, Report
|
||||
from onefuzztypes.primitives import Container
|
||||
|
||||
from ..azure.containers import auth_download_url
|
||||
from ..azure.creds import get_instance_url
|
||||
@ -33,7 +34,7 @@ def fail_task(report: Report, error: Exception) -> None:
|
||||
|
||||
|
||||
class Render:
|
||||
def __init__(self, container: str, filename: str, report: Report):
|
||||
def __init__(self, container: Container, filename: str, report: Report):
|
||||
self.report = report
|
||||
self.container = container
|
||||
self.filename = filename
|
||||
|
@ -11,13 +11,18 @@ from github3.exceptions import GitHubException
|
||||
from github3.issues import Issue
|
||||
from onefuzztypes.enums import GithubIssueSearchMatch
|
||||
from onefuzztypes.models import GithubIssueTemplate, Report
|
||||
from onefuzztypes.primitives import Container
|
||||
|
||||
from .common import Render, fail_task
|
||||
|
||||
|
||||
class GithubIssue:
|
||||
def __init__(
|
||||
self, config: GithubIssueTemplate, container: str, filename: str, report: Report
|
||||
self,
|
||||
config: GithubIssueTemplate,
|
||||
container: Container,
|
||||
filename: str,
|
||||
report: Report,
|
||||
):
|
||||
self.config = config
|
||||
self.report = report
|
||||
@ -95,7 +100,10 @@ class GithubIssue:
|
||||
|
||||
|
||||
def github_issue(
|
||||
config: GithubIssueTemplate, container: str, filename: str, report: Optional[Report]
|
||||
config: GithubIssueTemplate,
|
||||
container: Container,
|
||||
filename: str,
|
||||
report: Optional[Report],
|
||||
) -> None:
|
||||
if report is None:
|
||||
return
|
||||
|
@ -21,12 +21,12 @@ from onefuzztypes.models import (
|
||||
from onefuzztypes.primitives import Container, Event
|
||||
|
||||
from ..azure.containers import (
|
||||
StorageType,
|
||||
container_exists,
|
||||
get_container_metadata,
|
||||
get_file_sas_url,
|
||||
)
|
||||
from ..azure.queue import send_message
|
||||
from ..azure.storage import StorageType
|
||||
from ..dashboard import add_event
|
||||
from ..orm import ORMMixin
|
||||
from ..reports import get_report
|
||||
|
@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from onefuzztypes.models import Report, TeamsTemplate
|
||||
from onefuzztypes.primitives import Container
|
||||
|
||||
from ..azure.containers import auth_download_url
|
||||
from ..tasks.config import get_setup_container
|
||||
@ -51,7 +52,7 @@ def send_teams_webhook(
|
||||
|
||||
|
||||
def notify_teams(
|
||||
config: TeamsTemplate, container: str, filename: str, report: Optional[Report]
|
||||
config: TeamsTemplate, container: Container, filename: str, report: Optional[Report]
|
||||
) -> None:
|
||||
text = None
|
||||
facts: List[Dict[str, str]] = []
|
||||
|
Reference in New Issue
Block a user