diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..050765b99 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.ps1 text=crlf diff --git a/contrib/webhook-teams-service/webhook/__init__.py b/contrib/webhook-teams-service/webhook/__init__.py index 0dced4afe..6e36a1bad 100644 --- a/contrib/webhook-teams-service/webhook/__init__.py +++ b/contrib/webhook-teams-service/webhook/__init__.py @@ -1,75 +1,75 @@ -#!/usr/bin/env python -# -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# - - -import hmac -import json -import logging -import os -from hashlib import sha512 -from typing import Any, Dict - -import aiohttp -import azure.functions as func - - -def code_block(data: str) -> str: - data = data.replace("`", "``") - return "\n```\n%s\n```\n" % data - - -async def send_message(req: func.HttpRequest) -> bool: - data = req.get_json() - teams_url = os.environ.get("TEAMS_URL") - if teams_url is None: - raise Exception("missing TEAMS_URL") - - message: Dict[str, Any] = { - "@type": "MessageCard", - "@context": "https://schema.org/extensions", - "summary": data["instance_name"], - "sections": [ - { - "facts": [ - {"name": "instance", "value": data["instance_name"]}, - {"name": "event type", "value": data["event_type"]}, - ] - }, - {"text": code_block(json.dumps(data["event"], sort_keys=True))}, - ], - } - async with aiohttp.ClientSession() as client: - async with client.post(teams_url, json=message) as response: - return response.ok - - -def verify(req: func.HttpRequest) -> bool: - request_hmac = req.headers.get("X-Onefuzz-Digest") - if request_hmac is None: - raise Exception("missing X-Onefuzz-Digest") - - hmac_token = os.environ.get("HMAC_TOKEN") - if hmac_token is None: - raise Exception("missing HMAC_TOKEN") - - digest = hmac.new( - hmac_token.encode(), msg=req.get_body(), digestmod=sha512 - ).hexdigest() - if digest != request_hmac: - logging.error("invalid hmac") - return False - - return True - - -async def main(req: func.HttpRequest) -> func.HttpResponse: - if not verify(req): - return func.HttpResponse("no thanks") - - if await send_message(req): - return func.HttpResponse("unable to send message") - - return func.HttpResponse("thanks") +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# + + +import hmac +import json +import logging +import os +from hashlib import sha512 +from typing import Any, Dict + +import aiohttp +import azure.functions as func + + +def code_block(data: str) -> str: + data = data.replace("`", "``") + return "\n```\n%s\n```\n" % data + + +async def send_message(req: func.HttpRequest) -> bool: + data = req.get_json() + teams_url = os.environ.get("TEAMS_URL") + if teams_url is None: + raise Exception("missing TEAMS_URL") + + message: Dict[str, Any] = { + "@type": "MessageCard", + "@context": "https://schema.org/extensions", + "summary": data["instance_name"], + "sections": [ + { + "facts": [ + {"name": "instance", "value": data["instance_name"]}, + {"name": "event type", "value": data["event_type"]}, + ] + }, + {"text": code_block(json.dumps(data["event"], sort_keys=True))}, + ], + } + async with aiohttp.ClientSession() as client: + async with client.post(teams_url, json=message) as response: + return response.ok + + +def verify(req: func.HttpRequest) -> bool: + request_hmac = req.headers.get("X-Onefuzz-Digest") + if request_hmac is None: + raise Exception("missing X-Onefuzz-Digest") + + hmac_token = os.environ.get("HMAC_TOKEN") + if hmac_token is None: + raise Exception("missing HMAC_TOKEN") + + digest = hmac.new( + hmac_token.encode(), msg=req.get_body(), digestmod=sha512 + ).hexdigest() + if digest != request_hmac: + logging.error("invalid hmac") + return False + + return True + + +async def main(req: func.HttpRequest) -> func.HttpResponse: + if not verify(req): + return func.HttpResponse("no thanks") + + if await send_message(req): + return func.HttpResponse("unable to send message") + + return func.HttpResponse("thanks") diff --git a/src/ApiService/ApiService/Functions/QueueProxyHeartbeat.cs b/src/ApiService/ApiService/Functions/QueueProxyHeartbeat.cs index 07a2a63c6..c8ba7500c 100644 --- a/src/ApiService/ApiService/Functions/QueueProxyHeartbeat.cs +++ b/src/ApiService/ApiService/Functions/QueueProxyHeartbeat.cs @@ -1,40 +1,40 @@ -using System.Text.Json; -using Microsoft.Azure.Functions.Worker; -using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; - -namespace Microsoft.OneFuzz.Service.Functions; - -public class QueueProxyHearbeat { - private readonly ILogTracer _log; - - private readonly IProxyOperations _proxy; - - public QueueProxyHearbeat(ILogTracer log, IProxyOperations proxy) { - _log = log; - _proxy = proxy; - } - - //[Function("QueueProxyHearbeat")] - public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) { - _log.Info($"heartbeat: {msg}"); - - var hb = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); ; - var newHb = hb with { TimeStamp = DateTimeOffset.UtcNow }; - - var proxy = await _proxy.GetByProxyId(newHb.ProxyId); - - var log = _log.WithTag("ProxyId", newHb.ProxyId.ToString()); - - if (proxy == null) { - log.Warning($"invalid proxy id: {newHb.ProxyId}"); - return; - } - var newProxy = proxy with { Heartbeat = newHb }; - - var r = await _proxy.Replace(newProxy); - if (!r.IsOk) { - var (status, reason) = r.ErrorV; - log.Error($"Failed to replace proxy heartbeat record due to [{status}] {reason}"); - } - } -} +using System.Text.Json; +using Microsoft.Azure.Functions.Worker; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service.Functions; + +public class QueueProxyHearbeat { + private readonly ILogTracer _log; + + private readonly IProxyOperations _proxy; + + public QueueProxyHearbeat(ILogTracer log, IProxyOperations proxy) { + _log = log; + _proxy = proxy; + } + + //[Function("QueueProxyHearbeat")] + public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) { + _log.Info($"heartbeat: {msg}"); + + var hb = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); ; + var newHb = hb with { TimeStamp = DateTimeOffset.UtcNow }; + + var proxy = await _proxy.GetByProxyId(newHb.ProxyId); + + var log = _log.WithTag("ProxyId", newHb.ProxyId.ToString()); + + if (proxy == null) { + log.Warning($"invalid proxy id: {newHb.ProxyId}"); + return; + } + var newProxy = proxy with { Heartbeat = newHb }; + + var r = await _proxy.Replace(newProxy); + if (!r.IsOk) { + var (status, reason) = r.ErrorV; + log.Error($"Failed to replace proxy heartbeat record due to [{status}] {reason}"); + } + } +} diff --git a/src/ApiService/ApiService/Functions/QueueWebhooks.cs b/src/ApiService/ApiService/Functions/QueueWebhooks.cs index 38c388f46..54dfaff9a 100644 --- a/src/ApiService/ApiService/Functions/QueueWebhooks.cs +++ b/src/ApiService/ApiService/Functions/QueueWebhooks.cs @@ -1,24 +1,24 @@ -using System.Text.Json; -using Microsoft.Azure.Functions.Worker; -using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; - -namespace Microsoft.OneFuzz.Service.Functions; - -public class QueueWebhooks { - private readonly ILogTracer _log; - private readonly IWebhookMessageLogOperations _webhookMessageLog; - public QueueWebhooks(ILogTracer log, IWebhookMessageLogOperations webhookMessageLog) { - _log = log; - _webhookMessageLog = webhookMessageLog; - } - - //[Function("QueueWebhooks")] - public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) { - - _log.Info($"Webhook Message Queued: {msg}"); - - var obj = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); - - await _webhookMessageLog.ProcessFromQueue(obj); - } -} +using System.Text.Json; +using Microsoft.Azure.Functions.Worker; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service.Functions; + +public class QueueWebhooks { + private readonly ILogTracer _log; + private readonly IWebhookMessageLogOperations _webhookMessageLog; + public QueueWebhooks(ILogTracer log, IWebhookMessageLogOperations webhookMessageLog) { + _log = log; + _webhookMessageLog = webhookMessageLog; + } + + //[Function("QueueWebhooks")] + public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) { + + _log.Info($"Webhook Message Queued: {msg}"); + + var obj = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); + + await _webhookMessageLog.ProcessFromQueue(obj); + } +} diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 301c21db7..93212f929 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -1,328 +1,328 @@ -using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; - -namespace Microsoft.OneFuzz.Service; - -[SerializeValue] -public enum ErrorCode { - INVALID_REQUEST = 450, - INVALID_PERMISSION = 451, - MISSING_EULA_AGREEMENT = 452, - INVALID_JOB = 453, - INVALID_TASK = INVALID_JOB, - UNABLE_TO_ADD_TASK_TO_JOB = 454, - INVALID_CONTAINER = 455, - UNABLE_TO_RESIZE = 456, - UNAUTHORIZED = 457, - UNABLE_TO_USE_STOPPED_JOB = 458, - UNABLE_TO_CHANGE_JOB_DURATION = 459, - UNABLE_TO_CREATE_NETWORK = 460, - VM_CREATE_FAILED = 461, - MISSING_NOTIFICATION = 462, - INVALID_IMAGE = 463, - UNABLE_TO_CREATE = 464, - UNABLE_TO_PORT_FORWARD = 465, - UNABLE_TO_FIND = 467, - TASK_FAILED = 468, - INVALID_NODE = 469, - NOTIFICATION_FAILURE = 470, - UNABLE_TO_UPDATE = 471, - PROXY_FAILED = 472, - INVALID_CONFIGURATION = 473, - UNABLE_TO_CREATE_CONTAINER = 474, -} - -public enum VmState { - Init, - ExtensionsLaunch, - ExtensionsFailed, - VmAllocationFailed, - Running, - Stopping, - Stopped -} - -public enum WebhookMessageState { - Queued, - Retrying, - Succeeded, - Failed -} - -public enum TaskState { - Init, - Waiting, - Scheduled, - SettingUp, - Running, - Stopping, - Stopped, - WaitJob -} - -public enum TaskType { - Coverage, - LibfuzzerFuzz, - LibfuzzerCoverage, - LibfuzzerCrashReport, - LibfuzzerMerge, - LibfuzzerRegression, - GenericAnalysis, - GenericSupervisor, - GenericMerge, - GenericGenerator, - GenericCrashReport, - GenericRegression, - DotnetCoverage, -} - -public enum Os { - Windows, - Linux -} - -public enum ContainerType { - Analysis, - Coverage, - Crashes, - Inputs, - NoRepro, - ReadonlyInputs, - Reports, - Setup, - Tools, - UniqueInputs, - UniqueReports, - RegressionReports, - Logs -} - - -[SkipRename] -public enum StatsFormat { - AFL -} - -public enum TaskDebugFlag { - KeepNodeOnFailure, - KeepNodeOnCompletion, -} - -public enum ScalesetState { - Init, - Setup, - Resize, - Running, - Shutdown, - Halt, - CreationFailed -} - - -public enum JobState { - Init, - Enabled, - Stopping, - Stopped -} - -public static class JobStateHelper { - private static readonly IReadOnlySet _shuttingDown = new HashSet(new[] { JobState.Stopping, JobState.Stopped }); - private static readonly IReadOnlySet _avaiable = new HashSet(new[] { JobState.Init, JobState.Enabled }); - private static readonly IReadOnlySet _needsWork = new HashSet(new[] { JobState.Init, JobState.Stopping }); - - public static IReadOnlySet Available => _avaiable; - public static IReadOnlySet NeedsWork => _needsWork; - public static IReadOnlySet ShuttingDown => _shuttingDown; -} - - - -public static class ScalesetStateHelper { - private static readonly IReadOnlySet _canUpdate = new HashSet { ScalesetState.Init, ScalesetState.Resize }; - private static readonly IReadOnlySet _needsWork = - new HashSet{ - ScalesetState.Init, - ScalesetState.Setup, - ScalesetState.Resize, - ScalesetState.Shutdown, - ScalesetState.Halt - }; - private static readonly IReadOnlySet _available = new HashSet { ScalesetState.Resize, ScalesetState.Running }; - private static readonly IReadOnlySet _resizing = new HashSet { ScalesetState.Halt, ScalesetState.Init, ScalesetState.Setup }; - - /// set of states that indicate the scaleset can be updated - public static IReadOnlySet CanUpdate => _canUpdate; - - /// set of states that indicate work is needed during eventing - public static IReadOnlySet NeedsWork => _needsWork; - - /// set of states that indicate if it's available for work - public static IReadOnlySet Available => _available; - - /// set of states that indicate scaleset is resizing - public static IReadOnlySet Resizing => _resizing; -} - - -public static class VmStateHelper { - - private static readonly IReadOnlySet _needsWork = new HashSet { VmState.Init, VmState.ExtensionsLaunch, VmState.Stopping }; - private static readonly IReadOnlySet _available = new HashSet { VmState.Init, VmState.ExtensionsLaunch, VmState.ExtensionsFailed, VmState.VmAllocationFailed, VmState.Running, }; - - public static IReadOnlySet NeedsWork => _needsWork; - public static IReadOnlySet Available => _available; -} - -public static class TaskStateHelper { - - public static readonly IReadOnlySet AvailableStates = - new HashSet { TaskState.Waiting, TaskState.Scheduled, TaskState.SettingUp, TaskState.Running, TaskState.WaitJob }; - - public static readonly IReadOnlySet NeedsWorkStates = - new HashSet { TaskState.Init, TaskState.Stopping }; - - public static readonly IReadOnlySet ShuttingDownStates = - new HashSet { TaskState.Stopping, TaskState.Stopped }; - - public static readonly IReadOnlySet HasStartedStates = - new HashSet { TaskState.Running, TaskState.Stopping, TaskState.Stopped }; - - public static bool Available(this TaskState state) => AvailableStates.Contains(state); - - public static bool NeedsWork(this TaskState state) => NeedsWorkStates.Contains(state); - - public static bool ShuttingDown(this TaskState state) => ShuttingDownStates.Contains(state); - - public static bool HasStarted(this TaskState state) => HasStartedStates.Contains(state); - -} -public enum PoolState { - Init, - Running, - Shutdown, - Halt -} - -public static class PoolStateHelper { - private static readonly IReadOnlySet _needsWork = new HashSet { PoolState.Init, PoolState.Shutdown, PoolState.Halt }; - private static readonly IReadOnlySet _available = new HashSet { PoolState.Running }; - - public static IReadOnlySet NeedsWork => _needsWork; - public static IReadOnlySet Available => _available; -} - -[SkipRename] -public enum Architecture { - x86_64 -} - -public enum TaskFeature { - InputQueueFromContainer, - SupervisorExe, - SupervisorEnv, - SupervisorOptions, - SupervisorInputMarker, - StatsFile, - StatsFormat, - TargetExe, - TargetExeOptional, - TargetEnv, - TargetOptions, - AnalyzerExe, - AnalyzerEnv, - AnalyzerOptions, - RenameOutput, - TargetOptionsMerge, - TargetWorkers, - GeneratorExe, - GeneratorEnv, - GeneratorOptions, - WaitForFiles, - TargetTimeout, - CheckAsanLog, - CheckDebugger, - CheckRetryCount, - EnsembleSyncDelay, - PreserveExistingOutputs, - CheckFuzzerHelp, - ExpectCrashOnFailure, - ReportList, - MinimizedStackDepth, - CoverageFilter, - TargetMustUseInput -} - - -[Flags] -public enum ContainerPermission { - Read = 1 << 0, - Write = 1 << 1, - List = 1 << 2, - Delete = 1 << 3, -} - - -public enum Compare { - Equal, - AtLeast, - AtMost -} -public enum AgentMode { - Fuzz, - Repro, - Proxy -} - -public enum NodeState { - Init, - Free, - SettingUp, - Rebooting, - Ready, - Busy, - Done, - Shutdown, - Halt, -} - -public static class NodeStateHelper { - private static readonly IReadOnlySet _needsWork = - new HashSet(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt }); - - private static readonly IReadOnlySet _readyForReset - = new HashSet(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt }); - - private static readonly IReadOnlySet _canProcessNewWork = - new HashSet(new[] { NodeState.Free }); - - private static readonly IReadOnlySet _busy = - new HashSet(new[] { NodeState.Busy }); - - public static IReadOnlySet BusyStates => _busy; - - public static IReadOnlySet NeedsWorkStates => _needsWork; - - public static bool NeedsWork(this NodeState state) => _needsWork.Contains(state); - - ///If Node is in one of these states, ignore updates from the agent. - public static bool ReadyForReset(this NodeState state) => _readyForReset.Contains(state); - - public static bool CanProcessNewWork(this NodeState state) => _canProcessNewWork.Contains(state); -} - - -public enum NodeDisposalStrategy { - ScaleIn, - Decomission -} - - -public enum GithubIssueState { - Open, - Closed -} - -public enum GithubIssueSearchMatch { - Title, - Body -} +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service; + +[SerializeValue] +public enum ErrorCode { + INVALID_REQUEST = 450, + INVALID_PERMISSION = 451, + MISSING_EULA_AGREEMENT = 452, + INVALID_JOB = 453, + INVALID_TASK = INVALID_JOB, + UNABLE_TO_ADD_TASK_TO_JOB = 454, + INVALID_CONTAINER = 455, + UNABLE_TO_RESIZE = 456, + UNAUTHORIZED = 457, + UNABLE_TO_USE_STOPPED_JOB = 458, + UNABLE_TO_CHANGE_JOB_DURATION = 459, + UNABLE_TO_CREATE_NETWORK = 460, + VM_CREATE_FAILED = 461, + MISSING_NOTIFICATION = 462, + INVALID_IMAGE = 463, + UNABLE_TO_CREATE = 464, + UNABLE_TO_PORT_FORWARD = 465, + UNABLE_TO_FIND = 467, + TASK_FAILED = 468, + INVALID_NODE = 469, + NOTIFICATION_FAILURE = 470, + UNABLE_TO_UPDATE = 471, + PROXY_FAILED = 472, + INVALID_CONFIGURATION = 473, + UNABLE_TO_CREATE_CONTAINER = 474, +} + +public enum VmState { + Init, + ExtensionsLaunch, + ExtensionsFailed, + VmAllocationFailed, + Running, + Stopping, + Stopped +} + +public enum WebhookMessageState { + Queued, + Retrying, + Succeeded, + Failed +} + +public enum TaskState { + Init, + Waiting, + Scheduled, + SettingUp, + Running, + Stopping, + Stopped, + WaitJob +} + +public enum TaskType { + Coverage, + LibfuzzerFuzz, + LibfuzzerCoverage, + LibfuzzerCrashReport, + LibfuzzerMerge, + LibfuzzerRegression, + GenericAnalysis, + GenericSupervisor, + GenericMerge, + GenericGenerator, + GenericCrashReport, + GenericRegression, + DotnetCoverage, +} + +public enum Os { + Windows, + Linux +} + +public enum ContainerType { + Analysis, + Coverage, + Crashes, + Inputs, + NoRepro, + ReadonlyInputs, + Reports, + Setup, + Tools, + UniqueInputs, + UniqueReports, + RegressionReports, + Logs +} + + +[SkipRename] +public enum StatsFormat { + AFL +} + +public enum TaskDebugFlag { + KeepNodeOnFailure, + KeepNodeOnCompletion, +} + +public enum ScalesetState { + Init, + Setup, + Resize, + Running, + Shutdown, + Halt, + CreationFailed +} + + +public enum JobState { + Init, + Enabled, + Stopping, + Stopped +} + +public static class JobStateHelper { + private static readonly IReadOnlySet _shuttingDown = new HashSet(new[] { JobState.Stopping, JobState.Stopped }); + private static readonly IReadOnlySet _avaiable = new HashSet(new[] { JobState.Init, JobState.Enabled }); + private static readonly IReadOnlySet _needsWork = new HashSet(new[] { JobState.Init, JobState.Stopping }); + + public static IReadOnlySet Available => _avaiable; + public static IReadOnlySet NeedsWork => _needsWork; + public static IReadOnlySet ShuttingDown => _shuttingDown; +} + + + +public static class ScalesetStateHelper { + private static readonly IReadOnlySet _canUpdate = new HashSet { ScalesetState.Init, ScalesetState.Resize }; + private static readonly IReadOnlySet _needsWork = + new HashSet{ + ScalesetState.Init, + ScalesetState.Setup, + ScalesetState.Resize, + ScalesetState.Shutdown, + ScalesetState.Halt + }; + private static readonly IReadOnlySet _available = new HashSet { ScalesetState.Resize, ScalesetState.Running }; + private static readonly IReadOnlySet _resizing = new HashSet { ScalesetState.Halt, ScalesetState.Init, ScalesetState.Setup }; + + /// set of states that indicate the scaleset can be updated + public static IReadOnlySet CanUpdate => _canUpdate; + + /// set of states that indicate work is needed during eventing + public static IReadOnlySet NeedsWork => _needsWork; + + /// set of states that indicate if it's available for work + public static IReadOnlySet Available => _available; + + /// set of states that indicate scaleset is resizing + public static IReadOnlySet Resizing => _resizing; +} + + +public static class VmStateHelper { + + private static readonly IReadOnlySet _needsWork = new HashSet { VmState.Init, VmState.ExtensionsLaunch, VmState.Stopping }; + private static readonly IReadOnlySet _available = new HashSet { VmState.Init, VmState.ExtensionsLaunch, VmState.ExtensionsFailed, VmState.VmAllocationFailed, VmState.Running, }; + + public static IReadOnlySet NeedsWork => _needsWork; + public static IReadOnlySet Available => _available; +} + +public static class TaskStateHelper { + + public static readonly IReadOnlySet AvailableStates = + new HashSet { TaskState.Waiting, TaskState.Scheduled, TaskState.SettingUp, TaskState.Running, TaskState.WaitJob }; + + public static readonly IReadOnlySet NeedsWorkStates = + new HashSet { TaskState.Init, TaskState.Stopping }; + + public static readonly IReadOnlySet ShuttingDownStates = + new HashSet { TaskState.Stopping, TaskState.Stopped }; + + public static readonly IReadOnlySet HasStartedStates = + new HashSet { TaskState.Running, TaskState.Stopping, TaskState.Stopped }; + + public static bool Available(this TaskState state) => AvailableStates.Contains(state); + + public static bool NeedsWork(this TaskState state) => NeedsWorkStates.Contains(state); + + public static bool ShuttingDown(this TaskState state) => ShuttingDownStates.Contains(state); + + public static bool HasStarted(this TaskState state) => HasStartedStates.Contains(state); + +} +public enum PoolState { + Init, + Running, + Shutdown, + Halt +} + +public static class PoolStateHelper { + private static readonly IReadOnlySet _needsWork = new HashSet { PoolState.Init, PoolState.Shutdown, PoolState.Halt }; + private static readonly IReadOnlySet _available = new HashSet { PoolState.Running }; + + public static IReadOnlySet NeedsWork => _needsWork; + public static IReadOnlySet Available => _available; +} + +[SkipRename] +public enum Architecture { + x86_64 +} + +public enum TaskFeature { + InputQueueFromContainer, + SupervisorExe, + SupervisorEnv, + SupervisorOptions, + SupervisorInputMarker, + StatsFile, + StatsFormat, + TargetExe, + TargetExeOptional, + TargetEnv, + TargetOptions, + AnalyzerExe, + AnalyzerEnv, + AnalyzerOptions, + RenameOutput, + TargetOptionsMerge, + TargetWorkers, + GeneratorExe, + GeneratorEnv, + GeneratorOptions, + WaitForFiles, + TargetTimeout, + CheckAsanLog, + CheckDebugger, + CheckRetryCount, + EnsembleSyncDelay, + PreserveExistingOutputs, + CheckFuzzerHelp, + ExpectCrashOnFailure, + ReportList, + MinimizedStackDepth, + CoverageFilter, + TargetMustUseInput +} + + +[Flags] +public enum ContainerPermission { + Read = 1 << 0, + Write = 1 << 1, + List = 1 << 2, + Delete = 1 << 3, +} + + +public enum Compare { + Equal, + AtLeast, + AtMost +} +public enum AgentMode { + Fuzz, + Repro, + Proxy +} + +public enum NodeState { + Init, + Free, + SettingUp, + Rebooting, + Ready, + Busy, + Done, + Shutdown, + Halt, +} + +public static class NodeStateHelper { + private static readonly IReadOnlySet _needsWork = + new HashSet(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt }); + + private static readonly IReadOnlySet _readyForReset + = new HashSet(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt }); + + private static readonly IReadOnlySet _canProcessNewWork = + new HashSet(new[] { NodeState.Free }); + + private static readonly IReadOnlySet _busy = + new HashSet(new[] { NodeState.Busy }); + + public static IReadOnlySet BusyStates => _busy; + + public static IReadOnlySet NeedsWorkStates => _needsWork; + + public static bool NeedsWork(this NodeState state) => _needsWork.Contains(state); + + ///If Node is in one of these states, ignore updates from the agent. + public static bool ReadyForReset(this NodeState state) => _readyForReset.Contains(state); + + public static bool CanProcessNewWork(this NodeState state) => _canProcessNewWork.Contains(state); +} + + +public enum NodeDisposalStrategy { + ScaleIn, + Decomission +} + + +public enum GithubIssueState { + Open, + Closed +} + +public enum GithubIssueSearchMatch { + Title, + Body +} diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs index b41633366..a5eba155f 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs @@ -1,135 +1,135 @@ -using System.Threading.Tasks; -using ApiService.OneFuzzLib.Orm; -using Azure.Storage.Sas; - -namespace Microsoft.OneFuzz.Service; - -public interface IProxyOperations : IStatefulOrm { - Task GetByProxyId(Guid proxyId); - - Async.Task SetState(Proxy proxy, VmState state); - bool IsAlive(Proxy proxy); - Async.Task SaveProxyConfig(Proxy proxy); - bool IsOutdated(Proxy proxy); - Async.Task GetOrCreate(string region); - -} -public class ProxyOperations : StatefulOrm, IProxyOperations { - - - static TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7); - - public ProxyOperations(ILogTracer log, IOnefuzzContext context) - : base(log.WithTag("Component", "scaleset-proxy"), context) { - - } - - - public async Task GetByProxyId(Guid proxyId) { - - var data = QueryAsync(filter: $"RowKey eq '{proxyId}'"); - - return await data.FirstOrDefaultAsync(); - } - - public async Async.Task GetOrCreate(string region) { - var proxyList = QueryAsync(filter: $"region eq '{region}' and outdated eq false"); - - await foreach (var proxy in proxyList) { - if (IsOutdated(proxy)) { - await Replace(proxy with { Outdated = true }); - continue; - } - - if (!VmStateHelper.Available.Contains(proxy.State)) { - continue; - } - return proxy; - } - - _logTracer.Info($"creating proxy: region:{region}"); - var newProxy = new Proxy(region, Guid.NewGuid(), DateTimeOffset.UtcNow, VmState.Init, Auth.BuildAuth(), null, null, _context.ServiceConfiguration.OneFuzzVersion, null, false); - - await Replace(newProxy); - await _context.Events.SendEvent(new EventProxyCreated(region, newProxy.ProxyId)); - return newProxy; - } - - public bool IsAlive(Proxy proxy) { - var tenMinutesAgo = DateTimeOffset.UtcNow - TimeSpan.FromMinutes(10); - - if (proxy.Heartbeat != null && proxy.Heartbeat.TimeStamp < tenMinutesAgo) { - _logTracer.Info($"last heartbeat is more than an 10 minutes old: {proxy.Region} - last heartbeat:{proxy.Heartbeat} compared_to:{tenMinutesAgo}"); - return false; - } - - if (proxy.Heartbeat != null && proxy.TimeStamp != null && proxy.TimeStamp < tenMinutesAgo) { - _logTracer.Error($"no heartbeat in the last 10 minutes: {proxy.Region} timestamp: {proxy.TimeStamp} compared_to:{tenMinutesAgo}"); - return false; - } - - return true; - } - - public bool IsOutdated(Proxy proxy) { - if (!VmStateHelper.Available.Contains(proxy.State)) { - return false; - } - - if (proxy.Version != _context.ServiceConfiguration.OneFuzzVersion) { - _logTracer.Info($"mismatch version: proxy:{proxy.Version} service:{_context.ServiceConfiguration.OneFuzzVersion} state:{proxy.State}"); - return true; - } - - if (proxy.CreatedTimestamp != null) { - if (proxy.CreatedTimestamp < (DateTimeOffset.UtcNow - PROXY_LIFESPAN)) { - _logTracer.Info($"proxy older than 7 days:proxy-created:{proxy.CreatedTimestamp} state:{proxy.State}"); - return true; - } - } - return false; - } - - public async Async.Task SaveProxyConfig(Proxy proxy) { - var forwards = await GetForwards(proxy); - var url = (await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas"); - var queueSas = await _context.Queue.GetQueueSas("proxy", StorageType.Config, QueueSasPermissions.Add).EnsureNotNull("can't generate queue sas") ?? throw new Exception("Queue sas is null"); - - var proxyConfig = new ProxyConfig( - Url: url, - Notification: queueSas, - Region: proxy.Region, - ProxyId: proxy.ProxyId, - Forwards: forwards, - InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey.EnsureNotNull("missing InstrumentationKey"), - MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry.EnsureNotNull("missing Telemetry"), - InstanceId: await _context.Containers.GetInstanceId()); - - await _context.Containers.SaveBlob(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", _entityConverter.ToJsonString(proxyConfig), StorageType.Config); - } - - - public async Async.Task SetState(Proxy proxy, VmState state) { - if (proxy.State == state) { - return; - } - - await Replace(proxy with { State = state }); - - await _context.Events.SendEvent(new EventProxyStateUpdated(proxy.Region, proxy.ProxyId, proxy.State)); - } - - - public async Async.Task> GetForwards(Proxy proxy) { - var forwards = new List(); - - await foreach (var entry in _context.ProxyForwardOperations.SearchForward(region: proxy.Region, proxyId: proxy.ProxyId)) { - if (entry.EndTime < DateTimeOffset.UtcNow) { - await _context.ProxyForwardOperations.Delete(entry); - } else { - forwards.Add(new Forward(entry.Port, entry.DstPort, entry.DstIp)); - } - } - return forwards; - } -} +using System.Threading.Tasks; +using ApiService.OneFuzzLib.Orm; +using Azure.Storage.Sas; + +namespace Microsoft.OneFuzz.Service; + +public interface IProxyOperations : IStatefulOrm { + Task GetByProxyId(Guid proxyId); + + Async.Task SetState(Proxy proxy, VmState state); + bool IsAlive(Proxy proxy); + Async.Task SaveProxyConfig(Proxy proxy); + bool IsOutdated(Proxy proxy); + Async.Task GetOrCreate(string region); + +} +public class ProxyOperations : StatefulOrm, IProxyOperations { + + + static TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7); + + public ProxyOperations(ILogTracer log, IOnefuzzContext context) + : base(log.WithTag("Component", "scaleset-proxy"), context) { + + } + + + public async Task GetByProxyId(Guid proxyId) { + + var data = QueryAsync(filter: $"RowKey eq '{proxyId}'"); + + return await data.FirstOrDefaultAsync(); + } + + public async Async.Task GetOrCreate(string region) { + var proxyList = QueryAsync(filter: $"region eq '{region}' and outdated eq false"); + + await foreach (var proxy in proxyList) { + if (IsOutdated(proxy)) { + await Replace(proxy with { Outdated = true }); + continue; + } + + if (!VmStateHelper.Available.Contains(proxy.State)) { + continue; + } + return proxy; + } + + _logTracer.Info($"creating proxy: region:{region}"); + var newProxy = new Proxy(region, Guid.NewGuid(), DateTimeOffset.UtcNow, VmState.Init, Auth.BuildAuth(), null, null, _context.ServiceConfiguration.OneFuzzVersion, null, false); + + await Replace(newProxy); + await _context.Events.SendEvent(new EventProxyCreated(region, newProxy.ProxyId)); + return newProxy; + } + + public bool IsAlive(Proxy proxy) { + var tenMinutesAgo = DateTimeOffset.UtcNow - TimeSpan.FromMinutes(10); + + if (proxy.Heartbeat != null && proxy.Heartbeat.TimeStamp < tenMinutesAgo) { + _logTracer.Info($"last heartbeat is more than an 10 minutes old: {proxy.Region} - last heartbeat:{proxy.Heartbeat} compared_to:{tenMinutesAgo}"); + return false; + } + + if (proxy.Heartbeat != null && proxy.TimeStamp != null && proxy.TimeStamp < tenMinutesAgo) { + _logTracer.Error($"no heartbeat in the last 10 minutes: {proxy.Region} timestamp: {proxy.TimeStamp} compared_to:{tenMinutesAgo}"); + return false; + } + + return true; + } + + public bool IsOutdated(Proxy proxy) { + if (!VmStateHelper.Available.Contains(proxy.State)) { + return false; + } + + if (proxy.Version != _context.ServiceConfiguration.OneFuzzVersion) { + _logTracer.Info($"mismatch version: proxy:{proxy.Version} service:{_context.ServiceConfiguration.OneFuzzVersion} state:{proxy.State}"); + return true; + } + + if (proxy.CreatedTimestamp != null) { + if (proxy.CreatedTimestamp < (DateTimeOffset.UtcNow - PROXY_LIFESPAN)) { + _logTracer.Info($"proxy older than 7 days:proxy-created:{proxy.CreatedTimestamp} state:{proxy.State}"); + return true; + } + } + return false; + } + + public async Async.Task SaveProxyConfig(Proxy proxy) { + var forwards = await GetForwards(proxy); + var url = (await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas"); + var queueSas = await _context.Queue.GetQueueSas("proxy", StorageType.Config, QueueSasPermissions.Add).EnsureNotNull("can't generate queue sas") ?? throw new Exception("Queue sas is null"); + + var proxyConfig = new ProxyConfig( + Url: url, + Notification: queueSas, + Region: proxy.Region, + ProxyId: proxy.ProxyId, + Forwards: forwards, + InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey.EnsureNotNull("missing InstrumentationKey"), + MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry.EnsureNotNull("missing Telemetry"), + InstanceId: await _context.Containers.GetInstanceId()); + + await _context.Containers.SaveBlob(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", _entityConverter.ToJsonString(proxyConfig), StorageType.Config); + } + + + public async Async.Task SetState(Proxy proxy, VmState state) { + if (proxy.State == state) { + return; + } + + await Replace(proxy with { State = state }); + + await _context.Events.SendEvent(new EventProxyStateUpdated(proxy.Region, proxy.ProxyId, proxy.State)); + } + + + public async Async.Task> GetForwards(Proxy proxy) { + var forwards = new List(); + + await foreach (var entry in _context.ProxyForwardOperations.SearchForward(region: proxy.Region, proxyId: proxy.ProxyId)) { + if (entry.EndTime < DateTimeOffset.UtcNow) { + await _context.ProxyForwardOperations.Delete(entry); + } else { + forwards.Add(new Forward(entry.Port, entry.DstPort, entry.DstIp)); + } + } + return forwards; + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index bc29cd2b4..d610a39ea 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -1,253 +1,253 @@ -using System.Net; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text.Json; -using System.Threading.Tasks; -using ApiService.OneFuzzLib.Orm; -using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; - -namespace Microsoft.OneFuzz.Service; - -public interface IWebhookOperations : IOrm { - Async.Task SendEvent(EventMessage eventMessage); - Async.Task GetByWebhookId(Guid webhookId); - Async.Task Send(WebhookMessageLog messageLog); - Task Ping(Webhook webhook); -} - -public class WebhookOperations : Orm, IWebhookOperations { - - private readonly IHttpClientFactory _httpFactory; - - public WebhookOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context) - : base(log, context) { - _httpFactory = httpFactory; - } - - async public Async.Task SendEvent(EventMessage eventMessage) { - await foreach (var webhook in GetWebhooksCached()) { - if (!webhook.EventTypes.Contains(eventMessage.EventType)) { - continue; - } - await AddEvent(webhook, eventMessage); - } - } - - async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) { - var message = new WebhookMessageLog( - EventId: eventMessage.EventId, - EventType: eventMessage.EventType, - Event: eventMessage.Event, - InstanceId: eventMessage.InstanceId, - InstanceName: eventMessage.InstanceName, - WebhookId: webhook.WebhookId, - TryCount: 0 - ); - - var r = await _context.WebhookMessageLogOperations.Replace(message); - if (!r.IsOk) { - var (status, reason) = r.ErrorV; - _logTracer.Error($"Failed to replace webhook message log due to [{status}] {reason}"); - } - await _context.WebhookMessageLogOperations.QueueWebhook(message); - } - - public async Async.Task Send(WebhookMessageLog messageLog) { - var webhook = await GetByWebhookId(messageLog.WebhookId); - if (webhook == null || webhook.Url == null) { - throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found"); - } - - var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat); - - var headers = new Dictionary { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } }; - - if (digest != null) { - headers["X-Onefuzz-Digest"] = digest; - } - - var client = new Request(_httpFactory.CreateClient()); - _logTracer.Info(data); - var response = client.Post(url: webhook.Url, json: data, headers: headers); - var result = response.Result; - if (result.StatusCode == HttpStatusCode.Accepted) { - return true; - } - return false; - } - - public async Task Ping(Webhook webhook) { - var ping = new EventPing(Guid.NewGuid()); - var instanceId = await _context.Containers.GetInstanceId(); - var instanceName = _context.Creds.GetInstanceName(); - await AddEvent(webhook, new EventMessage(Guid.NewGuid(), EventType.Ping, ping, instanceId, instanceName)); - return ping; - } - - // Not converting to bytes, as it's not neccessary in C#. Just keeping as string. - public async Async.Task> BuildMessage(Guid webhookId, Guid eventId, EventType eventType, BaseEvent webhookEvent, String? secretToken, WebhookMessageFormat? messageFormat) { - var entityConverter = new EntityConverter(); - string data = ""; - if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) { - var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookEvent, DataVersion: "1.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) }; - data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions()); - } else { - var instanceId = await _context.Containers.GetInstanceId(); - var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName()); - - data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions()); - } - - string? digest = null; - var hmac = HMAC.Create("HMACSHA512"); - if (secretToken != null && hmac != null) { - hmac.Key = System.Text.Encoding.UTF8.GetBytes(secretToken); - digest = Convert.ToHexString(hmac.ComputeHash(System.Text.Encoding.UTF8.GetBytes(data))); - } - return new Tuple(data, digest); - - } - - public async Async.Task GetByWebhookId(Guid webhookId) { - var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}'"); - - return await data.FirstOrDefaultAsync(); - } - - //todo: caching - public IAsyncEnumerable GetWebhooksCached() { - return QueryAsync(); - } - -} - -public interface IWebhookMessageLogOperations : IOrm { - IAsyncEnumerable SearchExpired(); - public Async.Task ProcessFromQueue(WebhookMessageQueueObj obj); - public Async.Task QueueWebhook(WebhookMessageLog webhookLog); -} - - -public class WebhookMessageLogOperations : Orm, IWebhookMessageLogOperations { - const int EXPIRE_DAYS = 7; - const int MAX_TRIES = 5; - - - public WebhookMessageLogOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context) - : base(log, context) { - - } - - - public async Async.Task QueueWebhook(WebhookMessageLog webhookLog) { - var obj = new WebhookMessageQueueObj(webhookLog.WebhookId, webhookLog.EventId); - - TimeSpan? visibilityTimeout = webhookLog.State switch { - WebhookMessageState.Queued => TimeSpan.Zero, - WebhookMessageState.Retrying => TimeSpan.FromSeconds(30), - _ => null - }; - - if (visibilityTimeout == null) { - _logTracer.WithTags( - new[] { - ("WebhookId", webhookLog.WebhookId.ToString()), - ("EventId", webhookLog.EventId.ToString()) } - ). - Error($"invalid WebhookMessage queue state, not queuing. {webhookLog.WebhookId}:{webhookLog.EventId} - {webhookLog.State}"); - } else { - await _context.Queue.QueueObject("webhooks", obj, StorageType.Config, visibilityTimeout: visibilityTimeout); - } - } - - public async Async.Task ProcessFromQueue(WebhookMessageQueueObj obj) { - var message = await GetWebhookMessageById(obj.WebhookId, obj.EventId); - - if (message == null) { - _logTracer.WithTags( - new[] { - ("WebhookId", obj.WebhookId.ToString()), - ("EventId", obj.EventId.ToString()) } - ). - Error($"webhook message log not found for webhookId: {obj.WebhookId} and eventId: {obj.EventId}"); - } else { - await Process(message); - } - } - - private async Async.Task Process(WebhookMessageLog message) { - - if (message.State == WebhookMessageState.Failed || message.State == WebhookMessageState.Succeeded) { - _logTracer.WithTags( - new[] { - ("WebhookId", message.WebhookId.ToString()), - ("EventId", message.EventId.ToString()) } - ). - Error($"webhook message already handled. {message.WebhookId}:{message.EventId}"); - return; - } - - var newMessage = message with { TryCount = message.TryCount + 1 }; - - _logTracer.Info($"sending webhook: {message.WebhookId}:{message.EventId}"); - var success = await Send(newMessage); - if (success) { - newMessage = newMessage with { State = WebhookMessageState.Succeeded }; - await Replace(newMessage); - _logTracer.Info($"sent webhook event {newMessage.WebhookId}:{newMessage.EventId}"); - } else if (newMessage.TryCount < MAX_TRIES) { - newMessage = newMessage with { State = WebhookMessageState.Retrying }; - await Replace(newMessage); - await QueueWebhook(newMessage); - _logTracer.Warning($"sending webhook event failed, re-queued {newMessage.WebhookId}:{newMessage.EventId}"); - } else { - newMessage = newMessage with { State = WebhookMessageState.Failed }; - await Replace(newMessage); - _logTracer.Info($"sending webhook: {newMessage.WebhookId} event: {newMessage.EventId} failed {newMessage.TryCount} times."); - } - - } - - private async Async.Task Send(WebhookMessageLog message) { - var webhook = await _context.WebhookOperations.GetByWebhookId(message.WebhookId); - if (webhook == null) { - _logTracer.WithTags( - new[] { - ("WebhookId", message.WebhookId.ToString()), - } - ). - Error($"webhook not found for webhookId: {message.WebhookId}"); - return false; - } - - try { - return await _context.WebhookOperations.Send(message); - } catch (Exception exc) { - _logTracer.WithTags( - new[] { - ("WebhookId", message.WebhookId.ToString()) - } - ). - Exception(exc); - return false; - } - - } - - private void QueueObject(string v, WebhookMessageQueueObj obj, StorageType config, int? visibility_timeout) { - throw new NotImplementedException(); - } - - public IAsyncEnumerable SearchExpired() { - var expireTime = (DateTimeOffset.UtcNow - TimeSpan.FromDays(EXPIRE_DAYS)).ToString("o"); - - var timeFilter = $"Timestamp lt datetime'{expireTime}'"; - return QueryAsync(filter: timeFilter); - } - - public async Async.Task GetWebhookMessageById(Guid webhookId, Guid eventId) { - var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}' and RowKey eq '{eventId}'"); - - return await data.FirstOrDefaultAsync(); - } -} +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading.Tasks; +using ApiService.OneFuzzLib.Orm; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service; + +public interface IWebhookOperations : IOrm { + Async.Task SendEvent(EventMessage eventMessage); + Async.Task GetByWebhookId(Guid webhookId); + Async.Task Send(WebhookMessageLog messageLog); + Task Ping(Webhook webhook); +} + +public class WebhookOperations : Orm, IWebhookOperations { + + private readonly IHttpClientFactory _httpFactory; + + public WebhookOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context) + : base(log, context) { + _httpFactory = httpFactory; + } + + async public Async.Task SendEvent(EventMessage eventMessage) { + await foreach (var webhook in GetWebhooksCached()) { + if (!webhook.EventTypes.Contains(eventMessage.EventType)) { + continue; + } + await AddEvent(webhook, eventMessage); + } + } + + async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) { + var message = new WebhookMessageLog( + EventId: eventMessage.EventId, + EventType: eventMessage.EventType, + Event: eventMessage.Event, + InstanceId: eventMessage.InstanceId, + InstanceName: eventMessage.InstanceName, + WebhookId: webhook.WebhookId, + TryCount: 0 + ); + + var r = await _context.WebhookMessageLogOperations.Replace(message); + if (!r.IsOk) { + var (status, reason) = r.ErrorV; + _logTracer.Error($"Failed to replace webhook message log due to [{status}] {reason}"); + } + await _context.WebhookMessageLogOperations.QueueWebhook(message); + } + + public async Async.Task Send(WebhookMessageLog messageLog) { + var webhook = await GetByWebhookId(messageLog.WebhookId); + if (webhook == null || webhook.Url == null) { + throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found"); + } + + var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat); + + var headers = new Dictionary { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } }; + + if (digest != null) { + headers["X-Onefuzz-Digest"] = digest; + } + + var client = new Request(_httpFactory.CreateClient()); + _logTracer.Info(data); + var response = client.Post(url: webhook.Url, json: data, headers: headers); + var result = response.Result; + if (result.StatusCode == HttpStatusCode.Accepted) { + return true; + } + return false; + } + + public async Task Ping(Webhook webhook) { + var ping = new EventPing(Guid.NewGuid()); + var instanceId = await _context.Containers.GetInstanceId(); + var instanceName = _context.Creds.GetInstanceName(); + await AddEvent(webhook, new EventMessage(Guid.NewGuid(), EventType.Ping, ping, instanceId, instanceName)); + return ping; + } + + // Not converting to bytes, as it's not neccessary in C#. Just keeping as string. + public async Async.Task> BuildMessage(Guid webhookId, Guid eventId, EventType eventType, BaseEvent webhookEvent, String? secretToken, WebhookMessageFormat? messageFormat) { + var entityConverter = new EntityConverter(); + string data = ""; + if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) { + var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookEvent, DataVersion: "1.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) }; + data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions()); + } else { + var instanceId = await _context.Containers.GetInstanceId(); + var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName()); + + data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions()); + } + + string? digest = null; + var hmac = HMAC.Create("HMACSHA512"); + if (secretToken != null && hmac != null) { + hmac.Key = System.Text.Encoding.UTF8.GetBytes(secretToken); + digest = Convert.ToHexString(hmac.ComputeHash(System.Text.Encoding.UTF8.GetBytes(data))); + } + return new Tuple(data, digest); + + } + + public async Async.Task GetByWebhookId(Guid webhookId) { + var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}'"); + + return await data.FirstOrDefaultAsync(); + } + + //todo: caching + public IAsyncEnumerable GetWebhooksCached() { + return QueryAsync(); + } + +} + +public interface IWebhookMessageLogOperations : IOrm { + IAsyncEnumerable SearchExpired(); + public Async.Task ProcessFromQueue(WebhookMessageQueueObj obj); + public Async.Task QueueWebhook(WebhookMessageLog webhookLog); +} + + +public class WebhookMessageLogOperations : Orm, IWebhookMessageLogOperations { + const int EXPIRE_DAYS = 7; + const int MAX_TRIES = 5; + + + public WebhookMessageLogOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context) + : base(log, context) { + + } + + + public async Async.Task QueueWebhook(WebhookMessageLog webhookLog) { + var obj = new WebhookMessageQueueObj(webhookLog.WebhookId, webhookLog.EventId); + + TimeSpan? visibilityTimeout = webhookLog.State switch { + WebhookMessageState.Queued => TimeSpan.Zero, + WebhookMessageState.Retrying => TimeSpan.FromSeconds(30), + _ => null + }; + + if (visibilityTimeout == null) { + _logTracer.WithTags( + new[] { + ("WebhookId", webhookLog.WebhookId.ToString()), + ("EventId", webhookLog.EventId.ToString()) } + ). + Error($"invalid WebhookMessage queue state, not queuing. {webhookLog.WebhookId}:{webhookLog.EventId} - {webhookLog.State}"); + } else { + await _context.Queue.QueueObject("webhooks", obj, StorageType.Config, visibilityTimeout: visibilityTimeout); + } + } + + public async Async.Task ProcessFromQueue(WebhookMessageQueueObj obj) { + var message = await GetWebhookMessageById(obj.WebhookId, obj.EventId); + + if (message == null) { + _logTracer.WithTags( + new[] { + ("WebhookId", obj.WebhookId.ToString()), + ("EventId", obj.EventId.ToString()) } + ). + Error($"webhook message log not found for webhookId: {obj.WebhookId} and eventId: {obj.EventId}"); + } else { + await Process(message); + } + } + + private async Async.Task Process(WebhookMessageLog message) { + + if (message.State == WebhookMessageState.Failed || message.State == WebhookMessageState.Succeeded) { + _logTracer.WithTags( + new[] { + ("WebhookId", message.WebhookId.ToString()), + ("EventId", message.EventId.ToString()) } + ). + Error($"webhook message already handled. {message.WebhookId}:{message.EventId}"); + return; + } + + var newMessage = message with { TryCount = message.TryCount + 1 }; + + _logTracer.Info($"sending webhook: {message.WebhookId}:{message.EventId}"); + var success = await Send(newMessage); + if (success) { + newMessage = newMessage with { State = WebhookMessageState.Succeeded }; + await Replace(newMessage); + _logTracer.Info($"sent webhook event {newMessage.WebhookId}:{newMessage.EventId}"); + } else if (newMessage.TryCount < MAX_TRIES) { + newMessage = newMessage with { State = WebhookMessageState.Retrying }; + await Replace(newMessage); + await QueueWebhook(newMessage); + _logTracer.Warning($"sending webhook event failed, re-queued {newMessage.WebhookId}:{newMessage.EventId}"); + } else { + newMessage = newMessage with { State = WebhookMessageState.Failed }; + await Replace(newMessage); + _logTracer.Info($"sending webhook: {newMessage.WebhookId} event: {newMessage.EventId} failed {newMessage.TryCount} times."); + } + + } + + private async Async.Task Send(WebhookMessageLog message) { + var webhook = await _context.WebhookOperations.GetByWebhookId(message.WebhookId); + if (webhook == null) { + _logTracer.WithTags( + new[] { + ("WebhookId", message.WebhookId.ToString()), + } + ). + Error($"webhook not found for webhookId: {message.WebhookId}"); + return false; + } + + try { + return await _context.WebhookOperations.Send(message); + } catch (Exception exc) { + _logTracer.WithTags( + new[] { + ("WebhookId", message.WebhookId.ToString()) + } + ). + Exception(exc); + return false; + } + + } + + private void QueueObject(string v, WebhookMessageQueueObj obj, StorageType config, int? visibility_timeout) { + throw new NotImplementedException(); + } + + public IAsyncEnumerable SearchExpired() { + var expireTime = (DateTimeOffset.UtcNow - TimeSpan.FromDays(EXPIRE_DAYS)).ToString("o"); + + var timeFilter = $"Timestamp lt datetime'{expireTime}'"; + return QueryAsync(filter: timeFilter); + } + + public async Async.Task GetWebhookMessageById(Guid webhookId, Guid eventId) { + var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}' and RowKey eq '{eventId}'"); + + return await data.FirstOrDefaultAsync(); + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs index 0bbf1a162..89c500887 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs @@ -1,325 +1,325 @@ -using System.Collections.Concurrent; -using System.Linq.Expressions; -using System.Reflection; -using System.Text.Json; -using System.Text.Json.Serialization; -using Azure; -using Azure.Data.Tables; - -namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; - -public abstract record EntityBase { - [JsonIgnore] public ETag? ETag { get; set; } - public DateTimeOffset? TimeStamp { get; set; } - - // https://docs.microsoft.com/en-us/rest/api/storageservices/designing-a-scalable-partitioning-strategy-for-azure-table-storage#yyy - // Produce "good-quality-table-key" based on a DateTimeOffset timestamp - public static string NewSortedKey => $"{DateTimeOffset.MaxValue.Ticks - DateTimeOffset.UtcNow.Ticks}"; -} - -public abstract record StatefulEntityBase([property: JsonIgnore] T BaseState) : EntityBase() where T : Enum; - - - -/// How the value is populated -public enum InitMethod { - //T() will be used - DefaultConstructor, -} -[AttributeUsage(AttributeTargets.Parameter)] -public class DefaultValueAttribute : Attribute { - - public InitMethod InitMethod { get; } - public DefaultValueAttribute(InitMethod initMethod) { - InitMethod = initMethod; - } -} - -/// Indicates that the enum cases should no be renamed -[AttributeUsage(AttributeTargets.Enum)] -public class SerializeValueAttribute : Attribute { } - -/// Indicates that the enum cases should no be renamed -[AttributeUsage(AttributeTargets.Enum)] -public class SkipRenameAttribute : Attribute { } -[AttributeUsage(AttributeTargets.Parameter)] -public class RowKeyAttribute : Attribute { } -[AttributeUsage(AttributeTargets.Parameter)] -public class PartitionKeyAttribute : Attribute { } -[AttributeUsage(AttributeTargets.Property)] -public class TypeDiscrimnatorAttribute : Attribute { - public string FieldName { get; } - // the type of a function that takes the value of fieldName as an input and return the type - public Type ConverterType { get; } - - public TypeDiscrimnatorAttribute(string fieldName, Type converterType) { - if (!converterType.IsAssignableTo(typeof(ITypeProvider))) { - throw new ArgumentException($"the provided type needs to implement ITypeProvider"); - } - - FieldName = fieldName; - ConverterType = converterType; - } -} - -public interface ITypeProvider { - Type GetTypeInfo(object input); -} - -public enum EntityPropertyKind { - PartitionKey, - RowKey, - Column -} -public record EntityProperty( - string name, - string columnName, - Type type, - EntityPropertyKind kind, - (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator, - DefaultValueAttribute? defaultValue, - ParameterInfo parameterInfo - ); -public record EntityInfo(Type type, ILookup properties, Func constructor); - -class OnefuzzNamingPolicy : JsonNamingPolicy { - public override string ConvertName(string name) { - return CaseConverter.PascalToSnake(name); - } -} -public class EntityConverter { - private readonly JsonSerializerOptions _options; - - private readonly ConcurrentDictionary _cache; - - public EntityConverter() { - _options = GetJsonSerializerOptions(); - _cache = new ConcurrentDictionary(); - } - - public static JsonSerializerOptions GetJsonSerializerOptions() { - var options = new JsonSerializerOptions() { - PropertyNamingPolicy = new OnefuzzNamingPolicy(), - }; - options.Converters.Add(new CustomEnumConverterFactory()); - options.Converters.Add(new PolymorphicConverterFactory()); - return options; - } - - internal static Func BuildConstructerFrom(ConstructorInfo constructorInfo) { - var constructorParameters = Expression.Parameter(typeof(object?[])); - - var parameterExpressions = - constructorInfo.GetParameters().Select((parameterInfo, i) => { - var ithIndex = Expression.Constant(i); - var ithParameter = Expression.ArrayIndex(constructorParameters, ithIndex); - var unboxedIthParameter = Expression.Convert(ithParameter, parameterInfo.ParameterType); - return unboxedIthParameter; - - }).ToArray(); - - NewExpression constructorCall = Expression.New(constructorInfo, parameterExpressions); - - Func ctor = Expression.Lambda>(constructorCall, constructorParameters).Compile(); - return ctor; - } - - private static IEnumerable GetEntityProperties(ParameterInfo parameterInfo) { - var name = parameterInfo.Name.EnsureNotNull($"Invalid paramter {parameterInfo}"); - var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid paramter {parameterInfo}"); - var isRowkey = parameterInfo.GetCustomAttribute(typeof(RowKeyAttribute)) != null; - var isPartitionkey = parameterInfo.GetCustomAttribute(typeof(PartitionKeyAttribute)) != null; - - var discriminatorAttribute = typeof(T).GetProperty(name)?.GetCustomAttribute(); - var defaultValueAttribute = parameterInfo.GetCustomAttribute(); - - - (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null; - if (discriminatorAttribute != null) { - var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrive the type provider")); - discriminator = (discriminatorAttribute, t); - } - - - if (isPartitionkey) { - yield return new EntityProperty(name, "PartitionKey", parameterType, EntityPropertyKind.PartitionKey, discriminator, defaultValueAttribute, parameterInfo); - } - - if (isRowkey) { - yield return new EntityProperty(name, "RowKey", parameterType, EntityPropertyKind.RowKey, discriminator, defaultValueAttribute, parameterInfo); - } - - if (!isPartitionkey && !isRowkey) { - var columnName = typeof(T).GetProperty(name)?.GetCustomAttribute()?.Name ?? CaseConverter.PascalToSnake(name); - yield return new EntityProperty(name, columnName, parameterType, EntityPropertyKind.Column, discriminator, defaultValueAttribute, parameterInfo); - } - } - - - private EntityInfo GetEntityInfo() { - return _cache.GetOrAdd(typeof(T), type => { - var constructor = type.GetConstructors()[0]; - var parameterInfos = constructor.GetParameters(); - var parameters = - parameterInfos.SelectMany(GetEntityProperties).ToArray(); - - return new EntityInfo(typeof(T), parameters.ToLookup(x => x.name), BuildConstructerFrom(constructor)); - }); - } - - public string ToJsonString(T typedEntity) => JsonSerializer.Serialize(typedEntity, _options); - - public T? FromJsonString(string value) => JsonSerializer.Deserialize(value, _options); - - public TableEntity ToTableEntity(T typedEntity) where T : EntityBase { - if (typedEntity == null) { - throw new ArgumentNullException(nameof(typedEntity)); - } - - var type = typeof(T); - - var entityInfo = GetEntityInfo(); - Dictionary columnValues = entityInfo.properties.SelectMany(x => x).Select(prop => { - var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity); - if (value == null) { - return (prop.columnName, value: (object?)null); - } - if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey) { - return (prop.columnName, value?.ToString()); - } - if (prop.type == typeof(Guid) || prop.type == typeof(Guid?) || prop.type == typeof(Uri)) { - return (prop.columnName, value?.ToString()); - } - if (prop.type == typeof(bool) - || prop.type == typeof(bool?) - || prop.type == typeof(string) - || prop.type == typeof(DateTime) - || prop.type == typeof(DateTime?) - || prop.type == typeof(DateTimeOffset) - || prop.type == typeof(DateTimeOffset?) - || prop.type == typeof(int) - || prop.type == typeof(int?) - || prop.type == typeof(long) - || prop.type == typeof(long?) - || prop.type == typeof(double) - || prop.type == typeof(double?) - - ) { - return (prop.columnName, value); - } - - var serialized = JsonSerializer.Serialize(value, _options); - return (prop.columnName, serialized.Trim('"')); - - }).ToDictionary(x => x.columnName, x => x.value); - - var tableEntity = new TableEntity(columnValues); - - if (typedEntity.ETag.HasValue) { - tableEntity.ETag = typedEntity.ETag.Value; - } - - return tableEntity; - } - - - private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) { - var ef = info.properties[name].First(); - if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) { - if (ef.type == typeof(string)) - return entity.GetString(ef.kind.ToString()); - else if (ef.type == typeof(Guid)) - return Guid.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type == typeof(int)) - return int.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type == typeof(long)) - return long.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type.IsClass) - return ef.type.GetConstructor(new[] { typeof(string) })!.Invoke(new[] { entity.GetString(ef.kind.ToString()) }); - else { - throw new Exception($"invalid partition or row key type of {info.type} property {name}: {ef.type}"); - } - } - - var fieldName = ef.columnName; - var obj = entity[fieldName]; - if (obj == null) { - - if (ef.parameterInfo.HasDefaultValue) { - return ef.parameterInfo.DefaultValue; - } - - return ef.defaultValue switch { - DefaultValueAttribute { InitMethod: InitMethod.DefaultConstructor } => Activator.CreateInstance(ef.type), - _ => null, - }; - } - - try { - if (ef.type == typeof(string)) { - return entity.GetString(fieldName); - } else if (ef.type == typeof(bool) || ef.type == typeof(bool?)) { - return entity.GetBoolean(fieldName); - } else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?)) { - return entity.GetDateTimeOffset(fieldName); - } else if (ef.type == typeof(DateTime) || ef.type == typeof(DateTime?)) { - return entity.GetDateTime(fieldName); - } else if (ef.type == typeof(double) || ef.type == typeof(double?)) { - return entity.GetDouble(fieldName); - } else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?)) { - return (object?)Guid.Parse(entity.GetString(fieldName)); - } else if (ef.type == typeof(int) || ef.type == typeof(short) || ef.type == typeof(int?) || ef.type == typeof(short?)) { - return entity.GetInt32(fieldName); - } else if (ef.type == typeof(long) || ef.type == typeof(long?)) { - return entity.GetInt64(fieldName); - } else { - var outputType = ef.type; - if (ef.discriminator != null) { - var (attr, typeProvider) = ef.discriminator.Value; - var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}"); - outputType = typeProvider.GetTypeInfo(v); - } - - var objType = obj.GetType(); - if (objType == typeof(string)) { - var value = entity.GetString(fieldName); - if (value.StartsWith('[') || value.StartsWith('{') || value == "null") { - return JsonSerializer.Deserialize(value, outputType, options: _options); - } else { - return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options); - } - } else { - var value = entity.GetString(fieldName); - return JsonSerializer.Deserialize(value, outputType, options: _options); - } - } - } catch (Exception ex) { - throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}')", ex); - } - } - - - public T ToRecord(TableEntity entity) where T : EntityBase { - var entityInfo = GetEntityInfo(); - - object?[] parameters; - try { - parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity)).ToArray(); - } catch (Exception ex) { - throw new InvalidOperationException($"Unable to extract properties from TableEntity for {typeof(T)}", ex); - } - - try { - var entityRecord = (T)entityInfo.constructor.Invoke(parameters); - if (entity.ETag != default) { - entityRecord.ETag = entity.ETag; - } - entityRecord.TimeStamp = entity.Timestamp; - return entityRecord; - - } catch (Exception ex) { - var stringParam = string.Join(", ", parameters); - throw new InvalidOperationException($"Could not initialize object of type {typeof(T)} with the following parameters: {stringParam} constructor {entityInfo.constructor}", ex); - } - } -} +using System.Collections.Concurrent; +using System.Linq.Expressions; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; +using Azure; +using Azure.Data.Tables; + +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +public abstract record EntityBase { + [JsonIgnore] public ETag? ETag { get; set; } + public DateTimeOffset? TimeStamp { get; set; } + + // https://docs.microsoft.com/en-us/rest/api/storageservices/designing-a-scalable-partitioning-strategy-for-azure-table-storage#yyy + // Produce "good-quality-table-key" based on a DateTimeOffset timestamp + public static string NewSortedKey => $"{DateTimeOffset.MaxValue.Ticks - DateTimeOffset.UtcNow.Ticks}"; +} + +public abstract record StatefulEntityBase([property: JsonIgnore] T BaseState) : EntityBase() where T : Enum; + + + +/// How the value is populated +public enum InitMethod { + //T() will be used + DefaultConstructor, +} +[AttributeUsage(AttributeTargets.Parameter)] +public class DefaultValueAttribute : Attribute { + + public InitMethod InitMethod { get; } + public DefaultValueAttribute(InitMethod initMethod) { + InitMethod = initMethod; + } +} + +/// Indicates that the enum cases should no be renamed +[AttributeUsage(AttributeTargets.Enum)] +public class SerializeValueAttribute : Attribute { } + +/// Indicates that the enum cases should no be renamed +[AttributeUsage(AttributeTargets.Enum)] +public class SkipRenameAttribute : Attribute { } +[AttributeUsage(AttributeTargets.Parameter)] +public class RowKeyAttribute : Attribute { } +[AttributeUsage(AttributeTargets.Parameter)] +public class PartitionKeyAttribute : Attribute { } +[AttributeUsage(AttributeTargets.Property)] +public class TypeDiscrimnatorAttribute : Attribute { + public string FieldName { get; } + // the type of a function that takes the value of fieldName as an input and return the type + public Type ConverterType { get; } + + public TypeDiscrimnatorAttribute(string fieldName, Type converterType) { + if (!converterType.IsAssignableTo(typeof(ITypeProvider))) { + throw new ArgumentException($"the provided type needs to implement ITypeProvider"); + } + + FieldName = fieldName; + ConverterType = converterType; + } +} + +public interface ITypeProvider { + Type GetTypeInfo(object input); +} + +public enum EntityPropertyKind { + PartitionKey, + RowKey, + Column +} +public record EntityProperty( + string name, + string columnName, + Type type, + EntityPropertyKind kind, + (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator, + DefaultValueAttribute? defaultValue, + ParameterInfo parameterInfo + ); +public record EntityInfo(Type type, ILookup properties, Func constructor); + +class OnefuzzNamingPolicy : JsonNamingPolicy { + public override string ConvertName(string name) { + return CaseConverter.PascalToSnake(name); + } +} +public class EntityConverter { + private readonly JsonSerializerOptions _options; + + private readonly ConcurrentDictionary _cache; + + public EntityConverter() { + _options = GetJsonSerializerOptions(); + _cache = new ConcurrentDictionary(); + } + + public static JsonSerializerOptions GetJsonSerializerOptions() { + var options = new JsonSerializerOptions() { + PropertyNamingPolicy = new OnefuzzNamingPolicy(), + }; + options.Converters.Add(new CustomEnumConverterFactory()); + options.Converters.Add(new PolymorphicConverterFactory()); + return options; + } + + internal static Func BuildConstructerFrom(ConstructorInfo constructorInfo) { + var constructorParameters = Expression.Parameter(typeof(object?[])); + + var parameterExpressions = + constructorInfo.GetParameters().Select((parameterInfo, i) => { + var ithIndex = Expression.Constant(i); + var ithParameter = Expression.ArrayIndex(constructorParameters, ithIndex); + var unboxedIthParameter = Expression.Convert(ithParameter, parameterInfo.ParameterType); + return unboxedIthParameter; + + }).ToArray(); + + NewExpression constructorCall = Expression.New(constructorInfo, parameterExpressions); + + Func ctor = Expression.Lambda>(constructorCall, constructorParameters).Compile(); + return ctor; + } + + private static IEnumerable GetEntityProperties(ParameterInfo parameterInfo) { + var name = parameterInfo.Name.EnsureNotNull($"Invalid paramter {parameterInfo}"); + var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid paramter {parameterInfo}"); + var isRowkey = parameterInfo.GetCustomAttribute(typeof(RowKeyAttribute)) != null; + var isPartitionkey = parameterInfo.GetCustomAttribute(typeof(PartitionKeyAttribute)) != null; + + var discriminatorAttribute = typeof(T).GetProperty(name)?.GetCustomAttribute(); + var defaultValueAttribute = parameterInfo.GetCustomAttribute(); + + + (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null; + if (discriminatorAttribute != null) { + var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrive the type provider")); + discriminator = (discriminatorAttribute, t); + } + + + if (isPartitionkey) { + yield return new EntityProperty(name, "PartitionKey", parameterType, EntityPropertyKind.PartitionKey, discriminator, defaultValueAttribute, parameterInfo); + } + + if (isRowkey) { + yield return new EntityProperty(name, "RowKey", parameterType, EntityPropertyKind.RowKey, discriminator, defaultValueAttribute, parameterInfo); + } + + if (!isPartitionkey && !isRowkey) { + var columnName = typeof(T).GetProperty(name)?.GetCustomAttribute()?.Name ?? CaseConverter.PascalToSnake(name); + yield return new EntityProperty(name, columnName, parameterType, EntityPropertyKind.Column, discriminator, defaultValueAttribute, parameterInfo); + } + } + + + private EntityInfo GetEntityInfo() { + return _cache.GetOrAdd(typeof(T), type => { + var constructor = type.GetConstructors()[0]; + var parameterInfos = constructor.GetParameters(); + var parameters = + parameterInfos.SelectMany(GetEntityProperties).ToArray(); + + return new EntityInfo(typeof(T), parameters.ToLookup(x => x.name), BuildConstructerFrom(constructor)); + }); + } + + public string ToJsonString(T typedEntity) => JsonSerializer.Serialize(typedEntity, _options); + + public T? FromJsonString(string value) => JsonSerializer.Deserialize(value, _options); + + public TableEntity ToTableEntity(T typedEntity) where T : EntityBase { + if (typedEntity == null) { + throw new ArgumentNullException(nameof(typedEntity)); + } + + var type = typeof(T); + + var entityInfo = GetEntityInfo(); + Dictionary columnValues = entityInfo.properties.SelectMany(x => x).Select(prop => { + var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity); + if (value == null) { + return (prop.columnName, value: (object?)null); + } + if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey) { + return (prop.columnName, value?.ToString()); + } + if (prop.type == typeof(Guid) || prop.type == typeof(Guid?) || prop.type == typeof(Uri)) { + return (prop.columnName, value?.ToString()); + } + if (prop.type == typeof(bool) + || prop.type == typeof(bool?) + || prop.type == typeof(string) + || prop.type == typeof(DateTime) + || prop.type == typeof(DateTime?) + || prop.type == typeof(DateTimeOffset) + || prop.type == typeof(DateTimeOffset?) + || prop.type == typeof(int) + || prop.type == typeof(int?) + || prop.type == typeof(long) + || prop.type == typeof(long?) + || prop.type == typeof(double) + || prop.type == typeof(double?) + + ) { + return (prop.columnName, value); + } + + var serialized = JsonSerializer.Serialize(value, _options); + return (prop.columnName, serialized.Trim('"')); + + }).ToDictionary(x => x.columnName, x => x.value); + + var tableEntity = new TableEntity(columnValues); + + if (typedEntity.ETag.HasValue) { + tableEntity.ETag = typedEntity.ETag.Value; + } + + return tableEntity; + } + + + private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) { + var ef = info.properties[name].First(); + if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) { + if (ef.type == typeof(string)) + return entity.GetString(ef.kind.ToString()); + else if (ef.type == typeof(Guid)) + return Guid.Parse(entity.GetString(ef.kind.ToString())); + else if (ef.type == typeof(int)) + return int.Parse(entity.GetString(ef.kind.ToString())); + else if (ef.type == typeof(long)) + return long.Parse(entity.GetString(ef.kind.ToString())); + else if (ef.type.IsClass) + return ef.type.GetConstructor(new[] { typeof(string) })!.Invoke(new[] { entity.GetString(ef.kind.ToString()) }); + else { + throw new Exception($"invalid partition or row key type of {info.type} property {name}: {ef.type}"); + } + } + + var fieldName = ef.columnName; + var obj = entity[fieldName]; + if (obj == null) { + + if (ef.parameterInfo.HasDefaultValue) { + return ef.parameterInfo.DefaultValue; + } + + return ef.defaultValue switch { + DefaultValueAttribute { InitMethod: InitMethod.DefaultConstructor } => Activator.CreateInstance(ef.type), + _ => null, + }; + } + + try { + if (ef.type == typeof(string)) { + return entity.GetString(fieldName); + } else if (ef.type == typeof(bool) || ef.type == typeof(bool?)) { + return entity.GetBoolean(fieldName); + } else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?)) { + return entity.GetDateTimeOffset(fieldName); + } else if (ef.type == typeof(DateTime) || ef.type == typeof(DateTime?)) { + return entity.GetDateTime(fieldName); + } else if (ef.type == typeof(double) || ef.type == typeof(double?)) { + return entity.GetDouble(fieldName); + } else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?)) { + return (object?)Guid.Parse(entity.GetString(fieldName)); + } else if (ef.type == typeof(int) || ef.type == typeof(short) || ef.type == typeof(int?) || ef.type == typeof(short?)) { + return entity.GetInt32(fieldName); + } else if (ef.type == typeof(long) || ef.type == typeof(long?)) { + return entity.GetInt64(fieldName); + } else { + var outputType = ef.type; + if (ef.discriminator != null) { + var (attr, typeProvider) = ef.discriminator.Value; + var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}"); + outputType = typeProvider.GetTypeInfo(v); + } + + var objType = obj.GetType(); + if (objType == typeof(string)) { + var value = entity.GetString(fieldName); + if (value.StartsWith('[') || value.StartsWith('{') || value == "null") { + return JsonSerializer.Deserialize(value, outputType, options: _options); + } else { + return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options); + } + } else { + var value = entity.GetString(fieldName); + return JsonSerializer.Deserialize(value, outputType, options: _options); + } + } + } catch (Exception ex) { + throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}')", ex); + } + } + + + public T ToRecord(TableEntity entity) where T : EntityBase { + var entityInfo = GetEntityInfo(); + + object?[] parameters; + try { + parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity)).ToArray(); + } catch (Exception ex) { + throw new InvalidOperationException($"Unable to extract properties from TableEntity for {typeof(T)}", ex); + } + + try { + var entityRecord = (T)entityInfo.constructor.Invoke(parameters); + if (entity.ETag != default) { + entityRecord.ETag = entity.ETag; + } + entityRecord.TimeStamp = entity.Timestamp; + return entityRecord; + + } catch (Exception ex) { + var stringParam = string.Join(", ", parameters); + throw new InvalidOperationException($"Could not initialize object of type {typeof(T)} with the following parameters: {stringParam} constructor {entityInfo.constructor}", ex); + } + } +} diff --git a/src/agent/coverage/src/cobertura_test.xml b/src/agent/coverage/src/cobertura_test.xml index 6a3db435f..a484b4e50 100644 --- a/src/agent/coverage/src/cobertura_test.xml +++ b/src/agent/coverage/src/cobertura_test.xml @@ -1,42 +1,42 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/agent/srcview/README.md b/src/agent/srcview/README.md index aefd65a09..f34045fad 100644 --- a/src/agent/srcview/README.md +++ b/src/agent/srcview/README.md @@ -1,34 +1,34 @@ -# srcview - -A library for mapping module+offset to source:line. Note that you'll get -significantly better results if you use instruction level coverage as -opposed to branch. Alternatively you're welcome to post-process branch -coverage into instruction, but this project does not assist with that. - -## Docs - -`cargo doc --open` - -## Usage - -See [`examples/dump_cobertura.rs`](examples/dump_cobertura.rs). - -This can be run with `cargo run --example dump_cobertura res\example.pdb res\example.txt`. - -The biggest challenge when making this work is likely getting the absolute PDB -paths to match relative paths inside your repo. To make this a bit easier, you -can dump the PDB paths with: `cargo run --example dump_paths res\example.pdb` - -## ADO Integration - -See [`azure-pipelines.yml`](azure-pipelines.yml). - -## VSCode Integration - -Install [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters), -then place a file named `coverage.xml` at a location such that the relative -file paths match your repo. Note that this is a file you should generate -yourself, and that the `coverage.xml` included in this repo will probably not -help you. From there, navigate to any file you expect to see coverage in. If -the paths and coverage file correctly line up, you should see red or green bars +# srcview + +A library for mapping module+offset to source:line. Note that you'll get +significantly better results if you use instruction level coverage as +opposed to branch. Alternatively you're welcome to post-process branch +coverage into instruction, but this project does not assist with that. + +## Docs + +`cargo doc --open` + +## Usage + +See [`examples/dump_cobertura.rs`](examples/dump_cobertura.rs). + +This can be run with `cargo run --example dump_cobertura res\example.pdb res\example.txt`. + +The biggest challenge when making this work is likely getting the absolute PDB +paths to match relative paths inside your repo. To make this a bit easier, you +can dump the PDB paths with: `cargo run --example dump_paths res\example.pdb` + +## ADO Integration + +See [`azure-pipelines.yml`](azure-pipelines.yml). + +## VSCode Integration + +Install [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters), +then place a file named `coverage.xml` at a location such that the relative +file paths match your repo. Note that this is a file you should generate +yourself, and that the `coverage.xml` included in this repo will probably not +help you. From there, navigate to any file you expect to see coverage in. If +the paths and coverage file correctly line up, you should see red or green bars next to the source lines. \ No newline at end of file diff --git a/src/agent/srcview/TODO.md b/src/agent/srcview/TODO.md index 034ebb4ef..100e8096f 100644 --- a/src/agent/srcview/TODO.md +++ b/src/agent/srcview/TODO.md @@ -1,6 +1,6 @@ -# TODO -- add trace/info/warn -- flesh out modoff parser error -- consider using Cow to reduce memory usage -- consider mixed case module names? +# TODO +- add trace/info/warn +- flesh out modoff parser error +- consider using Cow to reduce memory usage +- consider mixed case module names? - redo xml generation to not be hand-rolled \ No newline at end of file diff --git a/src/api-service/__app__/.gitignore b/src/api-service/__app__/.gitignore index 1614f3818..4b03dc74a 100644 --- a/src/api-service/__app__/.gitignore +++ b/src/api-service/__app__/.gitignore @@ -1,4 +1,4 @@ -.direnv -.python_packages -__pycache__ +.direnv +.python_packages +__pycache__ .venv \ No newline at end of file diff --git a/src/api-service/__app__/timer_retention/__init__.py b/src/api-service/__app__/timer_retention/__init__.py index 4f77a3574..fc0bfa031 100644 --- a/src/api-service/__app__/timer_retention/__init__.py +++ b/src/api-service/__app__/timer_retention/__init__.py @@ -1,74 +1,74 @@ -#!/usr/bin/env python -# -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -import datetime -import logging - -import azure.functions as func -from onefuzztypes.enums import JobState, TaskState - -from ..onefuzzlib.jobs import Job -from ..onefuzzlib.notifications.main import Notification -from ..onefuzzlib.repro import Repro -from ..onefuzzlib.tasks.main import Task - -RETENTION_POLICY = datetime.timedelta(days=(18 * 30)) -SEARCH_EXTENT = datetime.timedelta(days=(20 * 30)) - - -def main(mytimer: func.TimerRequest) -> None: # noqa: F841 - - now = datetime.datetime.now(tz=datetime.timezone.utc) - - time_retained_older = now - RETENTION_POLICY - time_retained_newer = now - SEARCH_EXTENT - - time_filter = ( - f"Timestamp lt datetime'{time_retained_older.isoformat()}' " - f"and Timestamp gt datetime'{time_retained_newer.isoformat()}'" - ) - time_filter_newer = f"Timestamp gt datetime'{time_retained_older.isoformat()}'" - - # Collecting 'still relevant' task containers. - # NOTE: This must be done before potentially modifying tasks otherwise - # the task timestamps will not be useful. - used_containers = set() - for task in Task.search(raw_unchecked_filter=time_filter_newer): - task_containers = {x.name for x in task.config.containers} - used_containers.update(task_containers) - - for notification in Notification.search(raw_unchecked_filter=time_filter): - logging.debug( - "checking expired notification for removal: %s", - notification.notification_id, - ) - container = notification.container - if container not in used_containers: - logging.info( - "deleting expired notification: %s", notification.notification_id - ) - notification.delete() - - for job in Job.search( - query={"state": [JobState.stopped]}, raw_unchecked_filter=time_filter - ): - if job.user_info is not None and job.user_info.upn is not None: - logging.info("removing PII from job: %s", job.job_id) - job.user_info.upn = None - job.save() - - for task in Task.search( - query={"state": [TaskState.stopped]}, raw_unchecked_filter=time_filter - ): - if task.user_info is not None and task.user_info.upn is not None: - logging.info("removing PII from task: %s", task.task_id) - task.user_info.upn = None - task.save() - - for repro in Repro.search(raw_unchecked_filter=time_filter): - if repro.user_info is not None and repro.user_info.upn is not None: - logging.info("removing PII from repro: %s", repro.vm_id) - repro.user_info.upn = None - repro.save() +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import datetime +import logging + +import azure.functions as func +from onefuzztypes.enums import JobState, TaskState + +from ..onefuzzlib.jobs import Job +from ..onefuzzlib.notifications.main import Notification +from ..onefuzzlib.repro import Repro +from ..onefuzzlib.tasks.main import Task + +RETENTION_POLICY = datetime.timedelta(days=(18 * 30)) +SEARCH_EXTENT = datetime.timedelta(days=(20 * 30)) + + +def main(mytimer: func.TimerRequest) -> None: # noqa: F841 + + now = datetime.datetime.now(tz=datetime.timezone.utc) + + time_retained_older = now - RETENTION_POLICY + time_retained_newer = now - SEARCH_EXTENT + + time_filter = ( + f"Timestamp lt datetime'{time_retained_older.isoformat()}' " + f"and Timestamp gt datetime'{time_retained_newer.isoformat()}'" + ) + time_filter_newer = f"Timestamp gt datetime'{time_retained_older.isoformat()}'" + + # Collecting 'still relevant' task containers. + # NOTE: This must be done before potentially modifying tasks otherwise + # the task timestamps will not be useful. + used_containers = set() + for task in Task.search(raw_unchecked_filter=time_filter_newer): + task_containers = {x.name for x in task.config.containers} + used_containers.update(task_containers) + + for notification in Notification.search(raw_unchecked_filter=time_filter): + logging.debug( + "checking expired notification for removal: %s", + notification.notification_id, + ) + container = notification.container + if container not in used_containers: + logging.info( + "deleting expired notification: %s", notification.notification_id + ) + notification.delete() + + for job in Job.search( + query={"state": [JobState.stopped]}, raw_unchecked_filter=time_filter + ): + if job.user_info is not None and job.user_info.upn is not None: + logging.info("removing PII from job: %s", job.job_id) + job.user_info.upn = None + job.save() + + for task in Task.search( + query={"state": [TaskState.stopped]}, raw_unchecked_filter=time_filter + ): + if task.user_info is not None and task.user_info.upn is not None: + logging.info("removing PII from task: %s", task.task_id) + task.user_info.upn = None + task.save() + + for repro in Repro.search(raw_unchecked_filter=time_filter): + if repro.user_info is not None and repro.user_info.upn is not None: + logging.info("removing PII from repro: %s", repro.vm_id) + repro.user_info.upn = None + repro.save() diff --git a/src/deployment/.gitignore b/src/deployment/.gitignore index 21ef600e0..5462469df 100644 --- a/src/deployment/.gitignore +++ b/src/deployment/.gitignore @@ -1,73 +1,73 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# pyenv -.python-version - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# mypy -.mypy_cache/ - -test-cov.xml +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ + +test-cov.xml test-output.xml \ No newline at end of file diff --git a/src/deployment/config.json b/src/deployment/config.json index 6b0dd8320..3b138fa5f 100644 --- a/src/deployment/config.json +++ b/src/deployment/config.json @@ -1,6 +1,6 @@ -{ - "proxy_nsg_config": { - "allowed_ips": ["*"], - "allowed_service_tags": [] - } +{ + "proxy_nsg_config": { + "allowed_ips": ["*"], + "allowed_service_tags": [] + } } \ No newline at end of file diff --git a/src/integration-tests/libfuzzer-dotnet/problems/problems.cs b/src/integration-tests/libfuzzer-dotnet/problems/problems.cs index 5170fbe14..fecf98306 100644 --- a/src/integration-tests/libfuzzer-dotnet/problems/problems.cs +++ b/src/integration-tests/libfuzzer-dotnet/problems/problems.cs @@ -1,15 +1,15 @@ -using System; -namespace Problems { - public static class Problems { - public static void Func(ReadOnlySpan data) { - var count = 0; - if (data.Length < 4) { - return; - } - if (data[0] == 0) { count++; } - if (data[1] == 1) { count++; } - if (data[2] == 2) { count++; } - if (count >= 3) { throw new Exception("this is bad"); } - } - } -} +using System; +namespace Problems { + public static class Problems { + public static void Func(ReadOnlySpan data) { + var count = 0; + if (data.Length < 4) { + return; + } + if (data[0] == 0) { count++; } + if (data[1] == 1) { count++; } + if (data[2] == 2) { count++; } + if (count >= 3) { throw new Exception("this is bad"); } + } + } +} diff --git a/src/integration-tests/libfuzzer-dotnet/problems/problems.csproj b/src/integration-tests/libfuzzer-dotnet/problems/problems.csproj index f51c9c8f5..d4c395e8c 100644 --- a/src/integration-tests/libfuzzer-dotnet/problems/problems.csproj +++ b/src/integration-tests/libfuzzer-dotnet/problems/problems.csproj @@ -1,7 +1,7 @@ - - - - netstandard2.1 - - - + + + + netstandard2.1 + + + diff --git a/src/integration-tests/libfuzzer-dotnet/wrapper/program.cs b/src/integration-tests/libfuzzer-dotnet/wrapper/program.cs index bf00c6d78..5405795de 100644 --- a/src/integration-tests/libfuzzer-dotnet/wrapper/program.cs +++ b/src/integration-tests/libfuzzer-dotnet/wrapper/program.cs @@ -1,8 +1,8 @@ -using SharpFuzz; -namespace Wrapper { - public class Program { - public static void Main(string[] args) { - Fuzzer.LibFuzzer.Run(stream => { Problems.Problems.Func(stream); }); - } - } -} +using SharpFuzz; +namespace Wrapper { + public class Program { + public static void Main(string[] args) { + Fuzzer.LibFuzzer.Run(stream => { Problems.Problems.Func(stream); }); + } + } +} diff --git a/src/integration-tests/libfuzzer-dotnet/wrapper/wrapper.csproj b/src/integration-tests/libfuzzer-dotnet/wrapper/wrapper.csproj index 6829261a9..063bd6a6e 100644 --- a/src/integration-tests/libfuzzer-dotnet/wrapper/wrapper.csproj +++ b/src/integration-tests/libfuzzer-dotnet/wrapper/wrapper.csproj @@ -1,15 +1,15 @@ - - - - - - - - - - - Exe - netcoreapp3.1 - - - + + + + + + + + + + + Exe + netcoreapp3.1 + + + diff --git a/src/integration-tests/trivial-crash/fuzz.c b/src/integration-tests/trivial-crash/fuzz.c index 3e5251f6a..f44e00061 100644 --- a/src/integration-tests/trivial-crash/fuzz.c +++ b/src/integration-tests/trivial-crash/fuzz.c @@ -1,85 +1,85 @@ -#include -#include -#include - -#ifdef _WIN64 -#include -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#define STDIN_FILENO _fileno(stdin) -#define read _read -#else -#include -#endif - -#define SIZE 8192 -#define BUF_SIZE 32 - -int check(const char *data, size_t len) -{ - char buf[BUF_SIZE]; - memset(buf, 0, BUF_SIZE); - strncpy(buf, data, len); // BUG - This incorrectly uses length of src, not dst - - // do something to ensure this isn't optimized away - int buflen = strlen(buf); - for (int i = 0; i <= ((buflen % 2 == 1) ? buflen - 1 : buflen) / 2; i++) - { - if (buf[i] != buf[buflen - 1 - i]) - { - printf("not palindrome: "); - printf(buf); - printf("\n"); - break; - } - } - - return 0; -} - -int from_stdin() -{ - char input[SIZE] = {0}; - int size = read(STDIN_FILENO, input, SIZE); - return check(input, size); -} - -int from_file(char *filename) -{ - FILE *infile; - char *buffer; - long length; - int result; - - infile = fopen(filename, "r"); - - if (infile == NULL) - return 1; - - fseek(infile, 0L, SEEK_END); - length = ftell(infile); - - fseek(infile, 0L, SEEK_SET); - buffer = calloc(length, sizeof(char)); - if (buffer == NULL) - return 1; - - length = fread(buffer, sizeof(char), length, infile); - fclose(infile); - - result = check(buffer, length); - free(buffer); - return result; -} - -int main(int argc, char **argv) -{ - - if (argc == 1) - { - return from_stdin(); - } - else if (argc > 1) - { - return from_file(argv[1]); - } -} +#include +#include +#include + +#ifdef _WIN64 +#include +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#define STDIN_FILENO _fileno(stdin) +#define read _read +#else +#include +#endif + +#define SIZE 8192 +#define BUF_SIZE 32 + +int check(const char *data, size_t len) +{ + char buf[BUF_SIZE]; + memset(buf, 0, BUF_SIZE); + strncpy(buf, data, len); // BUG - This incorrectly uses length of src, not dst + + // do something to ensure this isn't optimized away + int buflen = strlen(buf); + for (int i = 0; i <= ((buflen % 2 == 1) ? buflen - 1 : buflen) / 2; i++) + { + if (buf[i] != buf[buflen - 1 - i]) + { + printf("not palindrome: "); + printf(buf); + printf("\n"); + break; + } + } + + return 0; +} + +int from_stdin() +{ + char input[SIZE] = {0}; + int size = read(STDIN_FILENO, input, SIZE); + return check(input, size); +} + +int from_file(char *filename) +{ + FILE *infile; + char *buffer; + long length; + int result; + + infile = fopen(filename, "r"); + + if (infile == NULL) + return 1; + + fseek(infile, 0L, SEEK_END); + length = ftell(infile); + + fseek(infile, 0L, SEEK_SET); + buffer = calloc(length, sizeof(char)); + if (buffer == NULL) + return 1; + + length = fread(buffer, sizeof(char), length, infile); + fclose(infile); + + result = check(buffer, length); + free(buffer); + return result; +} + +int main(int argc, char **argv) +{ + + if (argc == 1) + { + return from_stdin(); + } + else if (argc > 1) + { + return from_file(argv[1]); + } +}