mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-13 18:48:09 +00:00
Set autocrlf by default, update affected files (#2261)
This commit is contained in:
George Pollard
GitHub
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
* text=auto
|
||||||
|
*.ps1 text=crlf
|
||||||
@ -1,75 +1,75 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# Copyright (c) Microsoft Corporation.
|
# Copyright (c) Microsoft Corporation.
|
||||||
# Licensed under the MIT License.
|
# Licensed under the MIT License.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from hashlib import sha512
|
from hashlib import sha512
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import azure.functions as func
|
import azure.functions as func
|
||||||
|
|
||||||
|
|
||||||
def code_block(data: str) -> str:
|
def code_block(data: str) -> str:
|
||||||
data = data.replace("`", "``")
|
data = data.replace("`", "``")
|
||||||
return "\n```\n%s\n```\n" % data
|
return "\n```\n%s\n```\n" % data
|
||||||
|
|
||||||
|
|
||||||
async def send_message(req: func.HttpRequest) -> bool:
|
async def send_message(req: func.HttpRequest) -> bool:
|
||||||
data = req.get_json()
|
data = req.get_json()
|
||||||
teams_url = os.environ.get("TEAMS_URL")
|
teams_url = os.environ.get("TEAMS_URL")
|
||||||
if teams_url is None:
|
if teams_url is None:
|
||||||
raise Exception("missing TEAMS_URL")
|
raise Exception("missing TEAMS_URL")
|
||||||
|
|
||||||
message: Dict[str, Any] = {
|
message: Dict[str, Any] = {
|
||||||
"@type": "MessageCard",
|
"@type": "MessageCard",
|
||||||
"@context": "https://schema.org/extensions",
|
"@context": "https://schema.org/extensions",
|
||||||
"summary": data["instance_name"],
|
"summary": data["instance_name"],
|
||||||
"sections": [
|
"sections": [
|
||||||
{
|
{
|
||||||
"facts": [
|
"facts": [
|
||||||
{"name": "instance", "value": data["instance_name"]},
|
{"name": "instance", "value": data["instance_name"]},
|
||||||
{"name": "event type", "value": data["event_type"]},
|
{"name": "event type", "value": data["event_type"]},
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{"text": code_block(json.dumps(data["event"], sort_keys=True))},
|
{"text": code_block(json.dumps(data["event"], sort_keys=True))},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
async with aiohttp.ClientSession() as client:
|
async with aiohttp.ClientSession() as client:
|
||||||
async with client.post(teams_url, json=message) as response:
|
async with client.post(teams_url, json=message) as response:
|
||||||
return response.ok
|
return response.ok
|
||||||
|
|
||||||
|
|
||||||
def verify(req: func.HttpRequest) -> bool:
|
def verify(req: func.HttpRequest) -> bool:
|
||||||
request_hmac = req.headers.get("X-Onefuzz-Digest")
|
request_hmac = req.headers.get("X-Onefuzz-Digest")
|
||||||
if request_hmac is None:
|
if request_hmac is None:
|
||||||
raise Exception("missing X-Onefuzz-Digest")
|
raise Exception("missing X-Onefuzz-Digest")
|
||||||
|
|
||||||
hmac_token = os.environ.get("HMAC_TOKEN")
|
hmac_token = os.environ.get("HMAC_TOKEN")
|
||||||
if hmac_token is None:
|
if hmac_token is None:
|
||||||
raise Exception("missing HMAC_TOKEN")
|
raise Exception("missing HMAC_TOKEN")
|
||||||
|
|
||||||
digest = hmac.new(
|
digest = hmac.new(
|
||||||
hmac_token.encode(), msg=req.get_body(), digestmod=sha512
|
hmac_token.encode(), msg=req.get_body(), digestmod=sha512
|
||||||
).hexdigest()
|
).hexdigest()
|
||||||
if digest != request_hmac:
|
if digest != request_hmac:
|
||||||
logging.error("invalid hmac")
|
logging.error("invalid hmac")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
async def main(req: func.HttpRequest) -> func.HttpResponse:
|
async def main(req: func.HttpRequest) -> func.HttpResponse:
|
||||||
if not verify(req):
|
if not verify(req):
|
||||||
return func.HttpResponse("no thanks")
|
return func.HttpResponse("no thanks")
|
||||||
|
|
||||||
if await send_message(req):
|
if await send_message(req):
|
||||||
return func.HttpResponse("unable to send message")
|
return func.HttpResponse("unable to send message")
|
||||||
|
|
||||||
return func.HttpResponse("thanks")
|
return func.HttpResponse("thanks")
|
||||||
|
|||||||
@ -1,40 +1,40 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Microsoft.Azure.Functions.Worker;
|
using Microsoft.Azure.Functions.Worker;
|
||||||
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service.Functions;
|
namespace Microsoft.OneFuzz.Service.Functions;
|
||||||
|
|
||||||
public class QueueProxyHearbeat {
|
public class QueueProxyHearbeat {
|
||||||
private readonly ILogTracer _log;
|
private readonly ILogTracer _log;
|
||||||
|
|
||||||
private readonly IProxyOperations _proxy;
|
private readonly IProxyOperations _proxy;
|
||||||
|
|
||||||
public QueueProxyHearbeat(ILogTracer log, IProxyOperations proxy) {
|
public QueueProxyHearbeat(ILogTracer log, IProxyOperations proxy) {
|
||||||
_log = log;
|
_log = log;
|
||||||
_proxy = proxy;
|
_proxy = proxy;
|
||||||
}
|
}
|
||||||
|
|
||||||
//[Function("QueueProxyHearbeat")]
|
//[Function("QueueProxyHearbeat")]
|
||||||
public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) {
|
public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) {
|
||||||
_log.Info($"heartbeat: {msg}");
|
_log.Info($"heartbeat: {msg}");
|
||||||
|
|
||||||
var hb = JsonSerializer.Deserialize<ProxyHeartbeat>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); ;
|
var hb = JsonSerializer.Deserialize<ProxyHeartbeat>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}"); ;
|
||||||
var newHb = hb with { TimeStamp = DateTimeOffset.UtcNow };
|
var newHb = hb with { TimeStamp = DateTimeOffset.UtcNow };
|
||||||
|
|
||||||
var proxy = await _proxy.GetByProxyId(newHb.ProxyId);
|
var proxy = await _proxy.GetByProxyId(newHb.ProxyId);
|
||||||
|
|
||||||
var log = _log.WithTag("ProxyId", newHb.ProxyId.ToString());
|
var log = _log.WithTag("ProxyId", newHb.ProxyId.ToString());
|
||||||
|
|
||||||
if (proxy == null) {
|
if (proxy == null) {
|
||||||
log.Warning($"invalid proxy id: {newHb.ProxyId}");
|
log.Warning($"invalid proxy id: {newHb.ProxyId}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var newProxy = proxy with { Heartbeat = newHb };
|
var newProxy = proxy with { Heartbeat = newHb };
|
||||||
|
|
||||||
var r = await _proxy.Replace(newProxy);
|
var r = await _proxy.Replace(newProxy);
|
||||||
if (!r.IsOk) {
|
if (!r.IsOk) {
|
||||||
var (status, reason) = r.ErrorV;
|
var (status, reason) = r.ErrorV;
|
||||||
log.Error($"Failed to replace proxy heartbeat record due to [{status}] {reason}");
|
log.Error($"Failed to replace proxy heartbeat record due to [{status}] {reason}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,24 +1,24 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Microsoft.Azure.Functions.Worker;
|
using Microsoft.Azure.Functions.Worker;
|
||||||
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service.Functions;
|
namespace Microsoft.OneFuzz.Service.Functions;
|
||||||
|
|
||||||
public class QueueWebhooks {
|
public class QueueWebhooks {
|
||||||
private readonly ILogTracer _log;
|
private readonly ILogTracer _log;
|
||||||
private readonly IWebhookMessageLogOperations _webhookMessageLog;
|
private readonly IWebhookMessageLogOperations _webhookMessageLog;
|
||||||
public QueueWebhooks(ILogTracer log, IWebhookMessageLogOperations webhookMessageLog) {
|
public QueueWebhooks(ILogTracer log, IWebhookMessageLogOperations webhookMessageLog) {
|
||||||
_log = log;
|
_log = log;
|
||||||
_webhookMessageLog = webhookMessageLog;
|
_webhookMessageLog = webhookMessageLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
//[Function("QueueWebhooks")]
|
//[Function("QueueWebhooks")]
|
||||||
public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) {
|
public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) {
|
||||||
|
|
||||||
_log.Info($"Webhook Message Queued: {msg}");
|
_log.Info($"Webhook Message Queued: {msg}");
|
||||||
|
|
||||||
var obj = JsonSerializer.Deserialize<WebhookMessageQueueObj>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");
|
var obj = JsonSerializer.Deserialize<WebhookMessageQueueObj>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");
|
||||||
|
|
||||||
await _webhookMessageLog.ProcessFromQueue(obj);
|
await _webhookMessageLog.ProcessFromQueue(obj);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,328 +1,328 @@
|
|||||||
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service;
|
namespace Microsoft.OneFuzz.Service;
|
||||||
|
|
||||||
[SerializeValue]
|
[SerializeValue]
|
||||||
public enum ErrorCode {
|
public enum ErrorCode {
|
||||||
INVALID_REQUEST = 450,
|
INVALID_REQUEST = 450,
|
||||||
INVALID_PERMISSION = 451,
|
INVALID_PERMISSION = 451,
|
||||||
MISSING_EULA_AGREEMENT = 452,
|
MISSING_EULA_AGREEMENT = 452,
|
||||||
INVALID_JOB = 453,
|
INVALID_JOB = 453,
|
||||||
INVALID_TASK = INVALID_JOB,
|
INVALID_TASK = INVALID_JOB,
|
||||||
UNABLE_TO_ADD_TASK_TO_JOB = 454,
|
UNABLE_TO_ADD_TASK_TO_JOB = 454,
|
||||||
INVALID_CONTAINER = 455,
|
INVALID_CONTAINER = 455,
|
||||||
UNABLE_TO_RESIZE = 456,
|
UNABLE_TO_RESIZE = 456,
|
||||||
UNAUTHORIZED = 457,
|
UNAUTHORIZED = 457,
|
||||||
UNABLE_TO_USE_STOPPED_JOB = 458,
|
UNABLE_TO_USE_STOPPED_JOB = 458,
|
||||||
UNABLE_TO_CHANGE_JOB_DURATION = 459,
|
UNABLE_TO_CHANGE_JOB_DURATION = 459,
|
||||||
UNABLE_TO_CREATE_NETWORK = 460,
|
UNABLE_TO_CREATE_NETWORK = 460,
|
||||||
VM_CREATE_FAILED = 461,
|
VM_CREATE_FAILED = 461,
|
||||||
MISSING_NOTIFICATION = 462,
|
MISSING_NOTIFICATION = 462,
|
||||||
INVALID_IMAGE = 463,
|
INVALID_IMAGE = 463,
|
||||||
UNABLE_TO_CREATE = 464,
|
UNABLE_TO_CREATE = 464,
|
||||||
UNABLE_TO_PORT_FORWARD = 465,
|
UNABLE_TO_PORT_FORWARD = 465,
|
||||||
UNABLE_TO_FIND = 467,
|
UNABLE_TO_FIND = 467,
|
||||||
TASK_FAILED = 468,
|
TASK_FAILED = 468,
|
||||||
INVALID_NODE = 469,
|
INVALID_NODE = 469,
|
||||||
NOTIFICATION_FAILURE = 470,
|
NOTIFICATION_FAILURE = 470,
|
||||||
UNABLE_TO_UPDATE = 471,
|
UNABLE_TO_UPDATE = 471,
|
||||||
PROXY_FAILED = 472,
|
PROXY_FAILED = 472,
|
||||||
INVALID_CONFIGURATION = 473,
|
INVALID_CONFIGURATION = 473,
|
||||||
UNABLE_TO_CREATE_CONTAINER = 474,
|
UNABLE_TO_CREATE_CONTAINER = 474,
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum VmState {
|
public enum VmState {
|
||||||
Init,
|
Init,
|
||||||
ExtensionsLaunch,
|
ExtensionsLaunch,
|
||||||
ExtensionsFailed,
|
ExtensionsFailed,
|
||||||
VmAllocationFailed,
|
VmAllocationFailed,
|
||||||
Running,
|
Running,
|
||||||
Stopping,
|
Stopping,
|
||||||
Stopped
|
Stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum WebhookMessageState {
|
public enum WebhookMessageState {
|
||||||
Queued,
|
Queued,
|
||||||
Retrying,
|
Retrying,
|
||||||
Succeeded,
|
Succeeded,
|
||||||
Failed
|
Failed
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum TaskState {
|
public enum TaskState {
|
||||||
Init,
|
Init,
|
||||||
Waiting,
|
Waiting,
|
||||||
Scheduled,
|
Scheduled,
|
||||||
SettingUp,
|
SettingUp,
|
||||||
Running,
|
Running,
|
||||||
Stopping,
|
Stopping,
|
||||||
Stopped,
|
Stopped,
|
||||||
WaitJob
|
WaitJob
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum TaskType {
|
public enum TaskType {
|
||||||
Coverage,
|
Coverage,
|
||||||
LibfuzzerFuzz,
|
LibfuzzerFuzz,
|
||||||
LibfuzzerCoverage,
|
LibfuzzerCoverage,
|
||||||
LibfuzzerCrashReport,
|
LibfuzzerCrashReport,
|
||||||
LibfuzzerMerge,
|
LibfuzzerMerge,
|
||||||
LibfuzzerRegression,
|
LibfuzzerRegression,
|
||||||
GenericAnalysis,
|
GenericAnalysis,
|
||||||
GenericSupervisor,
|
GenericSupervisor,
|
||||||
GenericMerge,
|
GenericMerge,
|
||||||
GenericGenerator,
|
GenericGenerator,
|
||||||
GenericCrashReport,
|
GenericCrashReport,
|
||||||
GenericRegression,
|
GenericRegression,
|
||||||
DotnetCoverage,
|
DotnetCoverage,
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum Os {
|
public enum Os {
|
||||||
Windows,
|
Windows,
|
||||||
Linux
|
Linux
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum ContainerType {
|
public enum ContainerType {
|
||||||
Analysis,
|
Analysis,
|
||||||
Coverage,
|
Coverage,
|
||||||
Crashes,
|
Crashes,
|
||||||
Inputs,
|
Inputs,
|
||||||
NoRepro,
|
NoRepro,
|
||||||
ReadonlyInputs,
|
ReadonlyInputs,
|
||||||
Reports,
|
Reports,
|
||||||
Setup,
|
Setup,
|
||||||
Tools,
|
Tools,
|
||||||
UniqueInputs,
|
UniqueInputs,
|
||||||
UniqueReports,
|
UniqueReports,
|
||||||
RegressionReports,
|
RegressionReports,
|
||||||
Logs
|
Logs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
[SkipRename]
|
[SkipRename]
|
||||||
public enum StatsFormat {
|
public enum StatsFormat {
|
||||||
AFL
|
AFL
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum TaskDebugFlag {
|
public enum TaskDebugFlag {
|
||||||
KeepNodeOnFailure,
|
KeepNodeOnFailure,
|
||||||
KeepNodeOnCompletion,
|
KeepNodeOnCompletion,
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum ScalesetState {
|
public enum ScalesetState {
|
||||||
Init,
|
Init,
|
||||||
Setup,
|
Setup,
|
||||||
Resize,
|
Resize,
|
||||||
Running,
|
Running,
|
||||||
Shutdown,
|
Shutdown,
|
||||||
Halt,
|
Halt,
|
||||||
CreationFailed
|
CreationFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public enum JobState {
|
public enum JobState {
|
||||||
Init,
|
Init,
|
||||||
Enabled,
|
Enabled,
|
||||||
Stopping,
|
Stopping,
|
||||||
Stopped
|
Stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class JobStateHelper {
|
public static class JobStateHelper {
|
||||||
private static readonly IReadOnlySet<JobState> _shuttingDown = new HashSet<JobState>(new[] { JobState.Stopping, JobState.Stopped });
|
private static readonly IReadOnlySet<JobState> _shuttingDown = new HashSet<JobState>(new[] { JobState.Stopping, JobState.Stopped });
|
||||||
private static readonly IReadOnlySet<JobState> _avaiable = new HashSet<JobState>(new[] { JobState.Init, JobState.Enabled });
|
private static readonly IReadOnlySet<JobState> _avaiable = new HashSet<JobState>(new[] { JobState.Init, JobState.Enabled });
|
||||||
private static readonly IReadOnlySet<JobState> _needsWork = new HashSet<JobState>(new[] { JobState.Init, JobState.Stopping });
|
private static readonly IReadOnlySet<JobState> _needsWork = new HashSet<JobState>(new[] { JobState.Init, JobState.Stopping });
|
||||||
|
|
||||||
public static IReadOnlySet<JobState> Available => _avaiable;
|
public static IReadOnlySet<JobState> Available => _avaiable;
|
||||||
public static IReadOnlySet<JobState> NeedsWork => _needsWork;
|
public static IReadOnlySet<JobState> NeedsWork => _needsWork;
|
||||||
public static IReadOnlySet<JobState> ShuttingDown => _shuttingDown;
|
public static IReadOnlySet<JobState> ShuttingDown => _shuttingDown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static class ScalesetStateHelper {
|
public static class ScalesetStateHelper {
|
||||||
private static readonly IReadOnlySet<ScalesetState> _canUpdate = new HashSet<ScalesetState> { ScalesetState.Init, ScalesetState.Resize };
|
private static readonly IReadOnlySet<ScalesetState> _canUpdate = new HashSet<ScalesetState> { ScalesetState.Init, ScalesetState.Resize };
|
||||||
private static readonly IReadOnlySet<ScalesetState> _needsWork =
|
private static readonly IReadOnlySet<ScalesetState> _needsWork =
|
||||||
new HashSet<ScalesetState>{
|
new HashSet<ScalesetState>{
|
||||||
ScalesetState.Init,
|
ScalesetState.Init,
|
||||||
ScalesetState.Setup,
|
ScalesetState.Setup,
|
||||||
ScalesetState.Resize,
|
ScalesetState.Resize,
|
||||||
ScalesetState.Shutdown,
|
ScalesetState.Shutdown,
|
||||||
ScalesetState.Halt
|
ScalesetState.Halt
|
||||||
};
|
};
|
||||||
private static readonly IReadOnlySet<ScalesetState> _available = new HashSet<ScalesetState> { ScalesetState.Resize, ScalesetState.Running };
|
private static readonly IReadOnlySet<ScalesetState> _available = new HashSet<ScalesetState> { ScalesetState.Resize, ScalesetState.Running };
|
||||||
private static readonly IReadOnlySet<ScalesetState> _resizing = new HashSet<ScalesetState> { ScalesetState.Halt, ScalesetState.Init, ScalesetState.Setup };
|
private static readonly IReadOnlySet<ScalesetState> _resizing = new HashSet<ScalesetState> { ScalesetState.Halt, ScalesetState.Init, ScalesetState.Setup };
|
||||||
|
|
||||||
/// set of states that indicate the scaleset can be updated
|
/// set of states that indicate the scaleset can be updated
|
||||||
public static IReadOnlySet<ScalesetState> CanUpdate => _canUpdate;
|
public static IReadOnlySet<ScalesetState> CanUpdate => _canUpdate;
|
||||||
|
|
||||||
/// set of states that indicate work is needed during eventing
|
/// set of states that indicate work is needed during eventing
|
||||||
public static IReadOnlySet<ScalesetState> NeedsWork => _needsWork;
|
public static IReadOnlySet<ScalesetState> NeedsWork => _needsWork;
|
||||||
|
|
||||||
/// set of states that indicate if it's available for work
|
/// set of states that indicate if it's available for work
|
||||||
public static IReadOnlySet<ScalesetState> Available => _available;
|
public static IReadOnlySet<ScalesetState> Available => _available;
|
||||||
|
|
||||||
/// set of states that indicate scaleset is resizing
|
/// set of states that indicate scaleset is resizing
|
||||||
public static IReadOnlySet<ScalesetState> Resizing => _resizing;
|
public static IReadOnlySet<ScalesetState> Resizing => _resizing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static class VmStateHelper {
|
public static class VmStateHelper {
|
||||||
|
|
||||||
private static readonly IReadOnlySet<VmState> _needsWork = new HashSet<VmState> { VmState.Init, VmState.ExtensionsLaunch, VmState.Stopping };
|
private static readonly IReadOnlySet<VmState> _needsWork = new HashSet<VmState> { VmState.Init, VmState.ExtensionsLaunch, VmState.Stopping };
|
||||||
private static readonly IReadOnlySet<VmState> _available = new HashSet<VmState> { VmState.Init, VmState.ExtensionsLaunch, VmState.ExtensionsFailed, VmState.VmAllocationFailed, VmState.Running, };
|
private static readonly IReadOnlySet<VmState> _available = new HashSet<VmState> { VmState.Init, VmState.ExtensionsLaunch, VmState.ExtensionsFailed, VmState.VmAllocationFailed, VmState.Running, };
|
||||||
|
|
||||||
public static IReadOnlySet<VmState> NeedsWork => _needsWork;
|
public static IReadOnlySet<VmState> NeedsWork => _needsWork;
|
||||||
public static IReadOnlySet<VmState> Available => _available;
|
public static IReadOnlySet<VmState> Available => _available;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class TaskStateHelper {
|
public static class TaskStateHelper {
|
||||||
|
|
||||||
public static readonly IReadOnlySet<TaskState> AvailableStates =
|
public static readonly IReadOnlySet<TaskState> AvailableStates =
|
||||||
new HashSet<TaskState> { TaskState.Waiting, TaskState.Scheduled, TaskState.SettingUp, TaskState.Running, TaskState.WaitJob };
|
new HashSet<TaskState> { TaskState.Waiting, TaskState.Scheduled, TaskState.SettingUp, TaskState.Running, TaskState.WaitJob };
|
||||||
|
|
||||||
public static readonly IReadOnlySet<TaskState> NeedsWorkStates =
|
public static readonly IReadOnlySet<TaskState> NeedsWorkStates =
|
||||||
new HashSet<TaskState> { TaskState.Init, TaskState.Stopping };
|
new HashSet<TaskState> { TaskState.Init, TaskState.Stopping };
|
||||||
|
|
||||||
public static readonly IReadOnlySet<TaskState> ShuttingDownStates =
|
public static readonly IReadOnlySet<TaskState> ShuttingDownStates =
|
||||||
new HashSet<TaskState> { TaskState.Stopping, TaskState.Stopped };
|
new HashSet<TaskState> { TaskState.Stopping, TaskState.Stopped };
|
||||||
|
|
||||||
public static readonly IReadOnlySet<TaskState> HasStartedStates =
|
public static readonly IReadOnlySet<TaskState> HasStartedStates =
|
||||||
new HashSet<TaskState> { TaskState.Running, TaskState.Stopping, TaskState.Stopped };
|
new HashSet<TaskState> { TaskState.Running, TaskState.Stopping, TaskState.Stopped };
|
||||||
|
|
||||||
public static bool Available(this TaskState state) => AvailableStates.Contains(state);
|
public static bool Available(this TaskState state) => AvailableStates.Contains(state);
|
||||||
|
|
||||||
public static bool NeedsWork(this TaskState state) => NeedsWorkStates.Contains(state);
|
public static bool NeedsWork(this TaskState state) => NeedsWorkStates.Contains(state);
|
||||||
|
|
||||||
public static bool ShuttingDown(this TaskState state) => ShuttingDownStates.Contains(state);
|
public static bool ShuttingDown(this TaskState state) => ShuttingDownStates.Contains(state);
|
||||||
|
|
||||||
public static bool HasStarted(this TaskState state) => HasStartedStates.Contains(state);
|
public static bool HasStarted(this TaskState state) => HasStartedStates.Contains(state);
|
||||||
|
|
||||||
}
|
}
|
||||||
public enum PoolState {
|
public enum PoolState {
|
||||||
Init,
|
Init,
|
||||||
Running,
|
Running,
|
||||||
Shutdown,
|
Shutdown,
|
||||||
Halt
|
Halt
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class PoolStateHelper {
|
public static class PoolStateHelper {
|
||||||
private static readonly IReadOnlySet<PoolState> _needsWork = new HashSet<PoolState> { PoolState.Init, PoolState.Shutdown, PoolState.Halt };
|
private static readonly IReadOnlySet<PoolState> _needsWork = new HashSet<PoolState> { PoolState.Init, PoolState.Shutdown, PoolState.Halt };
|
||||||
private static readonly IReadOnlySet<PoolState> _available = new HashSet<PoolState> { PoolState.Running };
|
private static readonly IReadOnlySet<PoolState> _available = new HashSet<PoolState> { PoolState.Running };
|
||||||
|
|
||||||
public static IReadOnlySet<PoolState> NeedsWork => _needsWork;
|
public static IReadOnlySet<PoolState> NeedsWork => _needsWork;
|
||||||
public static IReadOnlySet<PoolState> Available => _available;
|
public static IReadOnlySet<PoolState> Available => _available;
|
||||||
}
|
}
|
||||||
|
|
||||||
[SkipRename]
|
[SkipRename]
|
||||||
public enum Architecture {
|
public enum Architecture {
|
||||||
x86_64
|
x86_64
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum TaskFeature {
|
public enum TaskFeature {
|
||||||
InputQueueFromContainer,
|
InputQueueFromContainer,
|
||||||
SupervisorExe,
|
SupervisorExe,
|
||||||
SupervisorEnv,
|
SupervisorEnv,
|
||||||
SupervisorOptions,
|
SupervisorOptions,
|
||||||
SupervisorInputMarker,
|
SupervisorInputMarker,
|
||||||
StatsFile,
|
StatsFile,
|
||||||
StatsFormat,
|
StatsFormat,
|
||||||
TargetExe,
|
TargetExe,
|
||||||
TargetExeOptional,
|
TargetExeOptional,
|
||||||
TargetEnv,
|
TargetEnv,
|
||||||
TargetOptions,
|
TargetOptions,
|
||||||
AnalyzerExe,
|
AnalyzerExe,
|
||||||
AnalyzerEnv,
|
AnalyzerEnv,
|
||||||
AnalyzerOptions,
|
AnalyzerOptions,
|
||||||
RenameOutput,
|
RenameOutput,
|
||||||
TargetOptionsMerge,
|
TargetOptionsMerge,
|
||||||
TargetWorkers,
|
TargetWorkers,
|
||||||
GeneratorExe,
|
GeneratorExe,
|
||||||
GeneratorEnv,
|
GeneratorEnv,
|
||||||
GeneratorOptions,
|
GeneratorOptions,
|
||||||
WaitForFiles,
|
WaitForFiles,
|
||||||
TargetTimeout,
|
TargetTimeout,
|
||||||
CheckAsanLog,
|
CheckAsanLog,
|
||||||
CheckDebugger,
|
CheckDebugger,
|
||||||
CheckRetryCount,
|
CheckRetryCount,
|
||||||
EnsembleSyncDelay,
|
EnsembleSyncDelay,
|
||||||
PreserveExistingOutputs,
|
PreserveExistingOutputs,
|
||||||
CheckFuzzerHelp,
|
CheckFuzzerHelp,
|
||||||
ExpectCrashOnFailure,
|
ExpectCrashOnFailure,
|
||||||
ReportList,
|
ReportList,
|
||||||
MinimizedStackDepth,
|
MinimizedStackDepth,
|
||||||
CoverageFilter,
|
CoverageFilter,
|
||||||
TargetMustUseInput
|
TargetMustUseInput
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
[Flags]
|
[Flags]
|
||||||
public enum ContainerPermission {
|
public enum ContainerPermission {
|
||||||
Read = 1 << 0,
|
Read = 1 << 0,
|
||||||
Write = 1 << 1,
|
Write = 1 << 1,
|
||||||
List = 1 << 2,
|
List = 1 << 2,
|
||||||
Delete = 1 << 3,
|
Delete = 1 << 3,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public enum Compare {
|
public enum Compare {
|
||||||
Equal,
|
Equal,
|
||||||
AtLeast,
|
AtLeast,
|
||||||
AtMost
|
AtMost
|
||||||
}
|
}
|
||||||
public enum AgentMode {
|
public enum AgentMode {
|
||||||
Fuzz,
|
Fuzz,
|
||||||
Repro,
|
Repro,
|
||||||
Proxy
|
Proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum NodeState {
|
public enum NodeState {
|
||||||
Init,
|
Init,
|
||||||
Free,
|
Free,
|
||||||
SettingUp,
|
SettingUp,
|
||||||
Rebooting,
|
Rebooting,
|
||||||
Ready,
|
Ready,
|
||||||
Busy,
|
Busy,
|
||||||
Done,
|
Done,
|
||||||
Shutdown,
|
Shutdown,
|
||||||
Halt,
|
Halt,
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class NodeStateHelper {
|
public static class NodeStateHelper {
|
||||||
private static readonly IReadOnlySet<NodeState> _needsWork =
|
private static readonly IReadOnlySet<NodeState> _needsWork =
|
||||||
new HashSet<NodeState>(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt });
|
new HashSet<NodeState>(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt });
|
||||||
|
|
||||||
private static readonly IReadOnlySet<NodeState> _readyForReset
|
private static readonly IReadOnlySet<NodeState> _readyForReset
|
||||||
= new HashSet<NodeState>(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt });
|
= new HashSet<NodeState>(new[] { NodeState.Done, NodeState.Shutdown, NodeState.Halt });
|
||||||
|
|
||||||
private static readonly IReadOnlySet<NodeState> _canProcessNewWork =
|
private static readonly IReadOnlySet<NodeState> _canProcessNewWork =
|
||||||
new HashSet<NodeState>(new[] { NodeState.Free });
|
new HashSet<NodeState>(new[] { NodeState.Free });
|
||||||
|
|
||||||
private static readonly IReadOnlySet<NodeState> _busy =
|
private static readonly IReadOnlySet<NodeState> _busy =
|
||||||
new HashSet<NodeState>(new[] { NodeState.Busy });
|
new HashSet<NodeState>(new[] { NodeState.Busy });
|
||||||
|
|
||||||
public static IReadOnlySet<NodeState> BusyStates => _busy;
|
public static IReadOnlySet<NodeState> BusyStates => _busy;
|
||||||
|
|
||||||
public static IReadOnlySet<NodeState> NeedsWorkStates => _needsWork;
|
public static IReadOnlySet<NodeState> NeedsWorkStates => _needsWork;
|
||||||
|
|
||||||
public static bool NeedsWork(this NodeState state) => _needsWork.Contains(state);
|
public static bool NeedsWork(this NodeState state) => _needsWork.Contains(state);
|
||||||
|
|
||||||
///If Node is in one of these states, ignore updates from the agent.
|
///If Node is in one of these states, ignore updates from the agent.
|
||||||
public static bool ReadyForReset(this NodeState state) => _readyForReset.Contains(state);
|
public static bool ReadyForReset(this NodeState state) => _readyForReset.Contains(state);
|
||||||
|
|
||||||
public static bool CanProcessNewWork(this NodeState state) => _canProcessNewWork.Contains(state);
|
public static bool CanProcessNewWork(this NodeState state) => _canProcessNewWork.Contains(state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public enum NodeDisposalStrategy {
|
public enum NodeDisposalStrategy {
|
||||||
ScaleIn,
|
ScaleIn,
|
||||||
Decomission
|
Decomission
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public enum GithubIssueState {
|
public enum GithubIssueState {
|
||||||
Open,
|
Open,
|
||||||
Closed
|
Closed
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum GithubIssueSearchMatch {
|
public enum GithubIssueSearchMatch {
|
||||||
Title,
|
Title,
|
||||||
Body
|
Body
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,135 +1,135 @@
|
|||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using ApiService.OneFuzzLib.Orm;
|
using ApiService.OneFuzzLib.Orm;
|
||||||
using Azure.Storage.Sas;
|
using Azure.Storage.Sas;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service;
|
namespace Microsoft.OneFuzz.Service;
|
||||||
|
|
||||||
public interface IProxyOperations : IStatefulOrm<Proxy, VmState> {
|
public interface IProxyOperations : IStatefulOrm<Proxy, VmState> {
|
||||||
Task<Proxy?> GetByProxyId(Guid proxyId);
|
Task<Proxy?> GetByProxyId(Guid proxyId);
|
||||||
|
|
||||||
Async.Task SetState(Proxy proxy, VmState state);
|
Async.Task SetState(Proxy proxy, VmState state);
|
||||||
bool IsAlive(Proxy proxy);
|
bool IsAlive(Proxy proxy);
|
||||||
Async.Task SaveProxyConfig(Proxy proxy);
|
Async.Task SaveProxyConfig(Proxy proxy);
|
||||||
bool IsOutdated(Proxy proxy);
|
bool IsOutdated(Proxy proxy);
|
||||||
Async.Task<Proxy?> GetOrCreate(string region);
|
Async.Task<Proxy?> GetOrCreate(string region);
|
||||||
|
|
||||||
}
|
}
|
||||||
public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IProxyOperations {
|
public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IProxyOperations {
|
||||||
|
|
||||||
|
|
||||||
static TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7);
|
static TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7);
|
||||||
|
|
||||||
public ProxyOperations(ILogTracer log, IOnefuzzContext context)
|
public ProxyOperations(ILogTracer log, IOnefuzzContext context)
|
||||||
: base(log.WithTag("Component", "scaleset-proxy"), context) {
|
: base(log.WithTag("Component", "scaleset-proxy"), context) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public async Task<Proxy?> GetByProxyId(Guid proxyId) {
|
public async Task<Proxy?> GetByProxyId(Guid proxyId) {
|
||||||
|
|
||||||
var data = QueryAsync(filter: $"RowKey eq '{proxyId}'");
|
var data = QueryAsync(filter: $"RowKey eq '{proxyId}'");
|
||||||
|
|
||||||
return await data.FirstOrDefaultAsync();
|
return await data.FirstOrDefaultAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task<Proxy?> GetOrCreate(string region) {
|
public async Async.Task<Proxy?> GetOrCreate(string region) {
|
||||||
var proxyList = QueryAsync(filter: $"region eq '{region}' and outdated eq false");
|
var proxyList = QueryAsync(filter: $"region eq '{region}' and outdated eq false");
|
||||||
|
|
||||||
await foreach (var proxy in proxyList) {
|
await foreach (var proxy in proxyList) {
|
||||||
if (IsOutdated(proxy)) {
|
if (IsOutdated(proxy)) {
|
||||||
await Replace(proxy with { Outdated = true });
|
await Replace(proxy with { Outdated = true });
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!VmStateHelper.Available.Contains(proxy.State)) {
|
if (!VmStateHelper.Available.Contains(proxy.State)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
return proxy;
|
return proxy;
|
||||||
}
|
}
|
||||||
|
|
||||||
_logTracer.Info($"creating proxy: region:{region}");
|
_logTracer.Info($"creating proxy: region:{region}");
|
||||||
var newProxy = new Proxy(region, Guid.NewGuid(), DateTimeOffset.UtcNow, VmState.Init, Auth.BuildAuth(), null, null, _context.ServiceConfiguration.OneFuzzVersion, null, false);
|
var newProxy = new Proxy(region, Guid.NewGuid(), DateTimeOffset.UtcNow, VmState.Init, Auth.BuildAuth(), null, null, _context.ServiceConfiguration.OneFuzzVersion, null, false);
|
||||||
|
|
||||||
await Replace(newProxy);
|
await Replace(newProxy);
|
||||||
await _context.Events.SendEvent(new EventProxyCreated(region, newProxy.ProxyId));
|
await _context.Events.SendEvent(new EventProxyCreated(region, newProxy.ProxyId));
|
||||||
return newProxy;
|
return newProxy;
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool IsAlive(Proxy proxy) {
|
public bool IsAlive(Proxy proxy) {
|
||||||
var tenMinutesAgo = DateTimeOffset.UtcNow - TimeSpan.FromMinutes(10);
|
var tenMinutesAgo = DateTimeOffset.UtcNow - TimeSpan.FromMinutes(10);
|
||||||
|
|
||||||
if (proxy.Heartbeat != null && proxy.Heartbeat.TimeStamp < tenMinutesAgo) {
|
if (proxy.Heartbeat != null && proxy.Heartbeat.TimeStamp < tenMinutesAgo) {
|
||||||
_logTracer.Info($"last heartbeat is more than an 10 minutes old: {proxy.Region} - last heartbeat:{proxy.Heartbeat} compared_to:{tenMinutesAgo}");
|
_logTracer.Info($"last heartbeat is more than an 10 minutes old: {proxy.Region} - last heartbeat:{proxy.Heartbeat} compared_to:{tenMinutesAgo}");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (proxy.Heartbeat != null && proxy.TimeStamp != null && proxy.TimeStamp < tenMinutesAgo) {
|
if (proxy.Heartbeat != null && proxy.TimeStamp != null && proxy.TimeStamp < tenMinutesAgo) {
|
||||||
_logTracer.Error($"no heartbeat in the last 10 minutes: {proxy.Region} timestamp: {proxy.TimeStamp} compared_to:{tenMinutesAgo}");
|
_logTracer.Error($"no heartbeat in the last 10 minutes: {proxy.Region} timestamp: {proxy.TimeStamp} compared_to:{tenMinutesAgo}");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool IsOutdated(Proxy proxy) {
|
public bool IsOutdated(Proxy proxy) {
|
||||||
if (!VmStateHelper.Available.Contains(proxy.State)) {
|
if (!VmStateHelper.Available.Contains(proxy.State)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (proxy.Version != _context.ServiceConfiguration.OneFuzzVersion) {
|
if (proxy.Version != _context.ServiceConfiguration.OneFuzzVersion) {
|
||||||
_logTracer.Info($"mismatch version: proxy:{proxy.Version} service:{_context.ServiceConfiguration.OneFuzzVersion} state:{proxy.State}");
|
_logTracer.Info($"mismatch version: proxy:{proxy.Version} service:{_context.ServiceConfiguration.OneFuzzVersion} state:{proxy.State}");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (proxy.CreatedTimestamp != null) {
|
if (proxy.CreatedTimestamp != null) {
|
||||||
if (proxy.CreatedTimestamp < (DateTimeOffset.UtcNow - PROXY_LIFESPAN)) {
|
if (proxy.CreatedTimestamp < (DateTimeOffset.UtcNow - PROXY_LIFESPAN)) {
|
||||||
_logTracer.Info($"proxy older than 7 days:proxy-created:{proxy.CreatedTimestamp} state:{proxy.State}");
|
_logTracer.Info($"proxy older than 7 days:proxy-created:{proxy.CreatedTimestamp} state:{proxy.State}");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task SaveProxyConfig(Proxy proxy) {
|
public async Async.Task SaveProxyConfig(Proxy proxy) {
|
||||||
var forwards = await GetForwards(proxy);
|
var forwards = await GetForwards(proxy);
|
||||||
var url = (await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas");
|
var url = (await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas");
|
||||||
var queueSas = await _context.Queue.GetQueueSas("proxy", StorageType.Config, QueueSasPermissions.Add).EnsureNotNull("can't generate queue sas") ?? throw new Exception("Queue sas is null");
|
var queueSas = await _context.Queue.GetQueueSas("proxy", StorageType.Config, QueueSasPermissions.Add).EnsureNotNull("can't generate queue sas") ?? throw new Exception("Queue sas is null");
|
||||||
|
|
||||||
var proxyConfig = new ProxyConfig(
|
var proxyConfig = new ProxyConfig(
|
||||||
Url: url,
|
Url: url,
|
||||||
Notification: queueSas,
|
Notification: queueSas,
|
||||||
Region: proxy.Region,
|
Region: proxy.Region,
|
||||||
ProxyId: proxy.ProxyId,
|
ProxyId: proxy.ProxyId,
|
||||||
Forwards: forwards,
|
Forwards: forwards,
|
||||||
InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey.EnsureNotNull("missing InstrumentationKey"),
|
InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey.EnsureNotNull("missing InstrumentationKey"),
|
||||||
MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry.EnsureNotNull("missing Telemetry"),
|
MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry.EnsureNotNull("missing Telemetry"),
|
||||||
InstanceId: await _context.Containers.GetInstanceId());
|
InstanceId: await _context.Containers.GetInstanceId());
|
||||||
|
|
||||||
await _context.Containers.SaveBlob(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", _entityConverter.ToJsonString(proxyConfig), StorageType.Config);
|
await _context.Containers.SaveBlob(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", _entityConverter.ToJsonString(proxyConfig), StorageType.Config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public async Async.Task SetState(Proxy proxy, VmState state) {
|
public async Async.Task SetState(Proxy proxy, VmState state) {
|
||||||
if (proxy.State == state) {
|
if (proxy.State == state) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await Replace(proxy with { State = state });
|
await Replace(proxy with { State = state });
|
||||||
|
|
||||||
await _context.Events.SendEvent(new EventProxyStateUpdated(proxy.Region, proxy.ProxyId, proxy.State));
|
await _context.Events.SendEvent(new EventProxyStateUpdated(proxy.Region, proxy.ProxyId, proxy.State));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public async Async.Task<List<Forward>> GetForwards(Proxy proxy) {
|
public async Async.Task<List<Forward>> GetForwards(Proxy proxy) {
|
||||||
var forwards = new List<Forward>();
|
var forwards = new List<Forward>();
|
||||||
|
|
||||||
await foreach (var entry in _context.ProxyForwardOperations.SearchForward(region: proxy.Region, proxyId: proxy.ProxyId)) {
|
await foreach (var entry in _context.ProxyForwardOperations.SearchForward(region: proxy.Region, proxyId: proxy.ProxyId)) {
|
||||||
if (entry.EndTime < DateTimeOffset.UtcNow) {
|
if (entry.EndTime < DateTimeOffset.UtcNow) {
|
||||||
await _context.ProxyForwardOperations.Delete(entry);
|
await _context.ProxyForwardOperations.Delete(entry);
|
||||||
} else {
|
} else {
|
||||||
forwards.Add(new Forward(entry.Port, entry.DstPort, entry.DstIp));
|
forwards.Add(new Forward(entry.Port, entry.DstPort, entry.DstIp));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return forwards;
|
return forwards;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,253 +1,253 @@
|
|||||||
using System.Net;
|
using System.Net;
|
||||||
using System.Net.Http;
|
using System.Net.Http;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using ApiService.OneFuzzLib.Orm;
|
using ApiService.OneFuzzLib.Orm;
|
||||||
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service;
|
namespace Microsoft.OneFuzz.Service;
|
||||||
|
|
||||||
public interface IWebhookOperations : IOrm<Webhook> {
|
public interface IWebhookOperations : IOrm<Webhook> {
|
||||||
Async.Task SendEvent(EventMessage eventMessage);
|
Async.Task SendEvent(EventMessage eventMessage);
|
||||||
Async.Task<Webhook?> GetByWebhookId(Guid webhookId);
|
Async.Task<Webhook?> GetByWebhookId(Guid webhookId);
|
||||||
Async.Task<bool> Send(WebhookMessageLog messageLog);
|
Async.Task<bool> Send(WebhookMessageLog messageLog);
|
||||||
Task<EventPing> Ping(Webhook webhook);
|
Task<EventPing> Ping(Webhook webhook);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class WebhookOperations : Orm<Webhook>, IWebhookOperations {
|
public class WebhookOperations : Orm<Webhook>, IWebhookOperations {
|
||||||
|
|
||||||
private readonly IHttpClientFactory _httpFactory;
|
private readonly IHttpClientFactory _httpFactory;
|
||||||
|
|
||||||
public WebhookOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context)
|
public WebhookOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context)
|
||||||
: base(log, context) {
|
: base(log, context) {
|
||||||
_httpFactory = httpFactory;
|
_httpFactory = httpFactory;
|
||||||
}
|
}
|
||||||
|
|
||||||
async public Async.Task SendEvent(EventMessage eventMessage) {
|
async public Async.Task SendEvent(EventMessage eventMessage) {
|
||||||
await foreach (var webhook in GetWebhooksCached()) {
|
await foreach (var webhook in GetWebhooksCached()) {
|
||||||
if (!webhook.EventTypes.Contains(eventMessage.EventType)) {
|
if (!webhook.EventTypes.Contains(eventMessage.EventType)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
await AddEvent(webhook, eventMessage);
|
await AddEvent(webhook, eventMessage);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {
|
async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {
|
||||||
var message = new WebhookMessageLog(
|
var message = new WebhookMessageLog(
|
||||||
EventId: eventMessage.EventId,
|
EventId: eventMessage.EventId,
|
||||||
EventType: eventMessage.EventType,
|
EventType: eventMessage.EventType,
|
||||||
Event: eventMessage.Event,
|
Event: eventMessage.Event,
|
||||||
InstanceId: eventMessage.InstanceId,
|
InstanceId: eventMessage.InstanceId,
|
||||||
InstanceName: eventMessage.InstanceName,
|
InstanceName: eventMessage.InstanceName,
|
||||||
WebhookId: webhook.WebhookId,
|
WebhookId: webhook.WebhookId,
|
||||||
TryCount: 0
|
TryCount: 0
|
||||||
);
|
);
|
||||||
|
|
||||||
var r = await _context.WebhookMessageLogOperations.Replace(message);
|
var r = await _context.WebhookMessageLogOperations.Replace(message);
|
||||||
if (!r.IsOk) {
|
if (!r.IsOk) {
|
||||||
var (status, reason) = r.ErrorV;
|
var (status, reason) = r.ErrorV;
|
||||||
_logTracer.Error($"Failed to replace webhook message log due to [{status}] {reason}");
|
_logTracer.Error($"Failed to replace webhook message log due to [{status}] {reason}");
|
||||||
}
|
}
|
||||||
await _context.WebhookMessageLogOperations.QueueWebhook(message);
|
await _context.WebhookMessageLogOperations.QueueWebhook(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task<bool> Send(WebhookMessageLog messageLog) {
|
public async Async.Task<bool> Send(WebhookMessageLog messageLog) {
|
||||||
var webhook = await GetByWebhookId(messageLog.WebhookId);
|
var webhook = await GetByWebhookId(messageLog.WebhookId);
|
||||||
if (webhook == null || webhook.Url == null) {
|
if (webhook == null || webhook.Url == null) {
|
||||||
throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found");
|
throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found");
|
||||||
}
|
}
|
||||||
|
|
||||||
var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);
|
var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);
|
||||||
|
|
||||||
var headers = new Dictionary<string, string> { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } };
|
var headers = new Dictionary<string, string> { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } };
|
||||||
|
|
||||||
if (digest != null) {
|
if (digest != null) {
|
||||||
headers["X-Onefuzz-Digest"] = digest;
|
headers["X-Onefuzz-Digest"] = digest;
|
||||||
}
|
}
|
||||||
|
|
||||||
var client = new Request(_httpFactory.CreateClient());
|
var client = new Request(_httpFactory.CreateClient());
|
||||||
_logTracer.Info(data);
|
_logTracer.Info(data);
|
||||||
var response = client.Post(url: webhook.Url, json: data, headers: headers);
|
var response = client.Post(url: webhook.Url, json: data, headers: headers);
|
||||||
var result = response.Result;
|
var result = response.Result;
|
||||||
if (result.StatusCode == HttpStatusCode.Accepted) {
|
if (result.StatusCode == HttpStatusCode.Accepted) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<EventPing> Ping(Webhook webhook) {
|
public async Task<EventPing> Ping(Webhook webhook) {
|
||||||
var ping = new EventPing(Guid.NewGuid());
|
var ping = new EventPing(Guid.NewGuid());
|
||||||
var instanceId = await _context.Containers.GetInstanceId();
|
var instanceId = await _context.Containers.GetInstanceId();
|
||||||
var instanceName = _context.Creds.GetInstanceName();
|
var instanceName = _context.Creds.GetInstanceName();
|
||||||
await AddEvent(webhook, new EventMessage(Guid.NewGuid(), EventType.Ping, ping, instanceId, instanceName));
|
await AddEvent(webhook, new EventMessage(Guid.NewGuid(), EventType.Ping, ping, instanceId, instanceName));
|
||||||
return ping;
|
return ping;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Not converting to bytes, as it's not neccessary in C#. Just keeping as string.
|
// Not converting to bytes, as it's not neccessary in C#. Just keeping as string.
|
||||||
public async Async.Task<Tuple<string, string?>> BuildMessage(Guid webhookId, Guid eventId, EventType eventType, BaseEvent webhookEvent, String? secretToken, WebhookMessageFormat? messageFormat) {
|
public async Async.Task<Tuple<string, string?>> BuildMessage(Guid webhookId, Guid eventId, EventType eventType, BaseEvent webhookEvent, String? secretToken, WebhookMessageFormat? messageFormat) {
|
||||||
var entityConverter = new EntityConverter();
|
var entityConverter = new EntityConverter();
|
||||||
string data = "";
|
string data = "";
|
||||||
if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) {
|
if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) {
|
||||||
var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookEvent, DataVersion: "1.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) };
|
var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookEvent, DataVersion: "1.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) };
|
||||||
data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions());
|
data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions());
|
||||||
} else {
|
} else {
|
||||||
var instanceId = await _context.Containers.GetInstanceId();
|
var instanceId = await _context.Containers.GetInstanceId();
|
||||||
var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName());
|
var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName());
|
||||||
|
|
||||||
data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions());
|
data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions());
|
||||||
}
|
}
|
||||||
|
|
||||||
string? digest = null;
|
string? digest = null;
|
||||||
var hmac = HMAC.Create("HMACSHA512");
|
var hmac = HMAC.Create("HMACSHA512");
|
||||||
if (secretToken != null && hmac != null) {
|
if (secretToken != null && hmac != null) {
|
||||||
hmac.Key = System.Text.Encoding.UTF8.GetBytes(secretToken);
|
hmac.Key = System.Text.Encoding.UTF8.GetBytes(secretToken);
|
||||||
digest = Convert.ToHexString(hmac.ComputeHash(System.Text.Encoding.UTF8.GetBytes(data)));
|
digest = Convert.ToHexString(hmac.ComputeHash(System.Text.Encoding.UTF8.GetBytes(data)));
|
||||||
}
|
}
|
||||||
return new Tuple<string, string?>(data, digest);
|
return new Tuple<string, string?>(data, digest);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task<Webhook?> GetByWebhookId(Guid webhookId) {
|
public async Async.Task<Webhook?> GetByWebhookId(Guid webhookId) {
|
||||||
var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}'");
|
var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}'");
|
||||||
|
|
||||||
return await data.FirstOrDefaultAsync();
|
return await data.FirstOrDefaultAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
//todo: caching
|
//todo: caching
|
||||||
public IAsyncEnumerable<Webhook> GetWebhooksCached() {
|
public IAsyncEnumerable<Webhook> GetWebhooksCached() {
|
||||||
return QueryAsync();
|
return QueryAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public interface IWebhookMessageLogOperations : IOrm<WebhookMessageLog> {
|
public interface IWebhookMessageLogOperations : IOrm<WebhookMessageLog> {
|
||||||
IAsyncEnumerable<WebhookMessageLog> SearchExpired();
|
IAsyncEnumerable<WebhookMessageLog> SearchExpired();
|
||||||
public Async.Task ProcessFromQueue(WebhookMessageQueueObj obj);
|
public Async.Task ProcessFromQueue(WebhookMessageQueueObj obj);
|
||||||
public Async.Task QueueWebhook(WebhookMessageLog webhookLog);
|
public Async.Task QueueWebhook(WebhookMessageLog webhookLog);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public class WebhookMessageLogOperations : Orm<WebhookMessageLog>, IWebhookMessageLogOperations {
|
public class WebhookMessageLogOperations : Orm<WebhookMessageLog>, IWebhookMessageLogOperations {
|
||||||
const int EXPIRE_DAYS = 7;
|
const int EXPIRE_DAYS = 7;
|
||||||
const int MAX_TRIES = 5;
|
const int MAX_TRIES = 5;
|
||||||
|
|
||||||
|
|
||||||
public WebhookMessageLogOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context)
|
public WebhookMessageLogOperations(IHttpClientFactory httpFactory, ILogTracer log, IOnefuzzContext context)
|
||||||
: base(log, context) {
|
: base(log, context) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public async Async.Task QueueWebhook(WebhookMessageLog webhookLog) {
|
public async Async.Task QueueWebhook(WebhookMessageLog webhookLog) {
|
||||||
var obj = new WebhookMessageQueueObj(webhookLog.WebhookId, webhookLog.EventId);
|
var obj = new WebhookMessageQueueObj(webhookLog.WebhookId, webhookLog.EventId);
|
||||||
|
|
||||||
TimeSpan? visibilityTimeout = webhookLog.State switch {
|
TimeSpan? visibilityTimeout = webhookLog.State switch {
|
||||||
WebhookMessageState.Queued => TimeSpan.Zero,
|
WebhookMessageState.Queued => TimeSpan.Zero,
|
||||||
WebhookMessageState.Retrying => TimeSpan.FromSeconds(30),
|
WebhookMessageState.Retrying => TimeSpan.FromSeconds(30),
|
||||||
_ => null
|
_ => null
|
||||||
};
|
};
|
||||||
|
|
||||||
if (visibilityTimeout == null) {
|
if (visibilityTimeout == null) {
|
||||||
_logTracer.WithTags(
|
_logTracer.WithTags(
|
||||||
new[] {
|
new[] {
|
||||||
("WebhookId", webhookLog.WebhookId.ToString()),
|
("WebhookId", webhookLog.WebhookId.ToString()),
|
||||||
("EventId", webhookLog.EventId.ToString()) }
|
("EventId", webhookLog.EventId.ToString()) }
|
||||||
).
|
).
|
||||||
Error($"invalid WebhookMessage queue state, not queuing. {webhookLog.WebhookId}:{webhookLog.EventId} - {webhookLog.State}");
|
Error($"invalid WebhookMessage queue state, not queuing. {webhookLog.WebhookId}:{webhookLog.EventId} - {webhookLog.State}");
|
||||||
} else {
|
} else {
|
||||||
await _context.Queue.QueueObject("webhooks", obj, StorageType.Config, visibilityTimeout: visibilityTimeout);
|
await _context.Queue.QueueObject("webhooks", obj, StorageType.Config, visibilityTimeout: visibilityTimeout);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task ProcessFromQueue(WebhookMessageQueueObj obj) {
|
public async Async.Task ProcessFromQueue(WebhookMessageQueueObj obj) {
|
||||||
var message = await GetWebhookMessageById(obj.WebhookId, obj.EventId);
|
var message = await GetWebhookMessageById(obj.WebhookId, obj.EventId);
|
||||||
|
|
||||||
if (message == null) {
|
if (message == null) {
|
||||||
_logTracer.WithTags(
|
_logTracer.WithTags(
|
||||||
new[] {
|
new[] {
|
||||||
("WebhookId", obj.WebhookId.ToString()),
|
("WebhookId", obj.WebhookId.ToString()),
|
||||||
("EventId", obj.EventId.ToString()) }
|
("EventId", obj.EventId.ToString()) }
|
||||||
).
|
).
|
||||||
Error($"webhook message log not found for webhookId: {obj.WebhookId} and eventId: {obj.EventId}");
|
Error($"webhook message log not found for webhookId: {obj.WebhookId} and eventId: {obj.EventId}");
|
||||||
} else {
|
} else {
|
||||||
await Process(message);
|
await Process(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Async.Task Process(WebhookMessageLog message) {
|
private async Async.Task Process(WebhookMessageLog message) {
|
||||||
|
|
||||||
if (message.State == WebhookMessageState.Failed || message.State == WebhookMessageState.Succeeded) {
|
if (message.State == WebhookMessageState.Failed || message.State == WebhookMessageState.Succeeded) {
|
||||||
_logTracer.WithTags(
|
_logTracer.WithTags(
|
||||||
new[] {
|
new[] {
|
||||||
("WebhookId", message.WebhookId.ToString()),
|
("WebhookId", message.WebhookId.ToString()),
|
||||||
("EventId", message.EventId.ToString()) }
|
("EventId", message.EventId.ToString()) }
|
||||||
).
|
).
|
||||||
Error($"webhook message already handled. {message.WebhookId}:{message.EventId}");
|
Error($"webhook message already handled. {message.WebhookId}:{message.EventId}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var newMessage = message with { TryCount = message.TryCount + 1 };
|
var newMessage = message with { TryCount = message.TryCount + 1 };
|
||||||
|
|
||||||
_logTracer.Info($"sending webhook: {message.WebhookId}:{message.EventId}");
|
_logTracer.Info($"sending webhook: {message.WebhookId}:{message.EventId}");
|
||||||
var success = await Send(newMessage);
|
var success = await Send(newMessage);
|
||||||
if (success) {
|
if (success) {
|
||||||
newMessage = newMessage with { State = WebhookMessageState.Succeeded };
|
newMessage = newMessage with { State = WebhookMessageState.Succeeded };
|
||||||
await Replace(newMessage);
|
await Replace(newMessage);
|
||||||
_logTracer.Info($"sent webhook event {newMessage.WebhookId}:{newMessage.EventId}");
|
_logTracer.Info($"sent webhook event {newMessage.WebhookId}:{newMessage.EventId}");
|
||||||
} else if (newMessage.TryCount < MAX_TRIES) {
|
} else if (newMessage.TryCount < MAX_TRIES) {
|
||||||
newMessage = newMessage with { State = WebhookMessageState.Retrying };
|
newMessage = newMessage with { State = WebhookMessageState.Retrying };
|
||||||
await Replace(newMessage);
|
await Replace(newMessage);
|
||||||
await QueueWebhook(newMessage);
|
await QueueWebhook(newMessage);
|
||||||
_logTracer.Warning($"sending webhook event failed, re-queued {newMessage.WebhookId}:{newMessage.EventId}");
|
_logTracer.Warning($"sending webhook event failed, re-queued {newMessage.WebhookId}:{newMessage.EventId}");
|
||||||
} else {
|
} else {
|
||||||
newMessage = newMessage with { State = WebhookMessageState.Failed };
|
newMessage = newMessage with { State = WebhookMessageState.Failed };
|
||||||
await Replace(newMessage);
|
await Replace(newMessage);
|
||||||
_logTracer.Info($"sending webhook: {newMessage.WebhookId} event: {newMessage.EventId} failed {newMessage.TryCount} times.");
|
_logTracer.Info($"sending webhook: {newMessage.WebhookId} event: {newMessage.EventId} failed {newMessage.TryCount} times.");
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Async.Task<bool> Send(WebhookMessageLog message) {
|
private async Async.Task<bool> Send(WebhookMessageLog message) {
|
||||||
var webhook = await _context.WebhookOperations.GetByWebhookId(message.WebhookId);
|
var webhook = await _context.WebhookOperations.GetByWebhookId(message.WebhookId);
|
||||||
if (webhook == null) {
|
if (webhook == null) {
|
||||||
_logTracer.WithTags(
|
_logTracer.WithTags(
|
||||||
new[] {
|
new[] {
|
||||||
("WebhookId", message.WebhookId.ToString()),
|
("WebhookId", message.WebhookId.ToString()),
|
||||||
}
|
}
|
||||||
).
|
).
|
||||||
Error($"webhook not found for webhookId: {message.WebhookId}");
|
Error($"webhook not found for webhookId: {message.WebhookId}");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return await _context.WebhookOperations.Send(message);
|
return await _context.WebhookOperations.Send(message);
|
||||||
} catch (Exception exc) {
|
} catch (Exception exc) {
|
||||||
_logTracer.WithTags(
|
_logTracer.WithTags(
|
||||||
new[] {
|
new[] {
|
||||||
("WebhookId", message.WebhookId.ToString())
|
("WebhookId", message.WebhookId.ToString())
|
||||||
}
|
}
|
||||||
).
|
).
|
||||||
Exception(exc);
|
Exception(exc);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void QueueObject(string v, WebhookMessageQueueObj obj, StorageType config, int? visibility_timeout) {
|
private void QueueObject(string v, WebhookMessageQueueObj obj, StorageType config, int? visibility_timeout) {
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
}
|
}
|
||||||
|
|
||||||
public IAsyncEnumerable<WebhookMessageLog> SearchExpired() {
|
public IAsyncEnumerable<WebhookMessageLog> SearchExpired() {
|
||||||
var expireTime = (DateTimeOffset.UtcNow - TimeSpan.FromDays(EXPIRE_DAYS)).ToString("o");
|
var expireTime = (DateTimeOffset.UtcNow - TimeSpan.FromDays(EXPIRE_DAYS)).ToString("o");
|
||||||
|
|
||||||
var timeFilter = $"Timestamp lt datetime'{expireTime}'";
|
var timeFilter = $"Timestamp lt datetime'{expireTime}'";
|
||||||
return QueryAsync(filter: timeFilter);
|
return QueryAsync(filter: timeFilter);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Async.Task<WebhookMessageLog?> GetWebhookMessageById(Guid webhookId, Guid eventId) {
|
public async Async.Task<WebhookMessageLog?> GetWebhookMessageById(Guid webhookId, Guid eventId) {
|
||||||
var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}' and RowKey eq '{eventId}'");
|
var data = QueryAsync(filter: $"PartitionKey eq '{webhookId}' and RowKey eq '{eventId}'");
|
||||||
|
|
||||||
return await data.FirstOrDefaultAsync();
|
return await data.FirstOrDefaultAsync();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,325 +1,325 @@
|
|||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Linq.Expressions;
|
using System.Linq.Expressions;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
using Azure;
|
using Azure;
|
||||||
using Azure.Data.Tables;
|
using Azure.Data.Tables;
|
||||||
|
|
||||||
namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||||
|
|
||||||
public abstract record EntityBase {
|
public abstract record EntityBase {
|
||||||
[JsonIgnore] public ETag? ETag { get; set; }
|
[JsonIgnore] public ETag? ETag { get; set; }
|
||||||
public DateTimeOffset? TimeStamp { get; set; }
|
public DateTimeOffset? TimeStamp { get; set; }
|
||||||
|
|
||||||
// https://docs.microsoft.com/en-us/rest/api/storageservices/designing-a-scalable-partitioning-strategy-for-azure-table-storage#yyy
|
// https://docs.microsoft.com/en-us/rest/api/storageservices/designing-a-scalable-partitioning-strategy-for-azure-table-storage#yyy
|
||||||
// Produce "good-quality-table-key" based on a DateTimeOffset timestamp
|
// Produce "good-quality-table-key" based on a DateTimeOffset timestamp
|
||||||
public static string NewSortedKey => $"{DateTimeOffset.MaxValue.Ticks - DateTimeOffset.UtcNow.Ticks}";
|
public static string NewSortedKey => $"{DateTimeOffset.MaxValue.Ticks - DateTimeOffset.UtcNow.Ticks}";
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract record StatefulEntityBase<T>([property: JsonIgnore] T BaseState) : EntityBase() where T : Enum;
|
public abstract record StatefulEntityBase<T>([property: JsonIgnore] T BaseState) : EntityBase() where T : Enum;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/// How the value is populated
|
/// How the value is populated
|
||||||
public enum InitMethod {
|
public enum InitMethod {
|
||||||
//T() will be used
|
//T() will be used
|
||||||
DefaultConstructor,
|
DefaultConstructor,
|
||||||
}
|
}
|
||||||
[AttributeUsage(AttributeTargets.Parameter)]
|
[AttributeUsage(AttributeTargets.Parameter)]
|
||||||
public class DefaultValueAttribute : Attribute {
|
public class DefaultValueAttribute : Attribute {
|
||||||
|
|
||||||
public InitMethod InitMethod { get; }
|
public InitMethod InitMethod { get; }
|
||||||
public DefaultValueAttribute(InitMethod initMethod) {
|
public DefaultValueAttribute(InitMethod initMethod) {
|
||||||
InitMethod = initMethod;
|
InitMethod = initMethod;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Indicates that the enum cases should no be renamed
|
/// Indicates that the enum cases should no be renamed
|
||||||
[AttributeUsage(AttributeTargets.Enum)]
|
[AttributeUsage(AttributeTargets.Enum)]
|
||||||
public class SerializeValueAttribute : Attribute { }
|
public class SerializeValueAttribute : Attribute { }
|
||||||
|
|
||||||
/// Indicates that the enum cases should no be renamed
|
/// Indicates that the enum cases should no be renamed
|
||||||
[AttributeUsage(AttributeTargets.Enum)]
|
[AttributeUsage(AttributeTargets.Enum)]
|
||||||
public class SkipRenameAttribute : Attribute { }
|
public class SkipRenameAttribute : Attribute { }
|
||||||
[AttributeUsage(AttributeTargets.Parameter)]
|
[AttributeUsage(AttributeTargets.Parameter)]
|
||||||
public class RowKeyAttribute : Attribute { }
|
public class RowKeyAttribute : Attribute { }
|
||||||
[AttributeUsage(AttributeTargets.Parameter)]
|
[AttributeUsage(AttributeTargets.Parameter)]
|
||||||
public class PartitionKeyAttribute : Attribute { }
|
public class PartitionKeyAttribute : Attribute { }
|
||||||
[AttributeUsage(AttributeTargets.Property)]
|
[AttributeUsage(AttributeTargets.Property)]
|
||||||
public class TypeDiscrimnatorAttribute : Attribute {
|
public class TypeDiscrimnatorAttribute : Attribute {
|
||||||
public string FieldName { get; }
|
public string FieldName { get; }
|
||||||
// the type of a function that takes the value of fieldName as an input and return the type
|
// the type of a function that takes the value of fieldName as an input and return the type
|
||||||
public Type ConverterType { get; }
|
public Type ConverterType { get; }
|
||||||
|
|
||||||
public TypeDiscrimnatorAttribute(string fieldName, Type converterType) {
|
public TypeDiscrimnatorAttribute(string fieldName, Type converterType) {
|
||||||
if (!converterType.IsAssignableTo(typeof(ITypeProvider))) {
|
if (!converterType.IsAssignableTo(typeof(ITypeProvider))) {
|
||||||
throw new ArgumentException($"the provided type needs to implement ITypeProvider");
|
throw new ArgumentException($"the provided type needs to implement ITypeProvider");
|
||||||
}
|
}
|
||||||
|
|
||||||
FieldName = fieldName;
|
FieldName = fieldName;
|
||||||
ConverterType = converterType;
|
ConverterType = converterType;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public interface ITypeProvider {
|
public interface ITypeProvider {
|
||||||
Type GetTypeInfo(object input);
|
Type GetTypeInfo(object input);
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum EntityPropertyKind {
|
public enum EntityPropertyKind {
|
||||||
PartitionKey,
|
PartitionKey,
|
||||||
RowKey,
|
RowKey,
|
||||||
Column
|
Column
|
||||||
}
|
}
|
||||||
public record EntityProperty(
|
public record EntityProperty(
|
||||||
string name,
|
string name,
|
||||||
string columnName,
|
string columnName,
|
||||||
Type type,
|
Type type,
|
||||||
EntityPropertyKind kind,
|
EntityPropertyKind kind,
|
||||||
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator,
|
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator,
|
||||||
DefaultValueAttribute? defaultValue,
|
DefaultValueAttribute? defaultValue,
|
||||||
ParameterInfo parameterInfo
|
ParameterInfo parameterInfo
|
||||||
);
|
);
|
||||||
public record EntityInfo(Type type, ILookup<string, EntityProperty> properties, Func<object?[], object> constructor);
|
public record EntityInfo(Type type, ILookup<string, EntityProperty> properties, Func<object?[], object> constructor);
|
||||||
|
|
||||||
class OnefuzzNamingPolicy : JsonNamingPolicy {
|
class OnefuzzNamingPolicy : JsonNamingPolicy {
|
||||||
public override string ConvertName(string name) {
|
public override string ConvertName(string name) {
|
||||||
return CaseConverter.PascalToSnake(name);
|
return CaseConverter.PascalToSnake(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
public class EntityConverter {
|
public class EntityConverter {
|
||||||
private readonly JsonSerializerOptions _options;
|
private readonly JsonSerializerOptions _options;
|
||||||
|
|
||||||
private readonly ConcurrentDictionary<Type, EntityInfo> _cache;
|
private readonly ConcurrentDictionary<Type, EntityInfo> _cache;
|
||||||
|
|
||||||
public EntityConverter() {
|
public EntityConverter() {
|
||||||
_options = GetJsonSerializerOptions();
|
_options = GetJsonSerializerOptions();
|
||||||
_cache = new ConcurrentDictionary<Type, EntityInfo>();
|
_cache = new ConcurrentDictionary<Type, EntityInfo>();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static JsonSerializerOptions GetJsonSerializerOptions() {
|
public static JsonSerializerOptions GetJsonSerializerOptions() {
|
||||||
var options = new JsonSerializerOptions() {
|
var options = new JsonSerializerOptions() {
|
||||||
PropertyNamingPolicy = new OnefuzzNamingPolicy(),
|
PropertyNamingPolicy = new OnefuzzNamingPolicy(),
|
||||||
};
|
};
|
||||||
options.Converters.Add(new CustomEnumConverterFactory());
|
options.Converters.Add(new CustomEnumConverterFactory());
|
||||||
options.Converters.Add(new PolymorphicConverterFactory());
|
options.Converters.Add(new PolymorphicConverterFactory());
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo) {
|
internal static Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo) {
|
||||||
var constructorParameters = Expression.Parameter(typeof(object?[]));
|
var constructorParameters = Expression.Parameter(typeof(object?[]));
|
||||||
|
|
||||||
var parameterExpressions =
|
var parameterExpressions =
|
||||||
constructorInfo.GetParameters().Select((parameterInfo, i) => {
|
constructorInfo.GetParameters().Select((parameterInfo, i) => {
|
||||||
var ithIndex = Expression.Constant(i);
|
var ithIndex = Expression.Constant(i);
|
||||||
var ithParameter = Expression.ArrayIndex(constructorParameters, ithIndex);
|
var ithParameter = Expression.ArrayIndex(constructorParameters, ithIndex);
|
||||||
var unboxedIthParameter = Expression.Convert(ithParameter, parameterInfo.ParameterType);
|
var unboxedIthParameter = Expression.Convert(ithParameter, parameterInfo.ParameterType);
|
||||||
return unboxedIthParameter;
|
return unboxedIthParameter;
|
||||||
|
|
||||||
}).ToArray();
|
}).ToArray();
|
||||||
|
|
||||||
NewExpression constructorCall = Expression.New(constructorInfo, parameterExpressions);
|
NewExpression constructorCall = Expression.New(constructorInfo, parameterExpressions);
|
||||||
|
|
||||||
Func<object?[], object> ctor = Expression.Lambda<Func<object?[], object>>(constructorCall, constructorParameters).Compile();
|
Func<object?[], object> ctor = Expression.Lambda<Func<object?[], object>>(constructorCall, constructorParameters).Compile();
|
||||||
return ctor;
|
return ctor;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IEnumerable<EntityProperty> GetEntityProperties<T>(ParameterInfo parameterInfo) {
|
private static IEnumerable<EntityProperty> GetEntityProperties<T>(ParameterInfo parameterInfo) {
|
||||||
var name = parameterInfo.Name.EnsureNotNull($"Invalid paramter {parameterInfo}");
|
var name = parameterInfo.Name.EnsureNotNull($"Invalid paramter {parameterInfo}");
|
||||||
var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid paramter {parameterInfo}");
|
var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid paramter {parameterInfo}");
|
||||||
var isRowkey = parameterInfo.GetCustomAttribute(typeof(RowKeyAttribute)) != null;
|
var isRowkey = parameterInfo.GetCustomAttribute(typeof(RowKeyAttribute)) != null;
|
||||||
var isPartitionkey = parameterInfo.GetCustomAttribute(typeof(PartitionKeyAttribute)) != null;
|
var isPartitionkey = parameterInfo.GetCustomAttribute(typeof(PartitionKeyAttribute)) != null;
|
||||||
|
|
||||||
var discriminatorAttribute = typeof(T).GetProperty(name)?.GetCustomAttribute<TypeDiscrimnatorAttribute>();
|
var discriminatorAttribute = typeof(T).GetProperty(name)?.GetCustomAttribute<TypeDiscrimnatorAttribute>();
|
||||||
var defaultValueAttribute = parameterInfo.GetCustomAttribute<DefaultValueAttribute>();
|
var defaultValueAttribute = parameterInfo.GetCustomAttribute<DefaultValueAttribute>();
|
||||||
|
|
||||||
|
|
||||||
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null;
|
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null;
|
||||||
if (discriminatorAttribute != null) {
|
if (discriminatorAttribute != null) {
|
||||||
var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrive the type provider"));
|
var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrive the type provider"));
|
||||||
discriminator = (discriminatorAttribute, t);
|
discriminator = (discriminatorAttribute, t);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (isPartitionkey) {
|
if (isPartitionkey) {
|
||||||
yield return new EntityProperty(name, "PartitionKey", parameterType, EntityPropertyKind.PartitionKey, discriminator, defaultValueAttribute, parameterInfo);
|
yield return new EntityProperty(name, "PartitionKey", parameterType, EntityPropertyKind.PartitionKey, discriminator, defaultValueAttribute, parameterInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isRowkey) {
|
if (isRowkey) {
|
||||||
yield return new EntityProperty(name, "RowKey", parameterType, EntityPropertyKind.RowKey, discriminator, defaultValueAttribute, parameterInfo);
|
yield return new EntityProperty(name, "RowKey", parameterType, EntityPropertyKind.RowKey, discriminator, defaultValueAttribute, parameterInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isPartitionkey && !isRowkey) {
|
if (!isPartitionkey && !isRowkey) {
|
||||||
var columnName = typeof(T).GetProperty(name)?.GetCustomAttribute<JsonPropertyNameAttribute>()?.Name ?? CaseConverter.PascalToSnake(name);
|
var columnName = typeof(T).GetProperty(name)?.GetCustomAttribute<JsonPropertyNameAttribute>()?.Name ?? CaseConverter.PascalToSnake(name);
|
||||||
yield return new EntityProperty(name, columnName, parameterType, EntityPropertyKind.Column, discriminator, defaultValueAttribute, parameterInfo);
|
yield return new EntityProperty(name, columnName, parameterType, EntityPropertyKind.Column, discriminator, defaultValueAttribute, parameterInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private EntityInfo GetEntityInfo<T>() {
|
private EntityInfo GetEntityInfo<T>() {
|
||||||
return _cache.GetOrAdd(typeof(T), type => {
|
return _cache.GetOrAdd(typeof(T), type => {
|
||||||
var constructor = type.GetConstructors()[0];
|
var constructor = type.GetConstructors()[0];
|
||||||
var parameterInfos = constructor.GetParameters();
|
var parameterInfos = constructor.GetParameters();
|
||||||
var parameters =
|
var parameters =
|
||||||
parameterInfos.SelectMany(GetEntityProperties<T>).ToArray();
|
parameterInfos.SelectMany(GetEntityProperties<T>).ToArray();
|
||||||
|
|
||||||
return new EntityInfo(typeof(T), parameters.ToLookup(x => x.name), BuildConstructerFrom(constructor));
|
return new EntityInfo(typeof(T), parameters.ToLookup(x => x.name), BuildConstructerFrom(constructor));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ToJsonString<T>(T typedEntity) => JsonSerializer.Serialize(typedEntity, _options);
|
public string ToJsonString<T>(T typedEntity) => JsonSerializer.Serialize(typedEntity, _options);
|
||||||
|
|
||||||
public T? FromJsonString<T>(string value) => JsonSerializer.Deserialize<T>(value, _options);
|
public T? FromJsonString<T>(string value) => JsonSerializer.Deserialize<T>(value, _options);
|
||||||
|
|
||||||
public TableEntity ToTableEntity<T>(T typedEntity) where T : EntityBase {
|
public TableEntity ToTableEntity<T>(T typedEntity) where T : EntityBase {
|
||||||
if (typedEntity == null) {
|
if (typedEntity == null) {
|
||||||
throw new ArgumentNullException(nameof(typedEntity));
|
throw new ArgumentNullException(nameof(typedEntity));
|
||||||
}
|
}
|
||||||
|
|
||||||
var type = typeof(T);
|
var type = typeof(T);
|
||||||
|
|
||||||
var entityInfo = GetEntityInfo<T>();
|
var entityInfo = GetEntityInfo<T>();
|
||||||
Dictionary<string, object?> columnValues = entityInfo.properties.SelectMany(x => x).Select(prop => {
|
Dictionary<string, object?> columnValues = entityInfo.properties.SelectMany(x => x).Select(prop => {
|
||||||
var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity);
|
var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity);
|
||||||
if (value == null) {
|
if (value == null) {
|
||||||
return (prop.columnName, value: (object?)null);
|
return (prop.columnName, value: (object?)null);
|
||||||
}
|
}
|
||||||
if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey) {
|
if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey) {
|
||||||
return (prop.columnName, value?.ToString());
|
return (prop.columnName, value?.ToString());
|
||||||
}
|
}
|
||||||
if (prop.type == typeof(Guid) || prop.type == typeof(Guid?) || prop.type == typeof(Uri)) {
|
if (prop.type == typeof(Guid) || prop.type == typeof(Guid?) || prop.type == typeof(Uri)) {
|
||||||
return (prop.columnName, value?.ToString());
|
return (prop.columnName, value?.ToString());
|
||||||
}
|
}
|
||||||
if (prop.type == typeof(bool)
|
if (prop.type == typeof(bool)
|
||||||
|| prop.type == typeof(bool?)
|
|| prop.type == typeof(bool?)
|
||||||
|| prop.type == typeof(string)
|
|| prop.type == typeof(string)
|
||||||
|| prop.type == typeof(DateTime)
|
|| prop.type == typeof(DateTime)
|
||||||
|| prop.type == typeof(DateTime?)
|
|| prop.type == typeof(DateTime?)
|
||||||
|| prop.type == typeof(DateTimeOffset)
|
|| prop.type == typeof(DateTimeOffset)
|
||||||
|| prop.type == typeof(DateTimeOffset?)
|
|| prop.type == typeof(DateTimeOffset?)
|
||||||
|| prop.type == typeof(int)
|
|| prop.type == typeof(int)
|
||||||
|| prop.type == typeof(int?)
|
|| prop.type == typeof(int?)
|
||||||
|| prop.type == typeof(long)
|
|| prop.type == typeof(long)
|
||||||
|| prop.type == typeof(long?)
|
|| prop.type == typeof(long?)
|
||||||
|| prop.type == typeof(double)
|
|| prop.type == typeof(double)
|
||||||
|| prop.type == typeof(double?)
|
|| prop.type == typeof(double?)
|
||||||
|
|
||||||
) {
|
) {
|
||||||
return (prop.columnName, value);
|
return (prop.columnName, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
var serialized = JsonSerializer.Serialize(value, _options);
|
var serialized = JsonSerializer.Serialize(value, _options);
|
||||||
return (prop.columnName, serialized.Trim('"'));
|
return (prop.columnName, serialized.Trim('"'));
|
||||||
|
|
||||||
}).ToDictionary(x => x.columnName, x => x.value);
|
}).ToDictionary(x => x.columnName, x => x.value);
|
||||||
|
|
||||||
var tableEntity = new TableEntity(columnValues);
|
var tableEntity = new TableEntity(columnValues);
|
||||||
|
|
||||||
if (typedEntity.ETag.HasValue) {
|
if (typedEntity.ETag.HasValue) {
|
||||||
tableEntity.ETag = typedEntity.ETag.Value;
|
tableEntity.ETag = typedEntity.ETag.Value;
|
||||||
}
|
}
|
||||||
|
|
||||||
return tableEntity;
|
return tableEntity;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) {
|
private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) {
|
||||||
var ef = info.properties[name].First();
|
var ef = info.properties[name].First();
|
||||||
if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) {
|
if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) {
|
||||||
if (ef.type == typeof(string))
|
if (ef.type == typeof(string))
|
||||||
return entity.GetString(ef.kind.ToString());
|
return entity.GetString(ef.kind.ToString());
|
||||||
else if (ef.type == typeof(Guid))
|
else if (ef.type == typeof(Guid))
|
||||||
return Guid.Parse(entity.GetString(ef.kind.ToString()));
|
return Guid.Parse(entity.GetString(ef.kind.ToString()));
|
||||||
else if (ef.type == typeof(int))
|
else if (ef.type == typeof(int))
|
||||||
return int.Parse(entity.GetString(ef.kind.ToString()));
|
return int.Parse(entity.GetString(ef.kind.ToString()));
|
||||||
else if (ef.type == typeof(long))
|
else if (ef.type == typeof(long))
|
||||||
return long.Parse(entity.GetString(ef.kind.ToString()));
|
return long.Parse(entity.GetString(ef.kind.ToString()));
|
||||||
else if (ef.type.IsClass)
|
else if (ef.type.IsClass)
|
||||||
return ef.type.GetConstructor(new[] { typeof(string) })!.Invoke(new[] { entity.GetString(ef.kind.ToString()) });
|
return ef.type.GetConstructor(new[] { typeof(string) })!.Invoke(new[] { entity.GetString(ef.kind.ToString()) });
|
||||||
else {
|
else {
|
||||||
throw new Exception($"invalid partition or row key type of {info.type} property {name}: {ef.type}");
|
throw new Exception($"invalid partition or row key type of {info.type} property {name}: {ef.type}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var fieldName = ef.columnName;
|
var fieldName = ef.columnName;
|
||||||
var obj = entity[fieldName];
|
var obj = entity[fieldName];
|
||||||
if (obj == null) {
|
if (obj == null) {
|
||||||
|
|
||||||
if (ef.parameterInfo.HasDefaultValue) {
|
if (ef.parameterInfo.HasDefaultValue) {
|
||||||
return ef.parameterInfo.DefaultValue;
|
return ef.parameterInfo.DefaultValue;
|
||||||
}
|
}
|
||||||
|
|
||||||
return ef.defaultValue switch {
|
return ef.defaultValue switch {
|
||||||
DefaultValueAttribute { InitMethod: InitMethod.DefaultConstructor } => Activator.CreateInstance(ef.type),
|
DefaultValueAttribute { InitMethod: InitMethod.DefaultConstructor } => Activator.CreateInstance(ef.type),
|
||||||
_ => null,
|
_ => null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (ef.type == typeof(string)) {
|
if (ef.type == typeof(string)) {
|
||||||
return entity.GetString(fieldName);
|
return entity.GetString(fieldName);
|
||||||
} else if (ef.type == typeof(bool) || ef.type == typeof(bool?)) {
|
} else if (ef.type == typeof(bool) || ef.type == typeof(bool?)) {
|
||||||
return entity.GetBoolean(fieldName);
|
return entity.GetBoolean(fieldName);
|
||||||
} else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?)) {
|
} else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?)) {
|
||||||
return entity.GetDateTimeOffset(fieldName);
|
return entity.GetDateTimeOffset(fieldName);
|
||||||
} else if (ef.type == typeof(DateTime) || ef.type == typeof(DateTime?)) {
|
} else if (ef.type == typeof(DateTime) || ef.type == typeof(DateTime?)) {
|
||||||
return entity.GetDateTime(fieldName);
|
return entity.GetDateTime(fieldName);
|
||||||
} else if (ef.type == typeof(double) || ef.type == typeof(double?)) {
|
} else if (ef.type == typeof(double) || ef.type == typeof(double?)) {
|
||||||
return entity.GetDouble(fieldName);
|
return entity.GetDouble(fieldName);
|
||||||
} else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?)) {
|
} else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?)) {
|
||||||
return (object?)Guid.Parse(entity.GetString(fieldName));
|
return (object?)Guid.Parse(entity.GetString(fieldName));
|
||||||
} else if (ef.type == typeof(int) || ef.type == typeof(short) || ef.type == typeof(int?) || ef.type == typeof(short?)) {
|
} else if (ef.type == typeof(int) || ef.type == typeof(short) || ef.type == typeof(int?) || ef.type == typeof(short?)) {
|
||||||
return entity.GetInt32(fieldName);
|
return entity.GetInt32(fieldName);
|
||||||
} else if (ef.type == typeof(long) || ef.type == typeof(long?)) {
|
} else if (ef.type == typeof(long) || ef.type == typeof(long?)) {
|
||||||
return entity.GetInt64(fieldName);
|
return entity.GetInt64(fieldName);
|
||||||
} else {
|
} else {
|
||||||
var outputType = ef.type;
|
var outputType = ef.type;
|
||||||
if (ef.discriminator != null) {
|
if (ef.discriminator != null) {
|
||||||
var (attr, typeProvider) = ef.discriminator.Value;
|
var (attr, typeProvider) = ef.discriminator.Value;
|
||||||
var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}");
|
var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}");
|
||||||
outputType = typeProvider.GetTypeInfo(v);
|
outputType = typeProvider.GetTypeInfo(v);
|
||||||
}
|
}
|
||||||
|
|
||||||
var objType = obj.GetType();
|
var objType = obj.GetType();
|
||||||
if (objType == typeof(string)) {
|
if (objType == typeof(string)) {
|
||||||
var value = entity.GetString(fieldName);
|
var value = entity.GetString(fieldName);
|
||||||
if (value.StartsWith('[') || value.StartsWith('{') || value == "null") {
|
if (value.StartsWith('[') || value.StartsWith('{') || value == "null") {
|
||||||
return JsonSerializer.Deserialize(value, outputType, options: _options);
|
return JsonSerializer.Deserialize(value, outputType, options: _options);
|
||||||
} else {
|
} else {
|
||||||
return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options);
|
return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
var value = entity.GetString(fieldName);
|
var value = entity.GetString(fieldName);
|
||||||
return JsonSerializer.Deserialize(value, outputType, options: _options);
|
return JsonSerializer.Deserialize(value, outputType, options: _options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}')", ex);
|
throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}')", ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public T ToRecord<T>(TableEntity entity) where T : EntityBase {
|
public T ToRecord<T>(TableEntity entity) where T : EntityBase {
|
||||||
var entityInfo = GetEntityInfo<T>();
|
var entityInfo = GetEntityInfo<T>();
|
||||||
|
|
||||||
object?[] parameters;
|
object?[] parameters;
|
||||||
try {
|
try {
|
||||||
parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity)).ToArray();
|
parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity)).ToArray();
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new InvalidOperationException($"Unable to extract properties from TableEntity for {typeof(T)}", ex);
|
throw new InvalidOperationException($"Unable to extract properties from TableEntity for {typeof(T)}", ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
var entityRecord = (T)entityInfo.constructor.Invoke(parameters);
|
var entityRecord = (T)entityInfo.constructor.Invoke(parameters);
|
||||||
if (entity.ETag != default) {
|
if (entity.ETag != default) {
|
||||||
entityRecord.ETag = entity.ETag;
|
entityRecord.ETag = entity.ETag;
|
||||||
}
|
}
|
||||||
entityRecord.TimeStamp = entity.Timestamp;
|
entityRecord.TimeStamp = entity.Timestamp;
|
||||||
return entityRecord;
|
return entityRecord;
|
||||||
|
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
var stringParam = string.Join(", ", parameters);
|
var stringParam = string.Join(", ", parameters);
|
||||||
throw new InvalidOperationException($"Could not initialize object of type {typeof(T)} with the following parameters: {stringParam} constructor {entityInfo.constructor}", ex);
|
throw new InvalidOperationException($"Could not initialize object of type {typeof(T)} with the following parameters: {stringParam} constructor {entityInfo.constructor}", ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,42 +1,42 @@
|
|||||||
<?xml version=\"1.0\" encoding=\"utf-8\"?>
|
<?xml version=\"1.0\" encoding=\"utf-8\"?>
|
||||||
<coverage line-rate=\"0.40\" branch-rate=\"0\" lines-covered=\"2\" lines-valid=\"5\" branches-covered=\"0\" branches-valid=\"0\" complexity=\"0\" version=\"0.1\" timestamp=\"1648844445\">
|
<coverage line-rate=\"0.40\" branch-rate=\"0\" lines-covered=\"2\" lines-valid=\"5\" branches-covered=\"0\" branches-valid=\"0\" complexity=\"0\" version=\"0.1\" timestamp=\"1648844445\">
|
||||||
<packages>
|
<packages>
|
||||||
<package name=\"C:/Users\" line-rate=\"0.50\" branch-rate=\"0\" complexity=\"0\">
|
<package name=\"C:/Users\" line-rate=\"0.50\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<classes>
|
<classes>
|
||||||
<class name=\"C:/Users/file1.txt\" filename=\"C:/Users/file1.txt\" line-rate=\"0.50\" branch-rate=\"0\" complexity=\"0\">
|
<class name=\"C:/Users/file1.txt\" filename=\"C:/Users/file1.txt\" line-rate=\"0.50\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<lines>
|
<lines>
|
||||||
<line number=\"5\" hits=\"3\" branch=\"false\" />
|
<line number=\"5\" hits=\"3\" branch=\"false\" />
|
||||||
<line number=\"10\" hits=\"0\" branch=\"false\" />
|
<line number=\"10\" hits=\"0\" branch=\"false\" />
|
||||||
</lines>
|
</lines>
|
||||||
</class>
|
</class>
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
<package name=\"C:/Users\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
<package name=\"C:/Users\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<classes>
|
<classes>
|
||||||
<class name=\"C:/Users/file2.txt\" filename=\"C:/Users/file2.txt\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
<class name=\"C:/Users/file2.txt\" filename=\"C:/Users/file2.txt\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<lines>
|
<lines>
|
||||||
<line number=\"1\" hits=\"0\" branch=\"false\" />
|
<line number=\"1\" hits=\"0\" branch=\"false\" />
|
||||||
</lines>
|
</lines>
|
||||||
</class>
|
</class>
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
<package name=\"Invalid file format: C:/Users/file/..\" line-rate=\"1.00\" branch-rate=\"0\" complexity=\"0\">
|
<package name=\"Invalid file format: C:/Users/file/..\" line-rate=\"1.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<classes>
|
<classes>
|
||||||
<class name=\"C:/Users/file/..\" filename=\"C:/Users/file/..\" line-rate=\"1.00\" branch-rate=\"0\" complexity=\"0\">
|
<class name=\"C:/Users/file/..\" filename=\"C:/Users/file/..\" line-rate=\"1.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<lines>
|
<lines>
|
||||||
<line number=\"1\" hits=\"1\" branch=\"false\" />
|
<line number=\"1\" hits=\"1\" branch=\"false\" />
|
||||||
</lines>
|
</lines>
|
||||||
</class>
|
</class>
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
<package name=\"Invalid file format: C:/Users/file/..\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
<package name=\"Invalid file format: C:/Users/file/..\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<classes>
|
<classes>
|
||||||
<class name=\"C:/Users/file/..\" filename=\"C:/Users/file/..\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
<class name=\"C:/Users/file/..\" filename=\"C:/Users/file/..\" line-rate=\"0.00\" branch-rate=\"0\" complexity=\"0\">
|
||||||
<lines>
|
<lines>
|
||||||
<line number=\"1\" hits=\"0\" branch=\"false\" />
|
<line number=\"1\" hits=\"0\" branch=\"false\" />
|
||||||
</lines>
|
</lines>
|
||||||
</class>
|
</class>
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
</packages>
|
</packages>
|
||||||
</coverage>
|
</coverage>
|
||||||
@ -1,34 +1,34 @@
|
|||||||
# srcview
|
# srcview
|
||||||
|
|
||||||
A library for mapping module+offset to source:line. Note that you'll get
|
A library for mapping module+offset to source:line. Note that you'll get
|
||||||
significantly better results if you use instruction level coverage as
|
significantly better results if you use instruction level coverage as
|
||||||
opposed to branch. Alternatively you're welcome to post-process branch
|
opposed to branch. Alternatively you're welcome to post-process branch
|
||||||
coverage into instruction, but this project does not assist with that.
|
coverage into instruction, but this project does not assist with that.
|
||||||
|
|
||||||
## Docs
|
## Docs
|
||||||
|
|
||||||
`cargo doc --open`
|
`cargo doc --open`
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
See [`examples/dump_cobertura.rs`](examples/dump_cobertura.rs).
|
See [`examples/dump_cobertura.rs`](examples/dump_cobertura.rs).
|
||||||
|
|
||||||
This can be run with `cargo run --example dump_cobertura res\example.pdb res\example.txt`.
|
This can be run with `cargo run --example dump_cobertura res\example.pdb res\example.txt`.
|
||||||
|
|
||||||
The biggest challenge when making this work is likely getting the absolute PDB
|
The biggest challenge when making this work is likely getting the absolute PDB
|
||||||
paths to match relative paths inside your repo. To make this a bit easier, you
|
paths to match relative paths inside your repo. To make this a bit easier, you
|
||||||
can dump the PDB paths with: `cargo run --example dump_paths res\example.pdb`
|
can dump the PDB paths with: `cargo run --example dump_paths res\example.pdb`
|
||||||
|
|
||||||
## ADO Integration
|
## ADO Integration
|
||||||
|
|
||||||
See [`azure-pipelines.yml`](azure-pipelines.yml).
|
See [`azure-pipelines.yml`](azure-pipelines.yml).
|
||||||
|
|
||||||
## VSCode Integration
|
## VSCode Integration
|
||||||
|
|
||||||
Install [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters),
|
Install [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters),
|
||||||
then place a file named `coverage.xml` at a location such that the relative
|
then place a file named `coverage.xml` at a location such that the relative
|
||||||
file paths match your repo. Note that this is a file you should generate
|
file paths match your repo. Note that this is a file you should generate
|
||||||
yourself, and that the `coverage.xml` included in this repo will probably not
|
yourself, and that the `coverage.xml` included in this repo will probably not
|
||||||
help you. From there, navigate to any file you expect to see coverage in. If
|
help you. From there, navigate to any file you expect to see coverage in. If
|
||||||
the paths and coverage file correctly line up, you should see red or green bars
|
the paths and coverage file correctly line up, you should see red or green bars
|
||||||
next to the source lines.
|
next to the source lines.
|
||||||
@ -1,6 +1,6 @@
|
|||||||
# TODO
|
# TODO
|
||||||
- add trace/info/warn
|
- add trace/info/warn
|
||||||
- flesh out modoff parser error
|
- flesh out modoff parser error
|
||||||
- consider using Cow to reduce memory usage
|
- consider using Cow to reduce memory usage
|
||||||
- consider mixed case module names?
|
- consider mixed case module names?
|
||||||
- redo xml generation to not be hand-rolled
|
- redo xml generation to not be hand-rolled
|
||||||
6
src/api-service/__app__/.gitignore
vendored
6
src/api-service/__app__/.gitignore
vendored
@ -1,4 +1,4 @@
|
|||||||
.direnv
|
.direnv
|
||||||
.python_packages
|
.python_packages
|
||||||
__pycache__
|
__pycache__
|
||||||
.venv
|
.venv
|
||||||
@ -1,74 +1,74 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# Copyright (c) Microsoft Corporation.
|
# Copyright (c) Microsoft Corporation.
|
||||||
# Licensed under the MIT License.
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import azure.functions as func
|
import azure.functions as func
|
||||||
from onefuzztypes.enums import JobState, TaskState
|
from onefuzztypes.enums import JobState, TaskState
|
||||||
|
|
||||||
from ..onefuzzlib.jobs import Job
|
from ..onefuzzlib.jobs import Job
|
||||||
from ..onefuzzlib.notifications.main import Notification
|
from ..onefuzzlib.notifications.main import Notification
|
||||||
from ..onefuzzlib.repro import Repro
|
from ..onefuzzlib.repro import Repro
|
||||||
from ..onefuzzlib.tasks.main import Task
|
from ..onefuzzlib.tasks.main import Task
|
||||||
|
|
||||||
RETENTION_POLICY = datetime.timedelta(days=(18 * 30))
|
RETENTION_POLICY = datetime.timedelta(days=(18 * 30))
|
||||||
SEARCH_EXTENT = datetime.timedelta(days=(20 * 30))
|
SEARCH_EXTENT = datetime.timedelta(days=(20 * 30))
|
||||||
|
|
||||||
|
|
||||||
def main(mytimer: func.TimerRequest) -> None: # noqa: F841
|
def main(mytimer: func.TimerRequest) -> None: # noqa: F841
|
||||||
|
|
||||||
now = datetime.datetime.now(tz=datetime.timezone.utc)
|
now = datetime.datetime.now(tz=datetime.timezone.utc)
|
||||||
|
|
||||||
time_retained_older = now - RETENTION_POLICY
|
time_retained_older = now - RETENTION_POLICY
|
||||||
time_retained_newer = now - SEARCH_EXTENT
|
time_retained_newer = now - SEARCH_EXTENT
|
||||||
|
|
||||||
time_filter = (
|
time_filter = (
|
||||||
f"Timestamp lt datetime'{time_retained_older.isoformat()}' "
|
f"Timestamp lt datetime'{time_retained_older.isoformat()}' "
|
||||||
f"and Timestamp gt datetime'{time_retained_newer.isoformat()}'"
|
f"and Timestamp gt datetime'{time_retained_newer.isoformat()}'"
|
||||||
)
|
)
|
||||||
time_filter_newer = f"Timestamp gt datetime'{time_retained_older.isoformat()}'"
|
time_filter_newer = f"Timestamp gt datetime'{time_retained_older.isoformat()}'"
|
||||||
|
|
||||||
# Collecting 'still relevant' task containers.
|
# Collecting 'still relevant' task containers.
|
||||||
# NOTE: This must be done before potentially modifying tasks otherwise
|
# NOTE: This must be done before potentially modifying tasks otherwise
|
||||||
# the task timestamps will not be useful.
|
# the task timestamps will not be useful.
|
||||||
used_containers = set()
|
used_containers = set()
|
||||||
for task in Task.search(raw_unchecked_filter=time_filter_newer):
|
for task in Task.search(raw_unchecked_filter=time_filter_newer):
|
||||||
task_containers = {x.name for x in task.config.containers}
|
task_containers = {x.name for x in task.config.containers}
|
||||||
used_containers.update(task_containers)
|
used_containers.update(task_containers)
|
||||||
|
|
||||||
for notification in Notification.search(raw_unchecked_filter=time_filter):
|
for notification in Notification.search(raw_unchecked_filter=time_filter):
|
||||||
logging.debug(
|
logging.debug(
|
||||||
"checking expired notification for removal: %s",
|
"checking expired notification for removal: %s",
|
||||||
notification.notification_id,
|
notification.notification_id,
|
||||||
)
|
)
|
||||||
container = notification.container
|
container = notification.container
|
||||||
if container not in used_containers:
|
if container not in used_containers:
|
||||||
logging.info(
|
logging.info(
|
||||||
"deleting expired notification: %s", notification.notification_id
|
"deleting expired notification: %s", notification.notification_id
|
||||||
)
|
)
|
||||||
notification.delete()
|
notification.delete()
|
||||||
|
|
||||||
for job in Job.search(
|
for job in Job.search(
|
||||||
query={"state": [JobState.stopped]}, raw_unchecked_filter=time_filter
|
query={"state": [JobState.stopped]}, raw_unchecked_filter=time_filter
|
||||||
):
|
):
|
||||||
if job.user_info is not None and job.user_info.upn is not None:
|
if job.user_info is not None and job.user_info.upn is not None:
|
||||||
logging.info("removing PII from job: %s", job.job_id)
|
logging.info("removing PII from job: %s", job.job_id)
|
||||||
job.user_info.upn = None
|
job.user_info.upn = None
|
||||||
job.save()
|
job.save()
|
||||||
|
|
||||||
for task in Task.search(
|
for task in Task.search(
|
||||||
query={"state": [TaskState.stopped]}, raw_unchecked_filter=time_filter
|
query={"state": [TaskState.stopped]}, raw_unchecked_filter=time_filter
|
||||||
):
|
):
|
||||||
if task.user_info is not None and task.user_info.upn is not None:
|
if task.user_info is not None and task.user_info.upn is not None:
|
||||||
logging.info("removing PII from task: %s", task.task_id)
|
logging.info("removing PII from task: %s", task.task_id)
|
||||||
task.user_info.upn = None
|
task.user_info.upn = None
|
||||||
task.save()
|
task.save()
|
||||||
|
|
||||||
for repro in Repro.search(raw_unchecked_filter=time_filter):
|
for repro in Repro.search(raw_unchecked_filter=time_filter):
|
||||||
if repro.user_info is not None and repro.user_info.upn is not None:
|
if repro.user_info is not None and repro.user_info.upn is not None:
|
||||||
logging.info("removing PII from repro: %s", repro.vm_id)
|
logging.info("removing PII from repro: %s", repro.vm_id)
|
||||||
repro.user_info.upn = None
|
repro.user_info.upn = None
|
||||||
repro.save()
|
repro.save()
|
||||||
|
|||||||
144
src/deployment/.gitignore
vendored
144
src/deployment/.gitignore
vendored
@ -1,73 +1,73 @@
|
|||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
*$py.class
|
*$py.class
|
||||||
|
|
||||||
# C extensions
|
# C extensions
|
||||||
*.so
|
*.so
|
||||||
|
|
||||||
# Distribution / packaging
|
# Distribution / packaging
|
||||||
.Python
|
.Python
|
||||||
build/
|
build/
|
||||||
develop-eggs/
|
develop-eggs/
|
||||||
dist/
|
dist/
|
||||||
downloads/
|
downloads/
|
||||||
eggs/
|
eggs/
|
||||||
.eggs/
|
.eggs/
|
||||||
lib/
|
lib/
|
||||||
lib64/
|
lib64/
|
||||||
parts/
|
parts/
|
||||||
sdist/
|
sdist/
|
||||||
var/
|
var/
|
||||||
wheels/
|
wheels/
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
.installed.cfg
|
.installed.cfg
|
||||||
*.egg
|
*.egg
|
||||||
MANIFEST
|
MANIFEST
|
||||||
|
|
||||||
# PyInstaller
|
# PyInstaller
|
||||||
# Usually these files are written by a python script from a template
|
# Usually these files are written by a python script from a template
|
||||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
*.manifest
|
*.manifest
|
||||||
*.spec
|
*.spec
|
||||||
|
|
||||||
# Installer logs
|
# Installer logs
|
||||||
pip-log.txt
|
pip-log.txt
|
||||||
pip-delete-this-directory.txt
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
# Unit test / coverage reports
|
# Unit test / coverage reports
|
||||||
htmlcov/
|
htmlcov/
|
||||||
.tox/
|
.tox/
|
||||||
.coverage
|
.coverage
|
||||||
.coverage.*
|
.coverage.*
|
||||||
.cache
|
.cache
|
||||||
nosetests.xml
|
nosetests.xml
|
||||||
coverage.xml
|
coverage.xml
|
||||||
*.cover
|
*.cover
|
||||||
.hypothesis/
|
.hypothesis/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
|
||||||
# Flask stuff:
|
# Flask stuff:
|
||||||
instance/
|
instance/
|
||||||
.webassets-cache
|
.webassets-cache
|
||||||
|
|
||||||
# Scrapy stuff:
|
# Scrapy stuff:
|
||||||
.scrapy
|
.scrapy
|
||||||
|
|
||||||
# pyenv
|
# pyenv
|
||||||
.python-version
|
.python-version
|
||||||
|
|
||||||
# Environments
|
# Environments
|
||||||
.env
|
.env
|
||||||
.venv
|
.venv
|
||||||
env/
|
env/
|
||||||
venv/
|
venv/
|
||||||
ENV/
|
ENV/
|
||||||
env.bak/
|
env.bak/
|
||||||
venv.bak/
|
venv.bak/
|
||||||
|
|
||||||
# mypy
|
# mypy
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
|
|
||||||
test-cov.xml
|
test-cov.xml
|
||||||
test-output.xml
|
test-output.xml
|
||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"proxy_nsg_config": {
|
"proxy_nsg_config": {
|
||||||
"allowed_ips": ["*"],
|
"allowed_ips": ["*"],
|
||||||
"allowed_service_tags": []
|
"allowed_service_tags": []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1,15 +1,15 @@
|
|||||||
using System;
|
using System;
|
||||||
namespace Problems {
|
namespace Problems {
|
||||||
public static class Problems {
|
public static class Problems {
|
||||||
public static void Func(ReadOnlySpan<byte> data) {
|
public static void Func(ReadOnlySpan<byte> data) {
|
||||||
var count = 0;
|
var count = 0;
|
||||||
if (data.Length < 4) {
|
if (data.Length < 4) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (data[0] == 0) { count++; }
|
if (data[0] == 0) { count++; }
|
||||||
if (data[1] == 1) { count++; }
|
if (data[1] == 1) { count++; }
|
||||||
if (data[2] == 2) { count++; }
|
if (data[2] == 2) { count++; }
|
||||||
if (count >= 3) { throw new Exception("this is bad"); }
|
if (count >= 3) { throw new Exception("this is bad"); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>netstandard2.1</TargetFramework>
|
<TargetFramework>netstandard2.1</TargetFramework>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@ -1,8 +1,8 @@
|
|||||||
using SharpFuzz;
|
using SharpFuzz;
|
||||||
namespace Wrapper {
|
namespace Wrapper {
|
||||||
public class Program {
|
public class Program {
|
||||||
public static void Main(string[] args) {
|
public static void Main(string[] args) {
|
||||||
Fuzzer.LibFuzzer.Run(stream => { Problems.Problems.Func(stream); });
|
Fuzzer.LibFuzzer.Run(stream => { Problems.Problems.Func(stream); });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,15 +1,15 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\problems\problems.csproj" />
|
<ProjectReference Include="..\problems\problems.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="SharpFuzz" Version="1.6.1" />
|
<PackageReference Include="SharpFuzz" Version="1.6.1" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<OutputType>Exe</OutputType>
|
<OutputType>Exe</OutputType>
|
||||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@ -1,85 +1,85 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#ifdef _WIN64
|
#ifdef _WIN64
|
||||||
#include <io.h>
|
#include <io.h>
|
||||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||||
#define STDIN_FILENO _fileno(stdin)
|
#define STDIN_FILENO _fileno(stdin)
|
||||||
#define read _read
|
#define read _read
|
||||||
#else
|
#else
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define SIZE 8192
|
#define SIZE 8192
|
||||||
#define BUF_SIZE 32
|
#define BUF_SIZE 32
|
||||||
|
|
||||||
int check(const char *data, size_t len)
|
int check(const char *data, size_t len)
|
||||||
{
|
{
|
||||||
char buf[BUF_SIZE];
|
char buf[BUF_SIZE];
|
||||||
memset(buf, 0, BUF_SIZE);
|
memset(buf, 0, BUF_SIZE);
|
||||||
strncpy(buf, data, len); // BUG - This incorrectly uses length of src, not dst
|
strncpy(buf, data, len); // BUG - This incorrectly uses length of src, not dst
|
||||||
|
|
||||||
// do something to ensure this isn't optimized away
|
// do something to ensure this isn't optimized away
|
||||||
int buflen = strlen(buf);
|
int buflen = strlen(buf);
|
||||||
for (int i = 0; i <= ((buflen % 2 == 1) ? buflen - 1 : buflen) / 2; i++)
|
for (int i = 0; i <= ((buflen % 2 == 1) ? buflen - 1 : buflen) / 2; i++)
|
||||||
{
|
{
|
||||||
if (buf[i] != buf[buflen - 1 - i])
|
if (buf[i] != buf[buflen - 1 - i])
|
||||||
{
|
{
|
||||||
printf("not palindrome: ");
|
printf("not palindrome: ");
|
||||||
printf(buf);
|
printf(buf);
|
||||||
printf("\n");
|
printf("\n");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int from_stdin()
|
int from_stdin()
|
||||||
{
|
{
|
||||||
char input[SIZE] = {0};
|
char input[SIZE] = {0};
|
||||||
int size = read(STDIN_FILENO, input, SIZE);
|
int size = read(STDIN_FILENO, input, SIZE);
|
||||||
return check(input, size);
|
return check(input, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
int from_file(char *filename)
|
int from_file(char *filename)
|
||||||
{
|
{
|
||||||
FILE *infile;
|
FILE *infile;
|
||||||
char *buffer;
|
char *buffer;
|
||||||
long length;
|
long length;
|
||||||
int result;
|
int result;
|
||||||
|
|
||||||
infile = fopen(filename, "r");
|
infile = fopen(filename, "r");
|
||||||
|
|
||||||
if (infile == NULL)
|
if (infile == NULL)
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
fseek(infile, 0L, SEEK_END);
|
fseek(infile, 0L, SEEK_END);
|
||||||
length = ftell(infile);
|
length = ftell(infile);
|
||||||
|
|
||||||
fseek(infile, 0L, SEEK_SET);
|
fseek(infile, 0L, SEEK_SET);
|
||||||
buffer = calloc(length, sizeof(char));
|
buffer = calloc(length, sizeof(char));
|
||||||
if (buffer == NULL)
|
if (buffer == NULL)
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
length = fread(buffer, sizeof(char), length, infile);
|
length = fread(buffer, sizeof(char), length, infile);
|
||||||
fclose(infile);
|
fclose(infile);
|
||||||
|
|
||||||
result = check(buffer, length);
|
result = check(buffer, length);
|
||||||
free(buffer);
|
free(buffer);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv)
|
int main(int argc, char **argv)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (argc == 1)
|
if (argc == 1)
|
||||||
{
|
{
|
||||||
return from_stdin();
|
return from_stdin();
|
||||||
}
|
}
|
||||||
else if (argc > 1)
|
else if (argc > 1)
|
||||||
{
|
{
|
||||||
return from_file(argv[1]);
|
return from_file(argv[1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user