mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-11 01:31:38 +00:00
Deploy update (#2650)
* Remove Python App Function Deployment Code and Code. * Updating yml and zip names. * Fixing ci.yml. * Typo. * Format * Trying to remove python service. * Updating directories. * Removing flag. * Format. * Fixng api-service-net ref. * Re-add requirement. * Fixing refs in bicep. * Specifying version. * Removing dotnet refs in integration tests. * Updating role assignment naming convention. * Adding ignore. * Update src/deny.toml Co-authored-by: George Pollard <porges@porg.es> * Update version. * Removing onefuzztypes dependency. * Switch app service plan to windows. * Update test version. * Changing version. * Returning version. * Trying to add onefuzz types back. * Force pipenv version. * Fix. * Syntax. * Renaming. * Trying different version. * Removing build step. * Fixing bicep parameter. * Retrying run with older version. * Trying pipenv with another version. * Forcing pipenv version in tox. * Adding pipenv fix and updating version. Co-authored-by: George Pollard <porges@porg.es>
This commit is contained in:
Noah McGregor Harper
GitHub
parent
6c981f613d
commit
e0634a3365
59
.github/workflows/ci.yml
vendored
59
.github/workflows/ci.yml
vendored
@ -192,7 +192,7 @@ jobs:
|
||||
black webhook --check
|
||||
isort --profile black webhook
|
||||
flake8 webhook
|
||||
deploy-onefuzz-via-azure-devops:
|
||||
python-safety:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
@ -205,7 +205,7 @@ jobs:
|
||||
set -ex
|
||||
cd contrib/deploy-onefuzz-via-azure-devops
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install tox pipenv
|
||||
python -m pip install tox pipenv==2022.11.11
|
||||
tox
|
||||
onefuzztypes:
|
||||
runs-on: ubuntu-20.04
|
||||
@ -253,54 +253,6 @@ jobs:
|
||||
name: build-artifacts
|
||||
path: artifacts
|
||||
service:
|
||||
needs:
|
||||
- onefuzztypes
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- run: src/ci/set-versions.sh
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.8
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: build-artifacts
|
||||
path: artifacts
|
||||
- name: Build Service
|
||||
run: |
|
||||
cd src/api-service/__app__
|
||||
echo ${GITHUB_RUN_ID} | tee onefuzzlib/build.id
|
||||
echo ${GITHUB_SHA} | tee onefuzzlib/git.version
|
||||
python -m pip install --upgrade pip
|
||||
pip install --target="./.python_packages/lib/site-packages" -r ./requirements.txt ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl
|
||||
zip -r api-service.zip .
|
||||
mkdir -p ${GITHUB_WORKSPACE}/artifacts/service
|
||||
cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: build-artifacts
|
||||
path: artifacts
|
||||
- name: lint
|
||||
run: |
|
||||
cd src/api-service
|
||||
rm -rf __app__/.python_packages
|
||||
python -m pip install --upgrade pip
|
||||
pip install ${GITHUB_WORKSPACE}/artifacts/sdk/onefuzztypes-*.whl
|
||||
pip install -r __app__/requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
pytest tests
|
||||
flake8 .
|
||||
bandit -r ./__app__/
|
||||
black ./__app__/ ./tests --check
|
||||
isort --profile black ./__app__/ ./tests --check
|
||||
mypy __app__ ./tests
|
||||
|
||||
# set a minimum confidence to ignore known false positives
|
||||
vulture --min-confidence 61 __app__
|
||||
|
||||
../ci/disable-py-cache.sh
|
||||
mypy __app__ ./tests
|
||||
service-net:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
@ -359,9 +311,9 @@ jobs:
|
||||
cd src/ApiService/ApiService/
|
||||
mv az-local.settings.json bin/Release/net7.0/local.settings.json
|
||||
cd bin/Release/net7.0/
|
||||
zip -r api-service-net.zip .
|
||||
mkdir -p ${GITHUB_WORKSPACE}/artifacts/service-net
|
||||
cp api-service-net.zip ${GITHUB_WORKSPACE}/artifacts/service-net
|
||||
zip -r api-service.zip .
|
||||
mkdir -p ${GITHUB_WORKSPACE}/artifacts/service
|
||||
cp api-service.zip ${GITHUB_WORKSPACE}/artifacts/service
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: build-artifacts
|
||||
@ -504,7 +456,6 @@ jobs:
|
||||
|
||||
cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/
|
||||
cp artifacts/service/api-service.zip src/deployment
|
||||
cp artifacts/service-net/api-service-net.zip src/deployment
|
||||
cp -r artifacts/third-party src/deployment
|
||||
echo $GITHUB_RUN_ID | tee src/deployment/.build.id
|
||||
echo $GITHUB_SHA | tee src/deployment/.sha
|
||||
|
||||
@ -5,7 +5,7 @@ skip_missing_interpreters=True
|
||||
|
||||
[testenv]
|
||||
deps =
|
||||
pipenv
|
||||
pipenv==2022.11.11
|
||||
|
||||
commands =
|
||||
python -m pip install --upgrade pip
|
||||
|
||||
@ -106,8 +106,7 @@ public sealed class Creds : ICreds {
|
||||
}
|
||||
|
||||
public Uri GetInstanceUrl()
|
||||
// TODO: remove -net when promoted to main version
|
||||
=> new($"https://{GetInstanceName()}-net.azurewebsites.net");
|
||||
=> new($"https://{GetInstanceName()}.azurewebsites.net");
|
||||
|
||||
public record ScaleSetIdentity(string principalId);
|
||||
|
||||
|
||||
@ -81,17 +81,9 @@ def get_instance_url() -> str:
|
||||
return "https://%s.azurewebsites.net" % get_instance_name()
|
||||
|
||||
|
||||
@cached
|
||||
def use_dotnet_agent_functions() -> bool:
|
||||
return os.environ.get("ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS") == "1"
|
||||
|
||||
|
||||
@cached
|
||||
def get_agent_instance_url() -> str:
|
||||
if use_dotnet_agent_functions():
|
||||
return "https://%s-net.azurewebsites.net" % get_instance_name()
|
||||
else:
|
||||
return get_instance_url()
|
||||
return get_instance_url()
|
||||
|
||||
|
||||
@cached
|
||||
|
||||
@ -1790,8 +1790,6 @@ class Onefuzz:
|
||||
client_secret: Optional[str] = None,
|
||||
authority: Optional[str] = None,
|
||||
tenant_domain: Optional[str] = None,
|
||||
_dotnet_endpoint: Optional[str] = None,
|
||||
_dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
|
||||
if endpoint:
|
||||
@ -1804,10 +1802,6 @@ class Onefuzz:
|
||||
self._backend.client_secret = client_secret
|
||||
if tenant_domain is not None:
|
||||
self._backend.config.tenant_domain = tenant_domain
|
||||
if _dotnet_endpoint is not None:
|
||||
self._backend.config.dotnet_endpoint = _dotnet_endpoint
|
||||
if _dotnet_functions is not None:
|
||||
self._backend.config.dotnet_functions = _dotnet_functions
|
||||
|
||||
if self._backend.is_feature_enabled(PreviewFeature.job_templates.name):
|
||||
self.job_templates._load_cache()
|
||||
@ -1851,8 +1845,6 @@ class Onefuzz:
|
||||
client_id: Optional[str] = None,
|
||||
enable_feature: Optional[PreviewFeature] = None,
|
||||
tenant_domain: Optional[str] = None,
|
||||
_dotnet_endpoint: Optional[str] = None,
|
||||
_dotnet_functions: Optional[List[str]] = None,
|
||||
reset: Optional[bool] = None,
|
||||
) -> BackendConfig:
|
||||
"""Configure onefuzz CLI"""
|
||||
@ -1883,10 +1875,6 @@ class Onefuzz:
|
||||
self._backend.enable_feature(enable_feature.name)
|
||||
if tenant_domain is not None:
|
||||
self._backend.config.tenant_domain = tenant_domain
|
||||
if _dotnet_endpoint is not None:
|
||||
self._backend.config.dotnet_endpoint = _dotnet_endpoint
|
||||
if _dotnet_functions is not None:
|
||||
self._backend.config.dotnet_functions = _dotnet_functions
|
||||
self._backend.app = None
|
||||
self._backend.save_config()
|
||||
|
||||
|
||||
@ -96,8 +96,6 @@ class BackendConfig(BaseModel):
|
||||
endpoint: Optional[str]
|
||||
features: Set[str] = Field(default_factory=set)
|
||||
tenant_domain: Optional[str]
|
||||
dotnet_endpoint: Optional[str]
|
||||
dotnet_functions: Optional[List[str]]
|
||||
|
||||
|
||||
class Backend:
|
||||
@ -301,10 +299,7 @@ class Backend:
|
||||
params: Optional[Any] = None,
|
||||
_retry_on_auth_failure: bool = True,
|
||||
) -> Response:
|
||||
if self.config.dotnet_functions and path in self.config.dotnet_functions:
|
||||
endpoint = self.config.dotnet_endpoint
|
||||
else:
|
||||
endpoint = self.config.endpoint
|
||||
endpoint = self.config.endpoint
|
||||
|
||||
if not endpoint:
|
||||
raise Exception("endpoint not configured")
|
||||
|
||||
@ -17,7 +17,7 @@ yanked = "deny"
|
||||
ignore = [
|
||||
"RUSTSEC-2022-0048", # xml-rs is unmaintained
|
||||
"RUSTSEC-2021-0139", # ansi_term is unmaintained
|
||||
"RUSTSEC-2021-0145", # atty bug: we are unaffected (no custom allocator)
|
||||
"RUSTSEC-2021-0145", # waiting for clap upgrade; we are unaffected by the bug (no custom allocator)
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@ -26,14 +26,9 @@ param workbookData object
|
||||
])
|
||||
param diagnosticsLogLevel string = 'Verbose'
|
||||
|
||||
param use_dotnet_agent_functions bool
|
||||
|
||||
var log_retention = 30
|
||||
var tenantId = subscription().tenantId
|
||||
|
||||
var python_functions_disabled = '0'
|
||||
var dotnet_functions_disabled = '1'
|
||||
|
||||
var scaleset_identity = '${name}-scalesetid'
|
||||
|
||||
var StorageBlobDataReader = '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1'
|
||||
@ -80,29 +75,16 @@ module operationalInsights 'bicep-templates/operational-insights.bicep' = {
|
||||
}
|
||||
}
|
||||
|
||||
module linuxServerFarm 'bicep-templates/server-farms.bicep' = {
|
||||
name: 'linux-server-farm'
|
||||
module serverFarm 'bicep-templates/server-farms.bicep' = {
|
||||
name: 'server-farm'
|
||||
params: {
|
||||
server_farm_name: name
|
||||
owner: owner
|
||||
location: location
|
||||
use_windows: false
|
||||
create: true
|
||||
use_windows: true
|
||||
}
|
||||
}
|
||||
|
||||
module dotNetServerFarm 'bicep-templates/server-farms.bicep' = {
|
||||
name: (enable_remote_debugging) ? 'windows-server-farm' : 'same-linux-server-farm'
|
||||
params: {
|
||||
server_farm_name: (enable_remote_debugging) ? '${name}-net' : name
|
||||
owner: owner
|
||||
location: location
|
||||
use_windows: enable_remote_debugging
|
||||
create: enable_remote_debugging
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var keyVaultName = 'of-kv-${uniqueString(resourceGroup().id)}'
|
||||
resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = {
|
||||
name: keyVaultName
|
||||
@ -120,7 +102,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = {
|
||||
}
|
||||
accessPolicies: [
|
||||
{
|
||||
objectId: pythonFunction.outputs.principalId
|
||||
objectId: function.outputs.principalId
|
||||
tenantId: tenantId
|
||||
permissions: {
|
||||
secrets: [
|
||||
@ -169,33 +151,15 @@ module autoscaleSettings 'bicep-templates/autoscale-settings.bicep' = {
|
||||
name: 'autoscaleSettings'
|
||||
params: {
|
||||
location: location
|
||||
server_farm_id: linuxServerFarm.outputs.id
|
||||
server_farm_id: serverFarm.outputs.id
|
||||
owner: owner
|
||||
workspaceId: operationalInsights.outputs.workspaceId
|
||||
logRetention: log_retention
|
||||
autoscale_name: 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}'
|
||||
create_new: true
|
||||
autoscale_name: 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}'
|
||||
function_diagnostics_settings_name: 'functionDiagnosticSettings'
|
||||
}
|
||||
}
|
||||
|
||||
module autoscaleSettingsNet 'bicep-templates/autoscale-settings.bicep' = {
|
||||
name: 'autoscaleSettingsNet'
|
||||
params: {
|
||||
location: location
|
||||
server_farm_id: dotNetServerFarm.outputs.id
|
||||
owner: owner
|
||||
workspaceId: operationalInsights.outputs.workspaceId
|
||||
logRetention: log_retention
|
||||
autoscale_name: (enable_remote_debugging) ? 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}-net' : 'onefuzz-autoscale-${uniqueString(resourceGroup().id)}'
|
||||
create_new: enable_remote_debugging
|
||||
function_diagnostics_settings_name: (enable_remote_debugging) ? 'functionDiagnosticSettings' : 'functionDiagnosticsSettingsNet'
|
||||
}
|
||||
dependsOn: [
|
||||
autoscaleSettings
|
||||
]
|
||||
}
|
||||
|
||||
module eventGrid 'bicep-templates/event-grid.bicep' = {
|
||||
name: 'event-grid'
|
||||
params:{
|
||||
@ -210,22 +174,22 @@ module eventGrid 'bicep-templates/event-grid.bicep' = {
|
||||
}
|
||||
|
||||
// try to make role assignments to deploy as late as possible in order to have principalId ready
|
||||
resource roleAssigmentsPy 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: {
|
||||
name: guid('${resourceGroup().id}${r.suffix}-python')
|
||||
resource roleAssignments 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: {
|
||||
name: guid('${resourceGroup().id}${r.suffix}-1f')
|
||||
properties: {
|
||||
roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}'
|
||||
principalId: pythonFunction.outputs.principalId
|
||||
principalId: function.outputs.principalId
|
||||
}
|
||||
dependsOn: [
|
||||
eventGrid
|
||||
keyVault
|
||||
linuxServerFarm
|
||||
serverFarm
|
||||
]
|
||||
}]
|
||||
|
||||
// try to make role assignments to deploy as late as possible in order to have principalId ready
|
||||
resource roleAssigmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: {
|
||||
name: guid('${resourceGroup().id}${r.suffix}-net')
|
||||
resource roleAssignmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: {
|
||||
name: guid('${resourceGroup().id}${r.suffix}-1f-net')
|
||||
properties: {
|
||||
roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}'
|
||||
principalId: netFunction.outputs.principalId
|
||||
@ -233,7 +197,7 @@ resource roleAssigmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-p
|
||||
dependsOn: [
|
||||
eventGrid
|
||||
keyVault
|
||||
dotNetServerFarm
|
||||
serverFarm
|
||||
]
|
||||
}]
|
||||
|
||||
@ -248,30 +212,29 @@ resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10
|
||||
dependsOn: [
|
||||
eventGrid
|
||||
keyVault
|
||||
linuxServerFarm
|
||||
dotNetServerFarm
|
||||
]
|
||||
serverFarm
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
module pythonFunction 'bicep-templates/function.bicep' = {
|
||||
name: 'pythonFunction'
|
||||
module function 'bicep-templates/function.bicep' = {
|
||||
name: 'function'
|
||||
params: {
|
||||
name: name
|
||||
linux_fx_version: 'Python|3.8'
|
||||
linux_fx_version: 'DOTNET-ISOLATED|7.0'
|
||||
|
||||
app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs
|
||||
app_func_audiences: app_func_audiences
|
||||
app_func_issuer: app_func_issuer
|
||||
|
||||
client_id: clientId
|
||||
diagnostics_log_level: diagnosticsLogLevel
|
||||
location: location
|
||||
log_retention: log_retention
|
||||
owner: owner
|
||||
server_farm_id: linuxServerFarm.outputs.id
|
||||
client_id: clientId
|
||||
use_windows: false
|
||||
enable_remote_debugging: false
|
||||
server_farm_id: serverFarm.outputs.id
|
||||
|
||||
use_windows: true
|
||||
enable_remote_debugging: enable_remote_debugging
|
||||
}
|
||||
}
|
||||
|
||||
@ -289,20 +252,20 @@ module netFunction 'bicep-templates/function.bicep' = {
|
||||
location: location
|
||||
log_retention: log_retention
|
||||
owner: owner
|
||||
server_farm_id: dotNetServerFarm.outputs.id
|
||||
server_farm_id: serverFarm.outputs.id
|
||||
|
||||
use_windows: enable_remote_debugging
|
||||
use_windows: true
|
||||
enable_remote_debugging: enable_remote_debugging
|
||||
}
|
||||
}
|
||||
|
||||
module pythonFunctionSettings 'bicep-templates/function-settings.bicep' = {
|
||||
name: 'pythonFunctionSettings'
|
||||
module functionSettings 'bicep-templates/function-settings.bicep' = {
|
||||
name: 'functionSettings'
|
||||
params: {
|
||||
name: name
|
||||
owner: owner
|
||||
functions_worker_runtime: 'python'
|
||||
functions_extension_version: '~3'
|
||||
functions_worker_runtime: 'dotnet-isolated'
|
||||
functions_extension_version: '~4'
|
||||
instance_name: name
|
||||
app_insights_app_id: operationalInsights.outputs.appInsightsAppId
|
||||
app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey
|
||||
@ -314,50 +277,10 @@ module pythonFunctionSettings 'bicep-templates/function-settings.bicep' = {
|
||||
keyvault_name: keyVaultName
|
||||
monitor_account_name: operationalInsights.outputs.monitorAccountName
|
||||
multi_tenant_domain: multi_tenant_domain
|
||||
functions_disabled: python_functions_disabled
|
||||
use_dotnet_agent_functions: use_dotnet_agent_functions
|
||||
enable_profiler: false
|
||||
all_function_names: [
|
||||
'agent_can_schedule' //0
|
||||
'agent_commands' //1
|
||||
'agent_events' //2
|
||||
'agent_registration' //3
|
||||
'containers' //4
|
||||
'download' //5
|
||||
'info' //6
|
||||
'instance_config' //7
|
||||
'jobs' //8
|
||||
'job_templates' //9
|
||||
'job_templates_manage' //10
|
||||
'negotiate' //11
|
||||
'node' //12
|
||||
'node_add_ssh_key' //13
|
||||
'notifications' //14
|
||||
'pool' //15
|
||||
'proxy' //16
|
||||
'queue_file_changes' //17
|
||||
'queue_node_heartbeat' //18
|
||||
'queue_proxy_update' //19
|
||||
'queue_signalr_events' //20
|
||||
'queue_task_heartbeat' //21
|
||||
'queue_updates' //22
|
||||
'queue_webhooks' //23
|
||||
'repro_vms' //24
|
||||
'scaleset' //25
|
||||
'tasks' //26
|
||||
'timer_daily' //27
|
||||
'timer_proxy' //28
|
||||
'timer_repro' //29
|
||||
'timer_retention' //30
|
||||
'timer_tasks' //31
|
||||
'timer_workers' //32
|
||||
'webhooks' //33
|
||||
'webhooks_logs' //34
|
||||
'webhooks_ping' //35
|
||||
]
|
||||
enable_profiler: enable_profiler
|
||||
}
|
||||
dependsOn: [
|
||||
pythonFunction
|
||||
function
|
||||
]
|
||||
}
|
||||
|
||||
@ -367,60 +290,20 @@ module netFunctionSettings 'bicep-templates/function-settings.bicep' = {
|
||||
params: {
|
||||
owner: owner
|
||||
name: '${name}-net'
|
||||
signal_r_connection_string: signalR.outputs.connectionString
|
||||
app_insights_app_id: operationalInsights.outputs.appInsightsAppId
|
||||
app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey
|
||||
functions_worker_runtime: 'dotnet-isolated'
|
||||
functions_extension_version: '~4'
|
||||
instance_name: name
|
||||
app_insights_app_id: operationalInsights.outputs.appInsightsAppId
|
||||
app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey
|
||||
client_secret: clientSecret
|
||||
signal_r_connection_string: signalR.outputs.connectionString
|
||||
func_sas_url: storage.outputs.FuncSasUrl
|
||||
func_storage_resource_id: storage.outputs.FuncId
|
||||
fuzz_storage_resource_id: storage.outputs.FuzzId
|
||||
keyvault_name: keyVaultName
|
||||
monitor_account_name: operationalInsights.outputs.monitorAccountName
|
||||
multi_tenant_domain: multi_tenant_domain
|
||||
functions_disabled: dotnet_functions_disabled
|
||||
use_dotnet_agent_functions: false // this doesn’t do anything on the .NET service
|
||||
enable_profiler: enable_profiler
|
||||
all_function_names: [
|
||||
'AgentCanSchedule' //0
|
||||
'AgentCommands' //1
|
||||
'AgentEvents' //2
|
||||
'AgentRegistration' //3
|
||||
'Containers' //4
|
||||
'Download' //5
|
||||
'Info' //6
|
||||
'InstanceConfig' //7
|
||||
'Jobs' //8
|
||||
'JobTemplates' //9
|
||||
'JobTemplatesManage' //10
|
||||
'Negotiate' //11
|
||||
'Node' //12
|
||||
'NodeAddSshKey' //13
|
||||
'Notifications' //14
|
||||
'Pool' //15
|
||||
'Proxy' //16
|
||||
'QueueFileChanges' //17
|
||||
'QueueNodeHeartbeat' //18
|
||||
'QueueProxyUpdate' //19
|
||||
'QueueSignalrEvents' //20
|
||||
'QueueTaskHeartbeat' //21
|
||||
'QueueUpdates' //22
|
||||
'QueueWebhooks' //23
|
||||
'ReproVms' //24
|
||||
'Scaleset' //25
|
||||
'Tasks' //26
|
||||
'TimerDaily' //27
|
||||
'TimerProxy' //28
|
||||
'TimerRepro' //29
|
||||
'TimerRetention' //30
|
||||
'TimerTasks' //31
|
||||
'TimerWorkers' //32
|
||||
'Webhooks' //33
|
||||
'WebhooksLogs' //34
|
||||
'WebhooksPing' //35
|
||||
]
|
||||
}
|
||||
dependsOn: [
|
||||
netFunction
|
||||
|
||||
@ -5,10 +5,9 @@ param workspaceId string
|
||||
param logRetention int
|
||||
param autoscale_name string
|
||||
param function_diagnostics_settings_name string
|
||||
param create_new bool
|
||||
|
||||
|
||||
resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = if (create_new) {
|
||||
resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = {
|
||||
name: autoscale_name
|
||||
location: location
|
||||
properties: {
|
||||
@ -72,7 +71,7 @@ resource autoscaleSettings 'Microsoft.Insights/autoscalesettings@2015-04-01' = i
|
||||
}
|
||||
}
|
||||
|
||||
resource functionDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (create_new) {
|
||||
resource functionDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = {
|
||||
name: function_diagnostics_settings_name
|
||||
scope: autoscaleSettings
|
||||
properties: {
|
||||
|
||||
@ -26,29 +26,14 @@ param monitor_account_name string
|
||||
param functions_worker_runtime string
|
||||
param functions_extension_version string
|
||||
|
||||
param functions_disabled string
|
||||
param use_dotnet_agent_functions bool
|
||||
|
||||
param all_function_names array
|
||||
|
||||
param enable_profiler bool
|
||||
|
||||
var disabledFunctionName = 'disabledFunctions-${functions_worker_runtime}'
|
||||
|
||||
var telemetry = 'd7a73cf4-5a1a-4030-85e1-e5b25867e45a'
|
||||
|
||||
resource function 'Microsoft.Web/sites@2021-02-01' existing = {
|
||||
name: name
|
||||
}
|
||||
|
||||
module disabledFunctions 'function-settings-disabled-apps.bicep' = {
|
||||
name: disabledFunctionName
|
||||
params:{
|
||||
functions_disabled_setting: functions_disabled
|
||||
allFunctions: all_function_names
|
||||
}
|
||||
}
|
||||
|
||||
var enable_profilers = enable_profiler ? {
|
||||
APPINSIGHTS_PROFILERFEATURE_VERSION : '1.0.0'
|
||||
DiagnosticServices_EXTENSION_VERSION: '~3'
|
||||
@ -78,6 +63,5 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = {
|
||||
ONEFUZZ_KEYVAULT: keyvault_name
|
||||
ONEFUZZ_OWNER: owner
|
||||
ONEFUZZ_CLIENT_SECRET: client_secret
|
||||
ONEFUZZ_USE_DOTNET_AGENT_FUNCTIONS: use_dotnet_agent_functions ? '1' : '0'
|
||||
}, disabledFunctions.outputs.appSettings, enable_profilers)
|
||||
}, enable_profilers)
|
||||
}
|
||||
|
||||
@ -2,11 +2,10 @@ param server_farm_name string
|
||||
param owner string
|
||||
param location string
|
||||
param use_windows bool
|
||||
param create bool
|
||||
|
||||
var kind = (use_windows) ? 'app' : 'linux'
|
||||
|
||||
resource serverFarms 'Microsoft.Web/serverfarms@2022-03-01' = if (create) {
|
||||
resource serverFarms 'Microsoft.Web/serverfarms@2022-03-01' = {
|
||||
name: server_farm_name
|
||||
location: location
|
||||
kind: kind
|
||||
|
||||
@ -4,7 +4,6 @@
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@ -95,12 +94,7 @@ FUNC_TOOLS_ERROR = (
|
||||
)
|
||||
|
||||
DOTNET_APPLICATION_SUFFIX = "-net"
|
||||
DOTNET_AGENT_FUNCTIONS = [
|
||||
"agent_can_schedule",
|
||||
"agent_commands",
|
||||
"agent_events",
|
||||
"agent_registration",
|
||||
]
|
||||
|
||||
logger = logging.getLogger("deploy")
|
||||
|
||||
|
||||
@ -145,7 +139,6 @@ class Client:
|
||||
client_id: Optional[str],
|
||||
client_secret: Optional[str],
|
||||
app_zip: str,
|
||||
app_net_zip: str,
|
||||
tools: str,
|
||||
instance_specific: str,
|
||||
third_party: str,
|
||||
@ -159,8 +152,6 @@ class Client:
|
||||
subscription_id: Optional[str],
|
||||
admins: List[UUID],
|
||||
allowed_aad_tenants: List[UUID],
|
||||
enable_dotnet: List[str],
|
||||
use_dotnet_agent_functions: bool,
|
||||
cli_app_id: str,
|
||||
auto_create_cli_app: bool,
|
||||
host_dotnet_on_windows: bool,
|
||||
@ -173,7 +164,6 @@ class Client:
|
||||
self.owner = owner
|
||||
self.nsg_config = nsg_config
|
||||
self.app_zip = app_zip
|
||||
self.app_net_zip = app_net_zip
|
||||
self.tools = tools
|
||||
self.instance_specific = instance_specific
|
||||
self.third_party = third_party
|
||||
@ -195,8 +185,6 @@ class Client:
|
||||
|
||||
self.arm_template = bicep_to_arm(bicep_template)
|
||||
|
||||
self.enable_dotnet = enable_dotnet
|
||||
self.use_dotnet_agent_functions = use_dotnet_agent_functions
|
||||
self.cli_app_id = cli_app_id
|
||||
self.auto_create_cli_app = auto_create_cli_app
|
||||
self.host_dotnet_on_windows = host_dotnet_on_windows
|
||||
@ -700,7 +688,6 @@ class Client:
|
||||
"signedExpiry": {"value": expiry},
|
||||
"multi_tenant_domain": multi_tenant_domain,
|
||||
"workbookData": {"value": self.workbook_data},
|
||||
"use_dotnet_agent_functions": {"value": self.use_dotnet_agent_functions},
|
||||
"enable_remote_debugging": {"value": self.host_dotnet_on_windows},
|
||||
"enable_profiler": {"value": self.enable_profiler},
|
||||
}
|
||||
@ -1122,8 +1109,9 @@ class Client:
|
||||
"functionapp",
|
||||
"publish",
|
||||
self.application_name,
|
||||
"--python",
|
||||
"--no-build",
|
||||
"--dotnet-version",
|
||||
"7.0",
|
||||
],
|
||||
env=dict(os.environ, CLI_DEBUG="1"),
|
||||
cwd=tmpdirname,
|
||||
@ -1142,9 +1130,9 @@ class Client:
|
||||
raise error
|
||||
|
||||
def deploy_dotnet_app(self) -> None:
|
||||
logger.info("deploying function app %s ", self.app_net_zip)
|
||||
logger.info("deploying function app %s ", self.app_zip)
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
with zipfile.ZipFile(self.app_net_zip, "r") as zip_ref:
|
||||
with zipfile.ZipFile(self.app_zip, "r") as zip_ref:
|
||||
func = shutil.which("func")
|
||||
assert func is not None
|
||||
|
||||
@ -1180,99 +1168,6 @@ class Client:
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
def enable_dotnet_func(self) -> None:
|
||||
if self.enable_dotnet:
|
||||
|
||||
def expand_agent(f: str) -> List[str]:
|
||||
# 'agent' is permitted as a shortcut for the agent functions
|
||||
if f == "agent":
|
||||
return DOTNET_AGENT_FUNCTIONS
|
||||
else:
|
||||
return [f]
|
||||
|
||||
enable_dotnet = itertools.chain.from_iterable(
|
||||
map(expand_agent, self.enable_dotnet)
|
||||
)
|
||||
|
||||
python_settings = []
|
||||
dotnet_settings = []
|
||||
|
||||
for function_name in enable_dotnet:
|
||||
format_name = function_name.split("_")
|
||||
dotnet_name = "".join(x.title() for x in format_name)
|
||||
# keep the python versions of http function to allow the service to be backward compatible
|
||||
# with older version of the CLI and the agents
|
||||
if function_name.startswith("queue_") or function_name.startswith(
|
||||
"timer_"
|
||||
):
|
||||
logger.info(f"disabling PYTHON function: {function_name}")
|
||||
disable_python = "1"
|
||||
else:
|
||||
logger.info(f"enabling PYTHON function: {function_name}")
|
||||
disable_python = "0"
|
||||
|
||||
python_settings.append(
|
||||
f"AzureWebJobs.{function_name}.Disabled={disable_python}"
|
||||
)
|
||||
|
||||
# enable dotnet function
|
||||
logger.info(f"enabling DOTNET function: {dotnet_name}")
|
||||
dotnet_settings.append(f"AzureWebJobs.{dotnet_name}.Disabled=0")
|
||||
|
||||
func = shutil.which("az")
|
||||
assert func is not None
|
||||
|
||||
max_tries = 5
|
||||
error: Optional[subprocess.CalledProcessError] = None
|
||||
for i in range(max_tries):
|
||||
try:
|
||||
logger.info("updating Python settings")
|
||||
subprocess.check_output(
|
||||
[
|
||||
func,
|
||||
"functionapp",
|
||||
"config",
|
||||
"appsettings",
|
||||
"set",
|
||||
"--name",
|
||||
self.application_name,
|
||||
"--resource-group",
|
||||
self.resource_group,
|
||||
"--settings",
|
||||
]
|
||||
+ python_settings,
|
||||
env=dict(os.environ, CLI_DEBUG="1"),
|
||||
)
|
||||
logger.info("updating .NET settings")
|
||||
subprocess.check_output(
|
||||
[
|
||||
func,
|
||||
"functionapp",
|
||||
"config",
|
||||
"appsettings",
|
||||
"set",
|
||||
"--name",
|
||||
self.application_name + DOTNET_APPLICATION_SUFFIX,
|
||||
"--resource-group",
|
||||
self.resource_group,
|
||||
"--settings",
|
||||
]
|
||||
+ dotnet_settings,
|
||||
env=dict(os.environ, CLI_DEBUG="1"),
|
||||
)
|
||||
break
|
||||
except subprocess.CalledProcessError as err:
|
||||
error = err
|
||||
if i + 1 < max_tries:
|
||||
logger.debug("func failure error: %s", err)
|
||||
logger.warning(
|
||||
"unable to update settings, waiting 60 seconds and trying again"
|
||||
)
|
||||
time.sleep(60)
|
||||
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
def update_registration(self) -> None:
|
||||
if not self.create_registration:
|
||||
return
|
||||
@ -1336,7 +1231,6 @@ def main() -> None:
|
||||
("dotnet-api", Client.deploy_dotnet_app),
|
||||
("export_appinsights", Client.add_log_export),
|
||||
("update_registration", Client.update_registration),
|
||||
("enable_dotnet", Client.enable_dotnet_func),
|
||||
]
|
||||
|
||||
formatter = argparse.ArgumentDefaultsHelpFormatter
|
||||
@ -1364,12 +1258,6 @@ def main() -> None:
|
||||
default="api-service.zip",
|
||||
help="(default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--app-net-zip",
|
||||
type=arg_file,
|
||||
default="api-service-net.zip",
|
||||
help="(default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tools", type=arg_dir, default="tools", help="(default: %(default)s)"
|
||||
)
|
||||
@ -1447,20 +1335,6 @@ def main() -> None:
|
||||
nargs="*",
|
||||
help="Set additional AAD tenants beyond the tenant the app is deployed in",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable_dotnet",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=[],
|
||||
help="Provide a space-seperated list of python function names to disable "
|
||||
"their functions and enable corresponding dotnet functions in the Azure "
|
||||
"Function App deployment",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use_dotnet_agent_functions",
|
||||
action="store_true",
|
||||
help="Tell the OneFuzz agent to use the dotnet endpoint",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cli_app_id",
|
||||
type=str,
|
||||
@ -1499,7 +1373,6 @@ def main() -> None:
|
||||
client_id=args.client_id,
|
||||
client_secret=args.client_secret,
|
||||
app_zip=args.app_zip,
|
||||
app_net_zip=args.app_net_zip,
|
||||
tools=args.tools,
|
||||
instance_specific=args.instance_specific,
|
||||
third_party=args.third_party,
|
||||
@ -1513,8 +1386,6 @@ def main() -> None:
|
||||
subscription_id=args.subscription_id,
|
||||
admins=args.set_admins,
|
||||
allowed_aad_tenants=args.allowed_aad_tenants or [],
|
||||
enable_dotnet=args.enable_dotnet,
|
||||
use_dotnet_agent_functions=args.use_dotnet_agent_functions,
|
||||
cli_app_id=args.cli_app_id,
|
||||
auto_create_cli_app=args.auto_create_cli_app,
|
||||
host_dotnet_on_windows=args.host_dotnet_on_windows,
|
||||
@ -1529,17 +1400,6 @@ def main() -> None:
|
||||
|
||||
logging.getLogger("deploy").setLevel(logging.INFO)
|
||||
|
||||
if args.use_dotnet_agent_functions:
|
||||
# validate that the agent functions are actually enabled
|
||||
if not (
|
||||
"agent" in args.enable_dotnet
|
||||
or all(map(lambda f: f in args.enable_dotnet, DOTNET_AGENT_FUNCTIONS))
|
||||
):
|
||||
logger.error(
|
||||
"If --use_dotnet_agent_functions is set, all agent functions must be enabled (--enable_dotnet agent)."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if args.rbac_only:
|
||||
logger.warning(
|
||||
"'rbac_only' specified. The deployment will execute "
|
||||
|
||||
@ -988,16 +988,12 @@ class Run(Command):
|
||||
poll: bool = False,
|
||||
stop_on_complete_check: bool = False,
|
||||
job_ids: List[UUID] = [],
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
self.onefuzz.__setup__(
|
||||
endpoint=endpoint,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
tester = TestOnefuzz(self.onefuzz, self.logger, test_id)
|
||||
result = tester.check_jobs(
|
||||
@ -1015,16 +1011,12 @@ class Run(Command):
|
||||
client_secret: Optional[str],
|
||||
authority: Optional[str] = None,
|
||||
job_ids: List[UUID] = [],
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
self.onefuzz.__setup__(
|
||||
endpoint=endpoint,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
tester = TestOnefuzz(self.onefuzz, self.logger, test_id)
|
||||
launch_result, repros = tester.launch_repro(job_ids=job_ids)
|
||||
@ -1043,15 +1035,10 @@ class Run(Command):
|
||||
region: Optional[Region] = None,
|
||||
os_list: List[OS] = [OS.linux, OS.windows],
|
||||
test_id: Optional[UUID] = None,
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
if test_id is None:
|
||||
test_id = uuid4()
|
||||
self.logger.info("launching test_id: %s", test_id)
|
||||
self.logger.info(
|
||||
"dotnet configuration: %s, %s", dotnet_endpoint, dotnet_functions
|
||||
)
|
||||
|
||||
def try_setup(data: Any) -> None:
|
||||
self.onefuzz.__setup__(
|
||||
@ -1059,8 +1046,6 @@ class Run(Command):
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
|
||||
retry(self.logger, try_setup, "trying to configure")
|
||||
@ -1080,8 +1065,6 @@ class Run(Command):
|
||||
targets: List[str] = list(TARGETS.keys()),
|
||||
test_id: Optional[UUID] = None,
|
||||
duration: int = 1,
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
if test_id is None:
|
||||
test_id = uuid4()
|
||||
@ -1093,8 +1076,6 @@ class Run(Command):
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
|
||||
retry(self.logger, try_setup, "trying to configure")
|
||||
@ -1116,16 +1097,12 @@ class Run(Command):
|
||||
authority: Optional[str],
|
||||
client_id: Optional[str],
|
||||
client_secret: Optional[str],
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
self.onefuzz.__setup__(
|
||||
endpoint=endpoint,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
tester = TestOnefuzz(self.onefuzz, self.logger, test_id=test_id)
|
||||
tester.cleanup()
|
||||
@ -1138,16 +1115,12 @@ class Run(Command):
|
||||
authority: Optional[str] = None,
|
||||
client_id: Optional[str],
|
||||
client_secret: Optional[str],
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
self.onefuzz.__setup__(
|
||||
endpoint=endpoint,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
tester = TestOnefuzz(self.onefuzz, self.logger, test_id=test_id)
|
||||
tester.check_logs_for_errors()
|
||||
@ -1162,8 +1135,6 @@ class Run(Command):
|
||||
skip_repro: bool = False,
|
||||
test_id: UUID,
|
||||
job_ids: List[UUID] = [],
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
|
||||
self.check_jobs(
|
||||
@ -1175,8 +1146,6 @@ class Run(Command):
|
||||
poll=True,
|
||||
stop_on_complete_check=True,
|
||||
job_ids=job_ids,
|
||||
dotnet_endpoint=dotnet_endpoint,
|
||||
dotnet_functions=dotnet_functions,
|
||||
)
|
||||
|
||||
if skip_repro:
|
||||
@ -1189,8 +1158,6 @@ class Run(Command):
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
job_ids=job_ids,
|
||||
dotnet_endpoint=dotnet_endpoint,
|
||||
dotnet_functions=dotnet_functions,
|
||||
)
|
||||
|
||||
def test(
|
||||
@ -1207,8 +1174,6 @@ class Run(Command):
|
||||
targets: List[str] = list(TARGETS.keys()),
|
||||
skip_repro: bool = False,
|
||||
duration: int = 1,
|
||||
dotnet_endpoint: Optional[str] = None,
|
||||
dotnet_functions: Optional[List[str]] = None,
|
||||
) -> None:
|
||||
success = True
|
||||
|
||||
@ -1222,8 +1187,6 @@ class Run(Command):
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
_dotnet_endpoint=dotnet_endpoint,
|
||||
_dotnet_functions=dotnet_functions,
|
||||
)
|
||||
|
||||
retry(self.logger, try_setup, "trying to configure")
|
||||
@ -1258,8 +1221,6 @@ class Run(Command):
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
authority=authority,
|
||||
dotnet_endpoint=dotnet_endpoint,
|
||||
dotnet_functions=dotnet_functions,
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error("testing failed: %s", repr(e))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user