diff --git a/contrib/deploy-onefuzz-via-azure-devops/.gitignore b/contrib/deploy-onefuzz-via-azure-devops/.gitignore new file mode 100644 index 000000000..1205b5189 --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/.gitignore @@ -0,0 +1,153 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,python +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +### VisualStudioCode ### +.vscode/* +.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python \ No newline at end of file diff --git a/contrib/deploy-onefuzz-via-azure-devops/.pylintrc b/contrib/deploy-onefuzz-via-azure-devops/.pylintrc new file mode 100644 index 000000000..711183520 --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/.pylintrc @@ -0,0 +1,31 @@ +[MASTER] +ignore=CVS .git .hg +init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" + +[MESSAGES CONTROL] +# Pointless whinging. +# W0613 = Unused argument +# R0201 = Method could be a function +# R0801 = Similar lines https://github.com/PyCQA/pylint/issues/214 +# R0901 = Too many ancestors +# R0902 = Too many instance attributes +# R0903 = Too few public methods +# R0904 = Too many public methods +# R0911 = Too many return statements +# R0913 = Too many arguments +# R0914 = Too many local variables +# +# Sometimes disabled depending on how bad a module is. +# C0111 = Missing docstring +# C0112 = Empty docstring + +disable=C0111,R0201,R0801,R0901,R0902,R0903,R0904,R0911,R0913,R0914 + +[FORMAT] +max-line-length=80 + +[REPORTS] +output-format=colorized + +[BASIC] +good-names=i,j,k,_,logger \ No newline at end of file diff --git a/contrib/deploy-onefuzz-via-azure-devops/Pipfile b/contrib/deploy-onefuzz-via-azure-devops/Pipfile new file mode 100644 index 000000000..b799f0f56 --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" + +[requires] +python_version = "3.8" diff --git a/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock b/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock new file mode 100644 index 000000000..d5a8b8c7a --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock @@ -0,0 +1,59 @@ +{ + "_meta": { + "hash": { + "sha256": "acbc8c4e7f2f98f1059b2a93d581ef43f4aa0c9741e64e6253adff8e35fbd99e" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + ], + "version": "==2020.6.20" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "requests": { + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "index": "pypi", + "version": "==2.24.0" + }, + "urllib3": { + "hashes": [ + "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", + "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.25.11" + } + }, + "develop": {} +} diff --git a/contrib/deploy-onefuzz-via-azure-devops/README.md b/contrib/deploy-onefuzz-via-azure-devops/README.md new file mode 100644 index 000000000..37bbedb37 --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/README.md @@ -0,0 +1,11 @@ +# What is this for? + +This section of code contains scripts which help to deploy latest releases of OneFuzz at demand. It uses Azure DevOps Build Pipeline. + +The script [deploy-onefuzz.yml](deploy-onefuzz.yml) can be used saved in Azure DevOps Build Pipeline or can be stored in the repository and can be pointed to it. + +It also contain supporting `python` scripts which helps to fetch latest version and artifacts from OneFuzz GitHub repository. + +# How to use it? + +This script is intended only for deploying newer updates. There are certain set of pipeline variables needs to be set as mentioned in [deploy-onefuzz.yml](deploy-onefuzz.yml) for authentication purposes to the OneFuzz instance. \ No newline at end of file diff --git a/contrib/deploy-onefuzz-via-azure-devops/__init__.py b/contrib/deploy-onefuzz-via-azure-devops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/deploy-onefuzz-via-azure-devops/deploy-onefuzz.yml b/contrib/deploy-onefuzz-via-azure-devops/deploy-onefuzz.yml new file mode 100644 index 000000000..8b87b7cbf --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/deploy-onefuzz.yml @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# It is designed to deploy latest versions in the Azure. There are fixed set of pipeline +# variables which can be used to update onefuzz instances on Azure. +# +# Following the OneFuzz document at +# https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#deploying-an-instance-of-onefuzz +# to deploy OneFuzz on Azure. +# +# List of custom variables: +# | Variable Name | Comments | +# |----------------------|-----------------------------------------------------------| +# |AZURE_CLIENT_ID | The appication ID created by you or the deployment script | +# |AZURE_CLIENT_SECRET | Secret created by App registration process | +# |AZURE_TENANT_ID | Tenant ID of the Azure Subscription | +# |CONTACT_EMAIL_ADDRESS | Email address for communication | +# |ONEFUZZ_DEPLOY_LOC | Deployment Folder location of this script location | +# |ONEFUZZ_INSTANCE_NAME | Instance name of Onefuzz Deployement | +# |ONEFUZZ_SERVICE_URL | OneFuzz service URL. Generally the url defined in App | +# | | Registration | +# |REGION | OneFuzz Region (prefer westus2) | +# |RESOURCE_GROUP_NAME | Resource gorup name for OneFuzz deployment | +# +# Note: Make sure to provide the App owners permission to onefuzz resource group + +--- +trigger: none + +stages: +- stage: Deploy + jobs: + - job: "deploy_oneFuzz" + pool: + vmImage: "ubuntu-latest" + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.8" + + - task: CmdLine@2 + name: onefuzz_release + displayName: "Downloading OneFuzz Artifacts" + inputs: + workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)" + script: | + set -ex + python -m pip install pipenv tox + pipenv install + artifact="artifact" + pipenv run python get_latest_version.py -path $artifact + version="$(pipenv run python get_latest_version.py -version)" + echo "Onefuzz version is $version" + echo "##vso[task.setvariable variable=version;isOutput=true]$version" + echo "##vso[task.setvariable variable=artifact]$artifact" + + - task: CmdLine@2 + displayName: "Installing Dependencies" + inputs: + workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)" + script: | + set -ex + unzip onefuzz-deployment-$(onefuzz_release.version).zip + pip install -r requirements.txt + wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt-get update + sudo apt-get install azure-functions-core-tools-3 + + - task: CmdLine@2 + displayName: "Deploying update to OneFuzz" + inputs: + workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)" + script: | + set -ex + az login --service-principal -u $(ONEFUZZ_SERVICE_URL) -p $(AZURE_CLIENT_SECRET) --tenant $(AZURE_TENANT_ID) + python deploy.py --client_id $(AZURE_CLIENT_ID) --client_secret $(AZURE_CLIENT_SECRET) $REGION $RESOURCE_GROUP_NAME $ONEFUZZ_INSTANCE_NAME $CONTACT_EMAIL_ADDRESS + echo "Deployed Onefuzz $(onefuzz_release.version)" + + - task: CopyFiles@2 + displayName: "Copying cli to Staging area" + inputs: + SourceFolder: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)" + Contents: + "onefuzz-cli-$(onefuzz_release.version).exe" + TargetFolder: $(Build.ArtifactStagingDirectory) + flattenFolders: true + + - task: PublishBuildArtifacts@1 + displayName: "Publish CLI exe" + inputs: + PathtoPublish: "$(Build.ArtifactStagingDirectory)" + ArtifactName: "onefuzz" + publishLocation: "Container" + +- stage: "Verify" + jobs: + - job: "verify_onefuzz_version" + pool: + vmImage: "windows-latest" + variables: + version: $[ stageDependencies.Deploy.deploy_oneFuzz.outputs['onefuzz_release.version'] ] + steps: + - download: current + artifact: onefuzz + + - task: Bash@3 + displayName: Test OneFuzz Deployment version + timeoutInMinutes: 1 + inputs: + workingDirectory: "$(Pipeline.Workspace)/onefuzz" + targetType: 'inline' + script: | + set -ex + ./onefuzz-cli-$(version).exe config --endpoint $(ONEFUZZ_SERVICE_URL) --client_id "$(AZURE_CLIENT_ID)" --client_secret "$(AZURE_CLIENT_SECRET)" + ./onefuzz-cli-$(version).exe --version + until ./onefuzz-cli-$(version).exe versions check --exact; do echo "waiting due to version mismatch"; sleep 1; done diff --git a/contrib/deploy-onefuzz-via-azure-devops/get_latest_version.py b/contrib/deploy-onefuzz-via-azure-devops/get_latest_version.py new file mode 100644 index 000000000..1279b58da --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/get_latest_version.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import argparse +import os +import requests + +BASE_URL = "https://api.github.com/repos/microsoft/onefuzz" + + +class Onefuzz: + def get_latest_version(self): + latest_releasee = requests.get(f"{BASE_URL}/releases/latest").json() + return (latest_releasee["id"], latest_releasee["name"]) + + def list_assets(self, release_id): + assets = requests.get(f"{BASE_URL}/releases/{release_id}/assets").json() + artifacts = [] + for asset in assets: + artifacts.append({"id": asset["id"], "name": asset["name"]}) + return artifacts + + def download_artifact(self, path, asset_id, asset_name): + headers = {"Accept": "application/octet-stream"} + asset = requests.get( + f"{BASE_URL}/releases/assets/{asset_id}", headers=headers + ) + with open(os.path.join(path, asset_name), "wb") as artifact: + artifact.write(asset.content) + + def download_artifacts(self, path, artifacts): + for artifact in artifacts: + self.download_artifact(path, artifact["id"], artifact["name"]) + + def onefuzz_release_artifacts(self, path): + release_id, _ = self.get_latest_version() + artifacts = self.list_assets(release_id) + self.download_artifacts(path, artifacts) + + +def main(): + def dir_path(path): + path = os.path.abspath(path) + if not os.path.isdir(path): + os.makedirs(path) + return path + + parser = argparse.ArgumentParser(description="Download artifacts") + parser.add_argument( + "-path", + type=dir_path, + help="Path to download binaries", + ) + parser.add_argument( + "-version", + action="store_true", + help="Get Onefuzz latest version", + ) + + args = parser.parse_args() + if args.path: + Onefuzz().onefuzz_release_artifacts(args.path) + + if args.version: + print(Onefuzz().get_latest_version()[1]) + + +if __name__ == "__main__": + main() diff --git a/contrib/deploy-onefuzz-via-azure-devops/tox.ini b/contrib/deploy-onefuzz-via-azure-devops/tox.ini new file mode 100644 index 000000000..205e30ea5 --- /dev/null +++ b/contrib/deploy-onefuzz-via-azure-devops/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = py38 +skip_missing_interpreters=True + +[testenv] +usedevelop = True + +deps = + pipenv + +commands = + pipenv install --dev + pipenv run pipenv check