ADO Pipeline to update a onefuzz instance with the latest release (#208)

contrib/deploy-onefuzz-via-azure-devops/.gitignore

@@ -0,0 +1,153 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,python
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+pytestdebug.log
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+doc/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+### VisualStudioCode ###
+.vscode/*
+.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python

contrib/deploy-onefuzz-via-azure-devops/.pylintrc

@@ -0,0 +1,31 @@
+[MASTER]
+ignore=CVS .git .hg
+init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))"
+
+[MESSAGES CONTROL]
+# Pointless whinging.
+#   W0613 = Unused argument
+#   R0201 = Method could be a function
+#   R0801 = Similar lines https://github.com/PyCQA/pylint/issues/214
+#   R0901 = Too many ancestors
+#   R0902 = Too many instance attributes
+#   R0903 = Too few public methods
+#   R0904 = Too many public methods
+#   R0911 = Too many return statements
+#   R0913 = Too many arguments
+#   R0914 = Too many local variables
+#
+# Sometimes disabled depending on how bad a module is.
+#   C0111 = Missing docstring
+#   C0112 = Empty docstring
+
+disable=C0111,R0201,R0801,R0901,R0902,R0903,R0904,R0911,R0913,R0914
+
+[FORMAT]
+max-line-length=80
+
+[REPORTS]
+output-format=colorized
+
+[BASIC]
+good-names=i,j,k,_,logger

contrib/deploy-onefuzz-via-azure-devops/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+
+[requires]
+python_version = "3.8"

contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock

@@ -0,0 +1,59 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "acbc8c4e7f2f98f1059b2a93d581ef43f4aa0c9741e64e6253adff8e35fbd99e"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.8"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
+                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+            ],
+            "version": "==2020.6.20"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
+                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+            ],
+            "index": "pypi",
+            "version": "==2.24.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
+                "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.25.11"
+        }
+    },
+    "develop": {}
+}

contrib/deploy-onefuzz-via-azure-devops/README.md

@@ -0,0 +1,11 @@
+# What is this for?
+
+This section of code contains scripts which help to deploy latest releases of OneFuzz at demand. It uses Azure DevOps Build Pipeline.
+
+The script [deploy-onefuzz.yml](deploy-onefuzz.yml) can be used saved in Azure DevOps Build Pipeline or can be stored in the repository and can be pointed to it.
+
+It also contain supporting `python` scripts which helps to fetch latest version and artifacts from OneFuzz GitHub repository.
+
+# How to use it?
+
+This script is intended only for deploying newer updates. There are certain set of pipeline variables needs to be set as mentioned in [deploy-onefuzz.yml](deploy-onefuzz.yml) for authentication purposes to the OneFuzz instance.

contrib/deploy-onefuzz-via-azure-devops/deploy-onefuzz.yml

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# It is designed to deploy latest versions in the Azure. There are fixed set of pipeline
+# variables which can be used to update onefuzz instances on Azure.
+#
+# Following the OneFuzz document at
+# https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#deploying-an-instance-of-onefuzz
+# to deploy OneFuzz on Azure.
+#
+# List of custom variables:
+# | Variable Name        | Comments                                                  |
+# |----------------------|-----------------------------------------------------------|
+# |AZURE_CLIENT_ID       | The appication ID created by you or the deployment script |
+# |AZURE_CLIENT_SECRET   | Secret created by App registration process                |
+# |AZURE_TENANT_ID       | Tenant ID of the Azure Subscription                       |
+# |CONTACT_EMAIL_ADDRESS | Email address for communication                           |
+# |ONEFUZZ_DEPLOY_LOC    | Deployment Folder location of this script location        |
+# |ONEFUZZ_INSTANCE_NAME | Instance name of Onefuzz Deployement                      |
+# |ONEFUZZ_SERVICE_URL   | OneFuzz service URL. Generally the url defined in App     |
+# |                      | Registration                                              |
+# |REGION                | OneFuzz Region (prefer westus2)                           |
+# |RESOURCE_GROUP_NAME   | Resource gorup name for OneFuzz deployment                |                        
+#
+# Note: Make sure to provide the App owners permission to onefuzz resource group
+
+---
+trigger: none
+
+stages:
+- stage: Deploy
+  jobs:
+    - job: "deploy_oneFuzz"
+      pool:
+        vmImage: "ubuntu-latest"
+      steps:
+        - task: UsePythonVersion@0
+          inputs:
+            versionSpec: "3.8"
+
+        - task: CmdLine@2
+          name: onefuzz_release
+          displayName: "Downloading OneFuzz Artifacts"
+          inputs:
+            workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)"
+            script: |
+              set -ex
+              python -m pip install pipenv tox
+              pipenv install
+              artifact="artifact"
+              pipenv run python get_latest_version.py -path $artifact
+              version="$(pipenv run python get_latest_version.py -version)"
+              echo "Onefuzz version is $version"
+              echo "##vso[task.setvariable variable=version;isOutput=true]$version"
+              echo "##vso[task.setvariable variable=artifact]$artifact"
+
+        - task: CmdLine@2
+          displayName: "Installing Dependencies"
+          inputs:
+            workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)"
+            script: |
+              set -ex
+              unzip onefuzz-deployment-$(onefuzz_release.version).zip
+              pip install -r requirements.txt
+              wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb
+              sudo dpkg -i packages-microsoft-prod.deb
+              sudo apt-get update
+              sudo apt-get install azure-functions-core-tools-3
+
+        - task: CmdLine@2
+          displayName: "Deploying update to OneFuzz"
+          inputs:
+            workingDirectory: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)"
+            script: |
+              set -ex
+              az login --service-principal -u $(ONEFUZZ_SERVICE_URL) -p $(AZURE_CLIENT_SECRET) --tenant $(AZURE_TENANT_ID)
+              python deploy.py --client_id $(AZURE_CLIENT_ID) --client_secret $(AZURE_CLIENT_SECRET) $REGION $RESOURCE_GROUP_NAME $ONEFUZZ_INSTANCE_NAME $CONTACT_EMAIL_ADDRESS
+              echo "Deployed Onefuzz $(onefuzz_release.version)"
+
+        - task: CopyFiles@2
+          displayName: "Copying cli to Staging area"
+          inputs:
+            SourceFolder: "$(ONEFUZZ_DEPLOY_LOC)/$(artifact)"
+            Contents:
+              "onefuzz-cli-$(onefuzz_release.version).exe"
+            TargetFolder: $(Build.ArtifactStagingDirectory)
+            flattenFolders: true
+
+        - task: PublishBuildArtifacts@1
+          displayName: "Publish CLI exe"
+          inputs:
+            PathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            ArtifactName: "onefuzz"
+            publishLocation: "Container"
+
+- stage: "Verify"
+  jobs:
+  - job: "verify_onefuzz_version"
+    pool:
+      vmImage: "windows-latest"
+    variables:
+      version: $[ stageDependencies.Deploy.deploy_oneFuzz.outputs['onefuzz_release.version'] ]
+    steps:
+      - download: current
+        artifact: onefuzz
+
+      - task: Bash@3
+        displayName: Test OneFuzz Deployment version
+        timeoutInMinutes: 1
+        inputs:
+          workingDirectory: "$(Pipeline.Workspace)/onefuzz"
+          targetType: 'inline'
+          script: |
+            set -ex
+            ./onefuzz-cli-$(version).exe config --endpoint $(ONEFUZZ_SERVICE_URL) --client_id "$(AZURE_CLIENT_ID)" --client_secret "$(AZURE_CLIENT_SECRET)"
+            ./onefuzz-cli-$(version).exe --version
+            until ./onefuzz-cli-$(version).exe versions check --exact; do echo "waiting due to version mismatch"; sleep 1; done

contrib/deploy-onefuzz-via-azure-devops/get_latest_version.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import argparse
+import os
+import requests
+
+BASE_URL = "https://api.github.com/repos/microsoft/onefuzz"
+
+
+class Onefuzz:
+    def get_latest_version(self):
+        latest_releasee = requests.get(f"{BASE_URL}/releases/latest").json()
+        return (latest_releasee["id"], latest_releasee["name"])
+
+    def list_assets(self, release_id):
+        assets = requests.get(f"{BASE_URL}/releases/{release_id}/assets").json()
+        artifacts = []
+        for asset in assets:
+            artifacts.append({"id": asset["id"], "name": asset["name"]})
+        return artifacts
+
+    def download_artifact(self, path, asset_id, asset_name):
+        headers = {"Accept": "application/octet-stream"}
+        asset = requests.get(
+            f"{BASE_URL}/releases/assets/{asset_id}", headers=headers
+        )
+        with open(os.path.join(path, asset_name), "wb") as artifact:
+            artifact.write(asset.content)
+
+    def download_artifacts(self, path, artifacts):
+        for artifact in artifacts:
+            self.download_artifact(path, artifact["id"], artifact["name"])
+
+    def onefuzz_release_artifacts(self, path):
+        release_id, _ = self.get_latest_version()
+        artifacts = self.list_assets(release_id)
+        self.download_artifacts(path, artifacts)
+
+
+def main():
+    def dir_path(path):
+        path = os.path.abspath(path)
+        if not os.path.isdir(path):
+            os.makedirs(path)
+        return path
+
+    parser = argparse.ArgumentParser(description="Download artifacts")
+    parser.add_argument(
+        "-path",
+        type=dir_path,
+        help="Path to download binaries",
+    )
+    parser.add_argument(
+        "-version",
+        action="store_true",
+        help="Get Onefuzz latest version",
+    )
+
+    args = parser.parse_args()
+    if args.path:
+        Onefuzz().onefuzz_release_artifacts(args.path)
+
+    if args.version:
+        print(Onefuzz().get_latest_version()[1])
+
+
+if __name__ == "__main__":
+    main()

contrib/deploy-onefuzz-via-azure-devops/tox.ini

@@ -0,0 +1,13 @@
+[tox]
+envlist = py38
+skip_missing_interpreters=True
+
+[testenv]
+usedevelop = True
+
+deps =
+  pipenv
+
+commands =
+  pipenv install --dev
+  pipenv run pipenv check