Integration tests project (#2083)

Move integration tests into their own project. This makes it easier to run unit tests if you don't want to (or don't have) `azurite` running, since you can `dotnet test` just that directory.

Also add a check into the `AzuriteStorage` class that will abort the entire test run if `azurite` is not running, which is simpler than reading lots of socket exceptions.

.github/workflows/ci.yml

@@ -289,8 +289,8 @@ jobs:
       run: |
         cd src/ApiService/
         azurite --silent &
-        dotnet test --no-restore --collect:"XPlat Code Coverage" --filter 'Category!=Integration'
+        dotnet test --no-restore --collect:"XPlat Code Coverage" --filter 'Category!=Live'
-        dotnet tool run reportgenerator -reports:Tests/TestResults/*/coverage.cobertura.xml -targetdir:coverage -reporttypes:MarkdownSummary
+        dotnet tool run reportgenerator -reports:*/TestResults/*/coverage.cobertura.xml -targetdir:coverage -reporttypes:MarkdownSummary
         kill %1
         cat coverage/*.md > $GITHUB_STEP_SUMMARY
 

src/ApiService/ApiService.sln

@@ -7,6 +7,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApiService", "ApiService\Ap
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{06C9FE9B-6DDD-45EC-B716-CB074512DAA9}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{AAB88572-BA8E-4661-913E-61FC9827005D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -21,6 +23,10 @@ Global
 		{06C9FE9B-6DDD-45EC-B716-CB074512DAA9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{06C9FE9B-6DDD-45EC-B716-CB074512DAA9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{06C9FE9B-6DDD-45EC-B716-CB074512DAA9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AAB88572-BA8E-4661-913E-61FC9827005D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AAB88572-BA8E-4661-913E-61FC9827005D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AAB88572-BA8E-4661-913E-61FC9827005D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AAB88572-BA8E-4661-913E-61FC9827005D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/ApiService/IntegrationTests/AgentEventsTests.cs

@@ -1,16 +1,15 @@
 using System;
 using System.Linq;
 using System.Net;
+using IntegrationTests.Fakes;
 using Microsoft.OneFuzz.Service;
-using Tests.Fakes;
 using Xunit;
 using Xunit.Abstractions;
-
 using Async = System.Threading.Tasks;
 
-namespace Tests.Functions;
+namespace IntegrationTests;
 
-[Trait("Category", "Integration")]
+[Trait("Category", "Live")]
 public class AzureStorageAgentEventsTest : AgentEventsTestsBase {
     public AzureStorageAgentEventsTest(ITestOutputHelper output)
         : base(output, Integration.AzureStorage.FromEnvironment()) { }

src/ApiService/IntegrationTests/DownloadTests.cs

@@ -2,16 +2,15 @@
 using System;
 using System.Net;
 using System.Net.Http;
+using IntegrationTests.Fakes;
 using Microsoft.OneFuzz.Service;
-using Tests.Fakes;
 using Xunit;
 using Xunit.Abstractions;
-
 using Async = System.Threading.Tasks;
 
-namespace Tests.Functions;
+namespace IntegrationTests;
 
-[Trait("Category", "Integration")]
+[Trait("Category", "Live")]
 public class AzureStorageDownloadTest : DownloadTestBase {
     public AzureStorageDownloadTest(ITestOutputHelper output)
         : base(output, Integration.AzureStorage.FromEnvironment()) { }

src/ApiService/IntegrationTests/Fakes/TestContext.cs

@@ -4,7 +4,7 @@ using Microsoft.OneFuzz.Service;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using Async = System.Threading.Tasks;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 
 // TestContext provides a minimal IOnefuzzContext implementation to allow running

src/ApiService/IntegrationTests/Fakes/TestCreds.cs

@@ -7,7 +7,7 @@ using Azure.ResourceManager.Resources;
 using Microsoft.OneFuzz.Service;
 using Task = System.Threading.Tasks.Task;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 class TestCreds : ICreds {
 

src/ApiService/IntegrationTests/Fakes/TestEvents.cs

@@ -3,7 +3,7 @@ using Microsoft.OneFuzz.Service;
 
 using Async = System.Threading.Tasks;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 public sealed class TestEvents : IEvents {
 

src/ApiService/IntegrationTests/Fakes/TestHttpRequestData.cs

@@ -9,7 +9,7 @@ using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.Extensions.Options;
 using Moq;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 sealed class TestHttpRequestData : HttpRequestData {
     private static readonly ObjectSerializer _serializer =

src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs

@@ -2,7 +2,7 @@
 using Microsoft.ApplicationInsights.DataContracts;
 using Microsoft.OneFuzz.Service;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 public sealed class TestServiceConfiguration : IServiceConfig {
     public TestServiceConfiguration(string tablePrefix) {

src/ApiService/IntegrationTests/Fakes/TestUserCredentials.cs

@@ -4,7 +4,7 @@ using Microsoft.OneFuzz.Service;
 
 using Async = System.Threading.Tasks;
 
-namespace Tests.Fakes;
+namespace IntegrationTests.Fakes;
 
 sealed class TestUserCredentials : UserCredentials {
 

src/ApiService/IntegrationTests/InfoTests.cs

@@ -1,16 +1,15 @@
 
 using System;
 using System.Net;
+using IntegrationTests.Fakes;
 using Microsoft.OneFuzz.Service;
-using Tests.Fakes;
 using Xunit;
 using Xunit.Abstractions;
-
 using Async = System.Threading.Tasks;
 
-namespace Tests.Functions;
+namespace IntegrationTests;
 
-[Trait("Category", "Integration")]
+[Trait("Category", "Live")]
 public class AzureStorageInfoTest : InfoTestBase {
     public AzureStorageInfoTest(ITestOutputHelper output)
         : base(output, Integration.AzureStorage.FromEnvironment()) { }

src/ApiService/IntegrationTests/Integration/AzureStorage.cs

@@ -5,7 +5,7 @@ using Microsoft.OneFuzz.Service;
 
 using Async = System.Threading.Tasks;
 
-namespace Tests.Integration;
+namespace IntegrationTests.Integration;
 
 // This exists solely to allow use of a fixed storage account in integration tests
 // against live Azure Storage.
@@ -15,11 +15,11 @@ sealed class AzureStorage : IStorage {
         var accountKey = Environment.GetEnvironmentVariable("AZURE_ACCOUNT_KEY");
 
         if (accountName is null) {
-            throw new Exception("AZURE_ACCOUNT_NAME must be set in environment to run integration tests (use --filter 'Category!=Integration' to skip them)");
+            throw new Exception("AZURE_ACCOUNT_NAME must be set in environment to run integration tests (use --filter 'Category!=Live' to skip them)");
         }
 
         if (accountKey is null) {
-            throw new Exception("AZURE_ACCOUNT_KEY must be set in environment to run integration tests (use --filter 'Category!=Integration' to skip them)");
+            throw new Exception("AZURE_ACCOUNT_KEY must be set in environment to run integration tests (use --filter 'Category!=Live' to skip them)");
         }
 
         return new AzureStorage(accountName, accountKey);
@@ -49,7 +49,7 @@ sealed class AzureStorage : IStorage {
         => Async.Task.FromResult((AccountName, AccountKey));
 
     public Task<string?> GetStorageAccountNameKeyByName(string accountName) {
-        return Async.Task.FromResult(AccountName)!;
+        return Async.Task.FromResult<string?>(AccountName);
     }
 
     public Uri GetTableEndpoint(string accountId)

src/ApiService/IntegrationTests/Integration/AzuriteStorage.cs

@@ -1,13 +1,25 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.OneFuzz.Service;
 
 using Async = System.Threading.Tasks;
 
-namespace Tests.Integration;
+namespace IntegrationTests.Integration;
 
+// An implementation of IStorage for communicating with the Azurite Storage emulator.
 sealed class AzuriteStorage : IStorage {
+    static AzuriteStorage() {
+        try {
+            using var client = new HttpClient();
+            client.GetAsync(new Uri("http://127.0.0.1:10000")).Wait();
+        } catch {
+            Console.Error.WriteLine("'azurite' must be running to run integration tests.");
+            Environment.Exit(1);
+        }
+    }
+
     public Uri GetBlobEndpoint(string _accountId)
         => new($"http://127.0.0.1:10000/devstoreaccount1");
 

src/ApiService/IntegrationTests/IntegrationTests.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="Moq" Version="4.17.2" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ApiService\ApiService.csproj" />
+  </ItemGroup>
+
+</Project>

src/ApiService/IntegrationTests/NodeTests.cs

@@ -3,16 +3,15 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using IntegrationTests.Fakes;
 using Microsoft.OneFuzz.Service;
-using Tests.Fakes;
 using Xunit;
 using Xunit.Abstractions;
-
 using Async = System.Threading.Tasks;
 
-namespace Tests.Functions;
+namespace IntegrationTests.Functions;
 
-[Trait("Category", "Integration")]
+[Trait("Category", "Live")]
 public class AzureStorageNodeTest : NodeTestBase {
     public AzureStorageNodeTest(ITestOutputHelper output)
         : base(output, Integration.AzureStorage.FromEnvironment()) { }

src/ApiService/IntegrationTests/README.md

@@ -0,0 +1,18 @@
+# Integration Tests
+
+The integration tests in this project allow specific Functions to be run against
+Azure Storage. They can be run in two modes:
+
+- **Against the [Azurite](https://github.com/Azure/Azurite) storage emulator**:
+  these tests are run by default. `azurite` must be started and running (e.g.
+  with `azurite -s &`).
+
+- **Against a real Azure Storage account**: to use this, the environment
+  variables `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_KEY` must be set.
+
+  These tests can be excluded by running `dotnet test` with the arguments
+  `--filter "Category!=Live"`.
+
+The same tests are used in each case. The way this is achieved in Xunit is by
+writing the tests in an (abstract) base class and then deriving two
+implementations from this base class, one for each “run configuration”. 

src/ApiService/IntegrationTests/TestLogTracer.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using Microsoft.OneFuzz.Service;
 using Xunit.Abstractions;
 
-namespace Tests;
+namespace IntegrationTests;
 
 sealed class TestLogTracer : ILogTracer {
     private readonly ITestOutputHelper _output;

src/ApiService/IntegrationTests/_FunctionTestBase.cs

@@ -5,13 +5,13 @@ using ApiService.OneFuzzLib.Orm;
 using Azure.Data.Tables;
 using Azure.Storage;
 using Azure.Storage.Blobs;
+using IntegrationTests.Fakes;
 using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.OneFuzz.Service;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using Tests.Fakes;
 using Xunit.Abstractions;
 
-namespace Tests.Functions;
+namespace IntegrationTests;
 
 // FunctionTestBase contains shared implementations for running
 // functions against live Azure Storage or the Azurite emulator.
@@ -20,7 +20,7 @@ namespace Tests.Functions;
 // with all the tests defined in it. Then, from that class
 // derive two non-abstract classes for XUnit to find:
 // - one for Azurite
-// - one for Azure Storage (marked with [Trait("Category", "Integration")])
+// - one for Azure Storage (marked with [Trait("Category", "Live")])
 //
 // See AgentEventsTests for an example.
 public abstract class FunctionTestBase : IDisposable {