Skip to content

Commit 49ee42f

Browse files
srebreksrebrek
authored
Speed up tests (#127)
* fix: Vision test fails Model used in the test is a model that does not implement IVision but can read via OCR. LLM service silently packed the images from OCR source to the IVision source and prevented OCR to work as intended. * tests: change test structure - Previous integration tests are moved to E2E test project as they need real api keys or local inference and their duration is long. - New integration test project holds tests that have mocked HTTP Handlers so we can test integration between modules without time-consuming external dependencies * tests: add result asserts and limit answer tokens to reduce model looping.
1 parent c76d68a commit 49ee42f

24 files changed

+937
-74
lines changed

MaIN.Core.IntegrationTests/BackendParamsTests.cs renamed to MaIN.Core.E2ETests/BackendParamsTests.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
using MaIN.Core.Hub;
22
using MaIN.Domain.Configuration;
3-
using MaIN.Domain.Entities;
43
using MaIN.Domain.Configuration.BackendInferenceParams;
4+
using MaIN.Domain.Entities;
55
using MaIN.Domain.Exceptions;
66
using MaIN.Domain.Models;
77
using MaIN.Domain.Models.Concrete;
88

9-
namespace MaIN.Core.IntegrationTests;
9+
namespace MaIN.Core.E2ETests;
1010

11+
[Collection("E2ETests")]
1112
public class BackendParamsTests : IntegrationTestBase
1213
{
1314
private const string TestQuestion = "What is 2+2? Answer with just the number.";

MaIN.Core.IntegrationTests/ChatTests.cs renamed to MaIN.Core.E2ETests/ChatTests.cs

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
using FuzzySharp;
2+
using MaIN.Core.E2ETests.Helpers;
23
using MaIN.Core.Hub;
3-
using MaIN.Core.IntegrationTests.Helpers;
44
using MaIN.Domain.Entities;
55
using MaIN.Domain.Models;
66
using MaIN.Domain.Models.Abstract;
77

8-
namespace MaIN.Core.IntegrationTests;
8+
namespace MaIN.Core.E2ETests;
99

10+
[Collection("E2ETests")]
1011
public class ChatTests : IntegrationTestBase
1112
{
1213
public ChatTests() : base()
@@ -16,7 +17,7 @@ public ChatTests() : base()
1617
[Fact]
1718
public async Task Should_AnswerQuestion_BasicChat()
1819
{
19-
var context = AIHub.Chat().WithModel(Models.Local.Gemma2_2b);
20+
var context = AIHub.Chat().WithModel(Models.Local.Qwen2_5_0_5b);
2021

2122
var result = await context
2223
.WithMessage("Where the hedgehog goes at night?")
@@ -28,28 +29,38 @@ public async Task Should_AnswerQuestion_BasicChat()
2829
}
2930

3031
[Fact]
31-
public async Task Should_AnswerDifferences_BetweenDocuments_ChatWithFiles()
32+
public async Task Should_AnswerFileSubject_ChatWithFiles()
3233
{
33-
List<string> files = ["./Files/Nicolaus_Copernicus.pdf", "./Files/Galileo_Galilei.pdf"];
34+
List<string> files = ["./Files/Nicolaus_Copernicus.pdf"];
3435

3536
var result = await AIHub.Chat()
36-
.WithModel(Models.Local.Gemma2_2b)
37-
.WithMessage("You have 2 documents in memory. Whats the difference of work between Galileo and Copernicus?. Give answer based on the documents.")
37+
.WithModel(Models.Local.Qwen2_5_0_5b)
38+
.WithMessage("Who is described in the file? Reply with ONLY their full name. No explanation, no punctuation. Example: Isaak Newton")
39+
.WithMemoryParams(new MemoryParams { AnswerTokens = 10 })
3840
.WithFiles(files)
3941
.CompleteAsync();
4042

4143
Assert.True(result.Done);
4244
Assert.NotNull(result.Message);
4345
Assert.NotEmpty(result.Message.Content);
46+
var ratio = Fuzz.PartialRatio("nicolaus copernicus", result.Message.Content.ToLowerInvariant());
47+
Assert.True(ratio > 50,
48+
$"""
49+
Fuzzy match failed!
50+
Expected > 50, but got {ratio}.
51+
Expected: 'nicolaus copernicus'
52+
Actual: '{result.Message.Content}'
53+
""");
4454
}
4555

4656
[Fact]
4757
public async Task Should_AnswerQuestion_FromExistingChat()
4858
{
4959
var result = AIHub.Chat()
50-
.WithModel(Models.Local.Gemma2_2b);
60+
.WithModel(Models.Local.Qwen2_5_0_5b);
5161

5262
await result.WithMessage("What do you think about math theories?")
63+
.WithMemoryParams(new MemoryParams { AnswerTokens = 10 })
5364
.CompleteAsync();
5465

5566
await result.WithMessage("And about physics?")
@@ -62,29 +73,53 @@ await result.WithMessage("And about physics?")
6273
}
6374

6475
[Fact]
65-
public async Task Should_AnswerGameFromImage_ChatWithVision()
76+
public async Task Should_AnswerGameFromImage_ChatWithImagesWithText()
6677
{
6778
List<string> images = ["./Files/gamex.jpg"];
79+
var expectedAnswer = "call of duty";
6880

6981
var result = await AIHub.Chat()
7082
.WithModel(Models.Local.Llama3_2_3b)
71-
.WithMessage("What is the title of the game? Answer only this question.")
72-
.WithMemoryParams(new MemoryParams
73-
{
74-
AnswerTokens = 1000
75-
})
83+
.WithMessage("What is the title of the game? Answer in 3 words.")
84+
.WithMemoryParams(new MemoryParams { AnswerTokens = 10 })
85+
.WithFiles(images)
86+
.CompleteAsync();
87+
88+
Assert.True(result.Done);
89+
Assert.NotNull(result.Message);
90+
Assert.NotEmpty(result.Message.Content);
91+
var ratio = Fuzz.PartialRatio(expectedAnswer, result.Message.Content.ToLowerInvariant());
92+
Assert.True(ratio > 50,
93+
$"""
94+
Fuzzy match failed!
95+
Expected > 50, but got {ratio}.
96+
Expexted: '{expectedAnswer}'
97+
Actual: '{result.Message.Content}'
98+
""");
99+
}
100+
101+
[Fact]
102+
public async Task Should_AnswerAppleFromImage_ChatWithImagesWithVision()
103+
{
104+
List<string> images = ["./Files/apple.jpg"];
105+
var expectedAnswer = "apple";
106+
107+
var result = await AIHub.Chat()
108+
.WithModel(Models.Local.Gemma3_4b)
109+
.WithMessage("What is this fruit? Answer in one word.")
110+
.WithMemoryParams(new MemoryParams { AnswerTokens = 10 })
76111
.WithFiles(images)
77112
.CompleteAsync();
78113

79114
Assert.True(result.Done);
80115
Assert.NotNull(result.Message);
81116
Assert.NotEmpty(result.Message.Content);
82-
var ratio = Fuzz.PartialRatio("call of duty", result.Message.Content.ToLowerInvariant());
117+
var ratio = Fuzz.PartialRatio(expectedAnswer, result.Message.Content.ToLowerInvariant());
83118
Assert.True(ratio > 50,
84119
$"""
85120
Fuzzy match failed!
86121
Expected > 50, but got {ratio}.
87-
Expexted: 'call of duty'
122+
Expexted: '{expectedAnswer}'
88123
Actual: '{result.Message.Content}'
89124
""");
90125
}
@@ -113,9 +148,9 @@ public async Task Should_GenerateImage_BasedOnPrompt()
113148
}
114149

115150
[Fact]
116-
public async Task Should_AnswerDifferences_BetweenDocuments_ChatWithFiles_UsingStreams()
151+
public async Task Should_AnswerFileSubject_ChatWithFiles_UsingStreams()
117152
{
118-
List<string> files = ["./Files/Nicolaus_Copernicus.pdf", "./Files/Galileo_Galilei.pdf"];
153+
List<string> files = ["./Files/Nicolaus_Copernicus.pdf"];
119154

120155
var fileStreams = new List<FileStream>();
121156

@@ -135,14 +170,25 @@ public async Task Should_AnswerDifferences_BetweenDocuments_ChatWithFiles_UsingS
135170
fileStreams.Add(fs);
136171
}
137172

173+
var expectedAnswer = "nicolaus copernicus";
174+
138175
var result = await AIHub.Chat()
139-
.WithModel(Models.Local.Gemma2_2b)
140-
.WithMessage("You have 2 documents in memory. Whats the difference of work between Galileo and Copernicus?. Give answer based on the documents.")
176+
.WithModel(Models.Local.Qwen2_5_0_5b)
177+
.WithMessage("Who is described in the file? Reply with ONLY their full name. No explanation, no punctuation. Example: Isaak Newton")
178+
.WithMemoryParams(new MemoryParams { AnswerTokens = 10 })
141179
.WithFiles(fileStreams)
142180
.CompleteAsync();
143181

144182
Assert.True(result.Done);
145183
Assert.NotNull(result.Message);
146184
Assert.NotEmpty(result.Message.Content);
185+
var ratio = Fuzz.PartialRatio(expectedAnswer, result.Message.Content.ToLowerInvariant());
186+
Assert.True(ratio > 50,
187+
$"""
188+
Fuzzy match failed!
189+
Expected > 50, but got {ratio}.
190+
Expected: '{expectedAnswer}'
191+
Actual: '{result.Message.Content}'
192+
""");
147193
}
148194
}

MaIN.Core.IntegrationTests/Files/Books.json renamed to MaIN.Core.E2ETests/Files/Books.json

File renamed without changes.

MaIN.Core.IntegrationTests/Files/Galileo_Galilei.pdf renamed to MaIN.Core.E2ETests/Files/Galileo_Galilei.pdf

File renamed without changes.

MaIN.Core.IntegrationTests/Files/Nicolaus_Copernicus.pdf renamed to MaIN.Core.E2ETests/Files/Nicolaus_Copernicus.pdf

File renamed without changes.

‎MaIN.Core.E2ETests/Files/apple.jpg‎

25.6 KB
Loading

MaIN.Core.IntegrationTests/Files/gamex.jpg renamed to MaIN.Core.E2ETests/Files/gamex.jpg

File renamed without changes.

MaIN.Core.IntegrationTests/Helpers/NetworkHelper.cs renamed to MaIN.Core.E2ETests/Helpers/NetworkHelper.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
using System;
21
using System.Net.Sockets;
32

4-
namespace MaIN.Core.IntegrationTests.Helpers;
3+
namespace MaIN.Core.E2ETests.Helpers;
54

65
public static class NetworkHelper
76
{

‎MaIN.Core.E2ETests/IntegrationTestBase.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using Microsoft.AspNetCore.Hosting;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using Microsoft.Extensions.Hosting;
4+
5+
namespace MaIN.Core.E2ETests;
6+
7+
public class IntegrationTestBase : IDisposable
8+
{
9+
protected readonly IHost _host;
10+
protected readonly IServiceProvider _services;
11+
12+
protected IntegrationTestBase()
13+
{
14+
_host = Host.CreateDefaultBuilder()
15+
.ConfigureServices((context, services) =>
16+
{
17+
services.AddMaIN(context.Configuration);
18+
ConfigureServices(services);
19+
})
20+
.Build();
21+
22+
_host.Services.UseMaIN();
23+
_host.Start();
24+
25+
_services = _host.Services;
26+
}
27+
28+
// Allow derived classes to add additional services or override existing ones
29+
protected virtual void ConfigureServices(IServiceCollection services)
30+
{
31+
}
32+
33+
protected T GetService<T>() where T : notnull => _services.GetRequiredService<T>();
34+
35+
public void Dispose()
36+
{
37+
_host.Dispose();
38+
GC.SuppressFinalize(this);
39+
}
40+
}

‎MaIN.Core.E2ETests/MaIN.Core.E2ETests.csproj‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFrameworks>net8.0;net10.0</TargetFrameworks>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<IsPackable>false</IsPackable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="FuzzySharp" Version="2.0.2" />
12+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
13+
<PackageReference Include="xunit" Version="2.9.3" />
14+
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
15+
<PrivateAssets>all</PrivateAssets>
16+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
17+
</PackageReference>
18+
<PackageReference Include="Xunit.SkippableFact" Version="1.5.61" />
19+
</ItemGroup>
20+
21+
<ItemGroup>
22+
<Using Include="Xunit" />
23+
</ItemGroup>
24+
25+
<ItemGroup>
26+
<ProjectReference Include="..\src\MaIN.Core\MaIN.Core.csproj" />
27+
</ItemGroup>
28+
29+
<ItemGroup>
30+
<None Update="Files\apple.jpg">
31+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
32+
</None>
33+
<None Update="Files\Books.json">
34+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
35+
</None>
36+
<None Update="Files\Galileo_Galilei.pdf">
37+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
38+
</None>
39+
<None Update="Files\gamex.jpg">
40+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
41+
</None>
42+
<None Update="Files\Nicolaus_Copernicus.pdf">
43+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
44+
</None>
45+
</ItemGroup>
46+
47+
</Project>

0 commit comments

Comments
 (0)