Merge pull request #35 from wisedev-code/feat/configurable-infer-memory-params

Feat/configurable infer memory params

Author: wisedev-pstach

MaIN.Core.IntegrationTests/ChatTests.cs

@@ -1,6 +1,7 @@
 using System.Text.Json;
 using MaIN.Core.Hub;
 using MaIN.Domain.Configuration;
+using MaIN.Domain.Entities;
 using Microsoft.AspNetCore.Mvc.Testing;
 using Microsoft.Extensions.Configuration;
 
@@ -67,6 +68,10 @@ public async Task Should_AnswerGameFromImage_ChatWithVision()
 
         var result = await AIHub.Chat()
             .WithModel("llama3.2:3b")
+            .WithMemoryParams(new MemoryParams
+            {
+                AnswerTokens = 1000
+            })
             .WithMessage("What is the title of game?")
             .WithFiles(images)
             .CompleteAsync();

Releases/0.1.1.md

@@ -0,0 +1,4 @@
+# 0.1.1 release
+
+- Expanded Inference parameters to allow for more LLM behavior configuration
+- Added Memory parameters to allow LLM behavior configuration when working with files

src/MaIN.Core.UnitTests/AgentContextTests.cs

@@ -132,7 +132,8 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent()
                 It.IsAny<Agent>(), 
                 It.IsAny<bool>(), 
                 It.IsAny<bool>(), 
-                It.IsAny<InferenceParams>()))
+                It.IsAny<InferenceParams>(),
+                It.IsAny<MemoryParams>()))
             .ReturnsAsync(agent);
 
         // Act
@@ -144,7 +145,8 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent()
                 It.IsAny<Agent>(), 
                 It.Is<bool>(f => f == true), 
                 It.Is<bool>(r => r == false), 
-                It.IsAny<InferenceParams>()),
+                It.IsAny<InferenceParams>(),
+                It.IsAny<MemoryParams>()),
             Times.Once);
         Assert.Equal(_agentContext, result);
     }

src/MaIN.Core/.nuspec

@@ -2,7 +2,7 @@
 <package>
   <metadata>
     <id>MaIN.NET</id>
-    <version>0.1.0</version>
+    <version>0.1.1</version>
     <authors>Wisedev</authors>
     <owners>Wisedev</owners>
     <icon>favicon.png</icon>

src/MaIN.Core/Hub/Contexts/AgentContext.cs

@@ -2,8 +2,6 @@
 using MaIN.Domain.Entities.Agents;
 using MaIN.Domain.Entities.Agents.AgentSource;
 using MaIN.Domain.Models;
-using MaIN.Services.Dtos;
-using MaIN.Services.Mappers;
 using MaIN.Services.Services.Abstract;
 using MaIN.Services.Services.Models;
 
@@ -13,6 +11,7 @@ public class AgentContext
 {
     private readonly IAgentService _agentService;
     private InferenceParams? _inferenceParams;
+    private MemoryParams? _memoryParams;
     private Agent _agent;
 
     internal AgentContext(IAgentService agentService)
@@ -84,7 +83,13 @@ public AgentContext WithInferenceParams(InferenceParams inferenceParams)
         _inferenceParams = inferenceParams;
         return this;
     }
-    
+
+    public AgentContext WithMemoryParams(MemoryParams memoryParams)
+    {
+        _memoryParams = memoryParams;
+        return this;
+    }
+
     public AgentContext WithCustomModel(string model, string path)
     {
         KnownModels.AddModel(model, path);
@@ -115,13 +120,13 @@ public AgentContext WithBehaviour(string name, string instruction)
 
     public async Task<AgentContext> CreateAsync(bool flow = false, bool interactiveResponse = false)
     {
-        await _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams);
+        await _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams);
         return this;
     }
 
     public AgentContext Create(bool flow = false, bool interactiveResponse = false)
     {
-        _ = _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams).Result;
+        _ = _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams).Result;
         return this;
     }
 

src/MaIN.Core/Hub/Contexts/ChatContext.cs

@@ -41,7 +41,13 @@ public ChatContext WithInferenceParams(InferenceParams inferenceParams)
         _chat.InterferenceParams = inferenceParams;
         return this;
     }
-    
+
+    public ChatContext WithMemoryParams(MemoryParams memoryParams)
+    {
+        _chat.MemoryParams = memoryParams;
+        return this;
+    }
+
     public ChatContext WithCustomModel(string model, string path)
     {
         KnownModels.AddModel(model, path);

src/MaIN.Domain/Entities/Chat.cs

@@ -9,6 +9,7 @@ public class Chat
     public ChatType Type { get; set; } = ChatType.Conversation;
     public bool Visual { get; set; }
     public InferenceParams InterferenceParams { get; set; } = new();
+    public MemoryParams MemoryParams { get; set; } = new();
     public Dictionary<string, string> Properties { get; init; } = [];
     public List<string> Memory { get; } = [];
 

src/MaIN.Domain/Entities/InferenceParams.cs

@@ -4,4 +4,17 @@ public class InferenceParams
 {
     public float Temperature { get; init; } = 0.8f;
     public int ContextSize { get; init; } = 1024;
+    public int GpuLayerCount { get; init; } = 30;
+    public uint SeqMax { get; init; } = 1;
+    public uint BatchSize { get; init; } = 512;
+    public uint UBatchSize { get; init; } = 512;
+    public bool Embeddings { get; init; } = false;
+    public int TypeK { get; init; } = 0;
+    public int TypeV { get; init; } = 0;
+    
+    public int TokensKeep { get; set; }
+    public int MaxTokens { get; set; }
+    
+    public int TopK { get; init; } = 40;
+    public float TopP { get; init; } = 0.9f;
 }

src/MaIN.Domain/Entities/MemoryParams.cs

@@ -0,0 +1,15 @@
+namespace MaIN.Domain.Entities;
+
+public class MemoryParams
+{
+    public int MaxMatchesCount { get; set; } = 5;
+    public float FrequencyPenalty { get; set; } = 1f;
+    public float Temperature { get; set; } = 0.6f;
+    /// <summary>
+    /// Maximum number of tokens to reserve for the answer.
+    /// If LLM supports 5000 tokens and AnswerToken is 500 then prompt sent will contain max 4500 tokens
+    /// (prompt + question + grounding information from memory)
+    /// If your response is invalid make sure you meet those limits.
+    /// </summary>
+    public int AnswerTokens { get; set; } = 500;
+}

src/MaIN.Services/Services/Abstract/IAgentService.cs

@@ -6,7 +6,7 @@ namespace MaIN.Services.Services.Abstract;
 public interface IAgentService
 {
     Task<Chat> Process(Chat chat, string agentId, bool translatePrompt = false);
-    Task<Agent> CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, InferenceParams? inferenceParams = null);
+    Task<Agent> CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, InferenceParams? inferenceParams = null, MemoryParams? memoryParams = null);
     Task<Chat> GetChatByAgent(string agentId);
     Task<Chat> Restart(string agentId);
     Task<List<Agent>> GetAgents();