Extend .WithSource to support APIs with API key authentication

src/MaIN.Domain/Entities/Agents/AgentSource/AgentAPISourceDetails.cs

@@ -8,4 +8,9 @@ public class AgentApiSourceDetails : AgentSourceDetailsBase, IAgentSource
     public string? Query { get; set; }
     public string? ResponseType { get; init; }
     public int? ChunkLimit { get; init; }
+    public string? AuthorisationToken { get; set; }
+    /// <summary>
+    /// Override payload and authorisation token
+    /// </summary>
+    public string? Curl { get; set; }
 }

src/MaIN.Services/MaIN.Services.csproj

@@ -28,4 +28,8 @@
       <PackageReference Include="Tesseract.Data.English" Version="4.0.0" />
     </ItemGroup>
 
+    <ItemGroup>
+      <NativeLibs Remove="Utils\CurlRequestParser.cs" />
+    </ItemGroup>
+
 </Project>

src/MaIN.Services/Services/DataSourceProvider.cs

@@ -4,6 +4,7 @@
 using MaIN.Domain.Entities.Agents.AgentSource;
 using MaIN.Services.Services.Abstract;
 using MaIN.Services.Utils;
+using Microsoft.IdentityModel.Tokens;
 using MongoDB.Bson;
 using MongoDB.Driver;
 
@@ -61,27 +62,85 @@ public async Task<string> FetchApiData(object? details, string? filter,
         apiDetails.Query = apiDetails.Query?.Replace("@filter@", filter);
         apiDetails.Url = apiDetails.Url.Replace("@filter@", filter);
 
+        HttpResponseMessage result = null;
+
         var request = new HttpRequestMessage(
             HttpMethod.Parse(apiDetails?.Method),
             apiDetails?.Url + apiDetails?.Query);
 
-        if (!string.IsNullOrEmpty(apiDetails?.Payload))
+
+        if (!apiDetails.AuthorisationToken.IsNullOrEmpty())
+        {
+            var token = apiDetails.AuthorisationToken;
+            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
+
+        }
+
+        if (!string.IsNullOrEmpty(apiDetails?.Curl))
         {
-            request.Content = new StringContent(
-                JsonSerializer.Serialize(apiDetails.Payload),
-                Encoding.UTF8,
-                "application/json");
+            CurlRequestParser.PopulateRequestFromCurl(request, apiDetails.Curl);
+            var a = await request.Content.ReadAsStringAsync();
         }
+        else
+        {
+            if (!string.IsNullOrEmpty(apiDetails?.Payload))
+            {
 
-        var result = await httpClient.SendAsync(request);
+                var jsonString = apiDetails.Payload;
+
+                if (!(apiDetails.Payload is string))
+                {
+                    jsonString = JsonSerializer.Serialize(apiDetails.Payload);
+
+                }
+                else
+                {
+                    try
+                    {
+                        JsonDocument.Parse(jsonString);
+                    }
+                    catch (JsonException ex)
+                    {
+                        try
+                        {
+                            jsonString = JsonSerializer.Serialize(apiDetails.Payload);
+                            if (!jsonString.StartsWith('{'))
+                            {
+                                jsonString = $"{{{jsonString}}}";
+                            }
+                            jsonString = JsonCleaner.CleanAndUnescape(jsonString);
+
+                            JsonDocument.Parse(jsonString);
+                        }
+                        catch
+                        {
+                            throw new Exception($"Invalid JSON: {ex.Message}");
+                        }
+
+
+                    }
+                }
+
+                request.Content = new StringContent(
+                    jsonString,
+                    Encoding.UTF8,
+                    "application/json");
+
+
+            }
+
+
+        }
+        //------------
+        result = await httpClient.SendAsync(request);
         if (!result.IsSuccessStatusCode)
         {
             throw new Exception(
                 $"API request failed with status code: {result.StatusCode}"); //TODO candidate for domain exception
         }
 
         var data = await result.Content.ReadAsStringAsync();
-
+        
         properties.TryAdd("api_response_type", apiDetails?.ResponseType ?? "JSON");
         if (apiDetails?.ChunkLimit != null)
         {

src/MaIN.Services/Services/LLMService/Memory/MemoryService.cs

@@ -1,4 +1,8 @@
+using MaIN.Services.Utils;
 using Microsoft.KernelMemory;
+using MongoDB.Bson.IO;
+using MongoDB.Bson.Serialization.Serializers;
+using System.Text.Json;
 
 namespace MaIN.Services.Services.LLMService.Memory;
 
@@ -35,7 +39,8 @@ private async Task ImportTextData(IKernelMemory memory, Dictionary<string, strin
 
         foreach (var item in textData)
         {
-            await memory.ImportTextAsync(item.Value, item.Key, cancellationToken: cancellationToken);
+            var cleanedValue = JsonCleaner.CleanAndUnescape(item.Value);
+            await memory.ImportTextAsync(cleanedValue, item.Key, cancellationToken: cancellationToken);
         }
     }
 

src/MaIN.Services/Utils/CurlRequestParser.cs

@@ -0,0 +1,158 @@
+using MaIN.Domain.Entities.Agents.AgentSource;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace MaIN.Services.Utils;
+public static class CurlRequestParser
+{
+    public static void PopulateRequestFromCurl(HttpRequestMessage request, string curlRaw)
+    {
+        if (request == null)
+            throw new ArgumentNullException(nameof(request));
+
+        if (string.IsNullOrWhiteSpace(curlRaw))
+            throw new ArgumentException("curl string is empty");
+
+        // Normalize multi-line curl: usuń backslash+newline i nadmiarowe spacje
+        string curl = Regex.Replace(curlRaw, @"\\\s*\n", " ");
+        curl = Regex.Replace(curl, @"\s{2,}", " ").Trim();
+
+        // Metoda HTTP - sprawdzaj -X i --request
+        if (request.Method == null || request.Method == HttpMethod.Get)
+        {
+            var methodMatch = Regex.Match(curl, @"(?:-X|--request)\s+(\w+)", RegexOptions.IgnoreCase);
+            if (methodMatch.Success)
+            {
+                request.Method = new HttpMethod(methodMatch.Groups[1].Value.ToUpperInvariant());
+            }
+            else if (curl.Contains("--get"))
+            {
+                request.Method = HttpMethod.Get;
+            }
+            else
+            {
+                request.Method ??= HttpMethod.Get; // domyślnie GET
+            }
+        }
+
+        // URL - najpierw --url, potem pierwszy URL po curl
+        if (request.RequestUri == null)
+        {
+            var urlMatch = Regex.Match(curl, @"(?:--url\s+|curl\s+)(['""]?)(https?://[^\s'""]+)\1", RegexOptions.IgnoreCase);
+            if (!urlMatch.Success)
+            {
+                urlMatch = Regex.Match(curl, @"(['""])(https?://[^\s'""]+)\1", RegexOptions.IgnoreCase);
+            }
+            if (urlMatch.Success)
+            {
+                request.RequestUri = new Uri(urlMatch.Groups[2].Value);
+            }
+            else
+            {
+                throw new InvalidOperationException("No URL found in curl string and RequestUri is null.");
+            }
+        }
+
+        // Nagłówki - obsługa -H i --header, w pojedynczych lub podwójnych cudzysłowach
+        var headerPattern = @"(?:-H|--header)\s+['""]?([^:'""]+):\s*([^'""]+)['""]?";
+        foreach (Match headerMatch in Regex.Matches(curl, headerPattern, RegexOptions.IgnoreCase))
+        {
+            string key = headerMatch.Groups[1].Value.Trim();
+            string value = headerMatch.Groups[2].Value.Trim();
+
+            // DODANA ZMIANA: Sprawdź, czy nagłówek Authorization już istnieje
+            if (key.Equals("Authorization", StringComparison.OrdinalIgnoreCase) && request.Headers.Contains("Authorization"))
+            {
+                // Jeśli nagłówek Authorization już istnieje, nie podmieniaj go
+                continue;
+            }
+
+            if (!request.Headers.TryAddWithoutValidation(key, value))
+            {
+                if (request.Content == null)
+                    request.Content = new StringContent(""); // dummy content for headers
+
+                request.Content.Headers.TryAddWithoutValidation(key, value);
+            }
+        }
+
+        // Dane (payload) - może być wiele -d lub --data
+        //var dataMatches = Regex.Matches(curl, @"(?:-d|--data(?:-raw)?)\s+(['""])(.*?)\1", RegexOptions.IgnoreCase | RegexOptions.Singleline);
+        var processedCurl = PreprocessCurlString(curl);
+        var dataMatches = Regex.Matches(processedCurl, @"(?:-d|--data(?:-raw)?)\s+(['""])((?:\\.|(?!\1).)*)\1", RegexOptions.IgnoreCase | RegexOptions.Singleline);
+
+
+        if (request.Method == HttpMethod.Get)
+        {
+            // GET - dodaj dane jako query string
+            var queryParams = new List<string>();
+            foreach (Match m in dataMatches)
+            {
+                var data = m.Groups[2].Value.Trim();
+                if (data.StartsWith("@"))
+                    throw new NotSupportedException("Nie obsługiwany -d z plikiem przy GET.");
+
+                queryParams.Add(data);
+            }
+
+            if (queryParams.Count > 0)
+            {
+                var builder = new UriBuilder(request.RequestUri);
+                var existingQuery = builder.Query;
+                string newQuery = existingQuery.Length > 1 ? existingQuery.Substring(1) + "&" + string.Join("&", queryParams) : string.Join("&", queryParams);
+                builder.Query = newQuery;
+                request.RequestUri = builder.Uri;
+            }
+        }
+        else
+        {
+            // POST, PUT itp. - ustaw body z pierwszego -d
+            if (dataMatches.Count > 0)
+            {
+                var rawData = dataMatches[0].Groups[2].Value;
+
+                rawData = rawData.Replace("__SINGLE_QUOTE__", "'");
+                rawData = rawData.Replace("__DOUBLE_QUOTE__", "\"");
+                rawData = rawData.Replace("__BACKSLASH__", "\\");
+
+                if (rawData.StartsWith("@"))
+                    throw new NotSupportedException("Obsługa -d z plikiem nie jest zaimplementowana.");
+
+                request.Content = new StringContent(rawData, Encoding.UTF8, "application/json");
+            }
+        }
+    }
+
+
+    public static string PreprocessCurlString(string curlRaw)
+    {
+        if (string.IsNullOrEmpty(curlRaw))
+            return curlRaw;
+
+        string result = curlRaw;
+
+        // 1. Replace shell escaped single quotes inside single-quoted strings: '\'' -> __SINGLE_QUOTE__
+        // This is the main culprit breaking regex
+        result = Regex.Replace(result, @"'\\''", "__SINGLE_QUOTE__");
+
+        // 2. Optionally, handle escaped backslashes or quotes inside double quotes:
+        // Replace \" with __DOUBLE_QUOTE__ and \\ with __BACKSLASH__ if needed
+        // (You can add this if you see your JSON payload uses double quotes with escapes)
+        result = result.Replace("\\\"", "__DOUBLE_QUOTE__");
+        result = result.Replace("\\\\", "__BACKSLASH__");
+
+        // 3. Normalize multi-line curl command: remove backslash+newline (already done in your code)
+        result = Regex.Replace(result, @"\\\s*\n", " ");
+
+        // 4. Replace multiple spaces by single space
+        result = Regex.Replace(result, @"\s{2,}", " ").Trim();
+
+        return result;
+    }
+}

src/MaIN.Services/Utils/JsonCleaner.cs

@@ -0,0 +1,53 @@
+using MaIN.Domain.Models;
+using System;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+
+public static class JsonCleaner
+{
+    public static string? CleanAndUnescape(string json, int maxDepth = 5)
+    {
+        if (string.IsNullOrWhiteSpace(json))
+            return null;
+
+        string current = json.Trim();
+        int depth = 0;
+
+        while (depth < maxDepth)
+        {
+            try
+            {
+                using JsonDocument doc = JsonDocument.Parse(current);
+                JsonElement root = doc.RootElement;
+
+                if (root.ValueKind == JsonValueKind.String)
+                {
+                    current = root.GetString()!.Trim();
+                    depth++;
+                    continue;
+                }
+                else if (root.ValueKind == JsonValueKind.Object || root.ValueKind == JsonValueKind.Array)
+                {
+                    // Serialize with relaxed escaping to unescape unicode characters
+                    var options = new JsonSerializerOptions
+                    {
+                        WriteIndented = true,
+                        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+                    };
+
+                    // Note: We serialize the JsonElement directly, which will output real Unicode chars
+                    return JsonSerializer.Serialize(root, options);
+                }
+                else
+                {
+                    return null;
+                }
+            }
+            catch (JsonException)
+            {
+                return null;
+            }
+        }
+        return null;
+    }
+}