Merge pull request #58 from AIDotNet/feature_StableDiffusion

fix 修改为静态类

README.md

@@ -54,6 +54,8 @@ https://antsk.ai-dotnet.com/
 ### 其他功能示例
 [视频示例](https://www.bilibili.com/video/BV1zH4y1h7Y9/)
 
+[在线文档：http://antsk.cn](http://antsk.cn)
+
 ## ❓如何开始？
 
 在这里我使用的是Postgres 作为数据存储和向量存储，因为Semantic Kernel和Kernel Memory都支持他，当然你也可以换成其他的。
@@ -173,7 +175,7 @@ DB我使用的是CodeFirst模式，只要配置好数据库链接，表结构是
 
 ## ✔️使用llamafactory
 ```
-1、首先需要确保你的环境已经安装了python和pip，如果使用镜像，例如v0.2.3.2版本已经包含了 python全套环境则无需此步骤
+1、首先需要确保你的环境已经安装了python和pip，如果使用镜像，例如p0.2.4版本已经包含了 python全套环境则无需此步骤
 2、进入模型添加页面选择llamafactory
 3、点击初始化，可以检查pip install 环境是否完成
 4、选择一个喜欢的模型

docker-compose.simple.yml

@@ -3,9 +3,9 @@ version: '3.8'
 services:
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.3
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.4
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.3.2
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.4
     ports:
       - 5000:5000
     networks:

docker-compose.yml

@@ -18,9 +18,9 @@ services:
       - ./pg/data:/var/lib/postgresql/data
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.3
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.4
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.3.2
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.4
     ports:
       - 5000:5000
     networks:

src/AntSK.Domain/AntSK.Domain.csproj

@@ -13,13 +13,15 @@
     <PackageReference Include="BlazorComponents.Terminal" Version="0.6.0" />
     <PackageReference Include="Blazored.LocalStorage" Version="4.5.0" />
   
+    <PackageReference Include="pythonnet" Version="3.0.3" />
+    
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
     
     <PackageReference Include="AutoMapper" Version="8.1.0" />
     <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
     <PackageReference Include="Markdig" Version="0.36.2" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="SqlSugarCore" Version="5.1.4.148" />
+    <PackageReference Include="SqlSugarCore" Version="5.1.4.149" />
     <PackageReference Include="System.Data.SQLite.Core" Version="1.0.118" />
     <PackageReference Include="RestSharp" Version="110.2.0" />
     
@@ -32,11 +34,11 @@
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Redis" Version="0.35.240321.1" />
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="0.35.240321.1" />
 
-    <PackageReference Include="LLamaSharp" Version="0.10.0" />
-    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.10.0" />
-    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.10.0" />
-    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.10.0" />
-    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.10.0" />
+    <PackageReference Include="LLamaSharp" Version="0.11.1" />
+    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.11.1" />
+    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.11.1" />
+    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.11.1" />
+    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.11.1" />
     
     
   </ItemGroup>

src/AntSK.Domain/AntSK.Domain.xml

@@ -99,6 +99,11 @@
             总数
             </summary>
         </member>
+        <member name="M:AntSK.Domain.Domain.Other.EmbeddingConfig.LoadModel(System.String,System.String)">
+            <summary>
+            模型写死
+            </summary>
+        </member>
         <member name="F:AntSK.Domain.Domain.Other.LLamaConfig.dicLLamaWeights">
             <summary>
             避免模型重复加载，本地缓存
@@ -771,6 +776,14 @@
             <param name="parameters"></param>
             <returns></returns>
         </member>
+        <member name="M:AntSK.Domain.Utils.ConvertUtils.ComparisonIgnoreCase(System.String,System.String)">
+            <summary>
+            忽略大小写匹配
+            </summary>
+            <param name="s"></param>
+            <param name="value"></param>
+            <returns></returns>
+        </member>
         <member name="M:AntSK.Domain.Utils.RepoFiles.SamplePluginsPath">
             <summary>
             Scan the local folders from the repo, looking for "samples/plugins" folder.

src/AntSK.Domain/Common/Embedding/BuilderBgeExtensions.cs

@@ -0,0 +1,21 @@
+using LLamaSharp.KernelMemory;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AntSK.Domain.Common.Embedding
+{
+    public static class BuilderBgeExtensions
+    {
+        public static IKernelMemoryBuilder WithBgeTextEmbeddingGeneration(this IKernelMemoryBuilder builder, HuggingfaceTextEmbeddingGenerator textEmbeddingGenerator)
+        {
+            builder.AddSingleton((ITextEmbeddingGenerator)textEmbeddingGenerator);
+            builder.AddIngestionEmbeddingGenerator(textEmbeddingGenerator);
+            return builder;
+        }
+    }
+}

src/AntSK.Domain/Common/Embedding/HuggingfaceTextEmbeddingGenerator.cs

@@ -0,0 +1,56 @@
+using LLama.Common;
+using LLama;
+using LLamaSharp.KernelMemory;
+using Microsoft.KernelMemory.AI;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using AntSK.Domain.Domain.Other;
+
+namespace AntSK.Domain.Common.Embedding
+{
+    public class HuggingfaceTextEmbeddingGenerator : ITextEmbeddingGenerator, ITextTokenizer, IDisposable
+    {
+        public int MaxTokens => 1024;
+
+        public int MaxTokenTotal => 1024;
+
+  
+        private readonly dynamic _embedder;
+
+        public HuggingfaceTextEmbeddingGenerator(string pyDllPath,string modelName)
+        {
+            _embedder = EmbeddingConfig.LoadModel(pyDllPath, modelName);
+        }
+
+        public void Dispose()
+        {
+            EmbeddingConfig.Dispose();
+        }
+
+        //public async Task<IList<ReadOnlyMemory<float>>> GenerateEmbeddingAsync(IList<string> data, CancellationToken cancellationToken = default)
+        //{
+        //    IList<ReadOnlyMemory<float>> results = new List<ReadOnlyMemory<float>>();
+
+        //    foreach (var d in data)
+        //    {
+        //        var embeddings = await EmbeddingConfig.GetEmbedding(d);
+        //        results.Add(new ReadOnlyMemory<float>(embeddings));
+        //    }
+        //    return results;
+        //}
+
+        public async Task<Microsoft.KernelMemory.Embedding> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
+        {
+            var embeddings = await EmbeddingConfig.GetEmbedding(text);
+            return new Microsoft.KernelMemory.Embedding(embeddings);
+        }
+
+        public int CountTokens(string text)
+        {
+            return EmbeddingConfig.TokenCount(text);
+        }
+    }
+}

src/AntSK.Domain/Domain/Interface/IChatService.cs

@@ -5,6 +5,7 @@ using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using System;
 using System.Collections.Generic;
+using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
@@ -16,7 +17,7 @@ namespace AntSK.Domain.Domain.Interface
         IAsyncEnumerable<StreamingKernelContent> SendChatByAppAsync(Apps app, string questions, ChatHistory history);
 
         IAsyncEnumerable<StreamingKernelContent> SendKmsByAppAsync(Apps app, string questions, ChatHistory history, string filePath, List<RelevantSource> relevantSources = null);
-
+        Task<string> SendImgByAppAsync(Apps app, string questions);
         Task<ChatHistory> GetChatHistory(List<MessageInfo> MessageList);
     }
 }

src/AntSK.Domain/Domain/Model/Enum/AIModelType.cs

@@ -21,12 +21,16 @@ namespace AntSK.Domain.Domain.Model.Enum
 
         [Display(Name = "灵积大模型")]
         DashScope = 5,
-        
+
         [Display(Name = "LLamaFactory")]
         LLamaFactory = 6,
-
+        [Display(Name = "Bge Embedding")]
+        BgeEmbedding = 7,
+        [Display(Name = "StableDiffusion")]
+        StableDiffusion = 8,
         [Display(Name = "模拟输出")]
         Mock = 100,
+
     }
 
     /// <summary>
@@ -36,5 +40,6 @@ namespace AntSK.Domain.Domain.Model.Enum
     {
         Chat = 1,
         Embedding = 2,
+        Image=3,
     }
 }

src/AntSK.Domain/Domain/Model/Enum/AppType.cs

@@ -9,6 +9,7 @@ namespace AntSK.Domain.Domain.Model.Enum
     public enum AppType
     {
         chat = 1,
-        kms = 2
+        kms = 2,
+        img=3
     }
 }

src/AntSK.Domain/Domain/Other/EmbeddingConfig.cs

@@ -0,0 +1,88 @@
+using Python.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using static Python.Runtime.Py;
+
+namespace AntSK.Domain.Domain.Other
+{
+    public static class EmbeddingConfig
+    {
+        public static dynamic model { get; set; }
+
+        static object lockobj = new object();
+
+
+
+        /// <summary>
+        /// 模型写死
+        /// </summary>
+        public static dynamic LoadModel(string pythondllPath, string modelName)
+        {
+            lock (lockobj)
+            {
+                if (model == null)
+                {
+                    //Runtime.PythonDLL = @"D:\Programs\Python\Python311\python311.dll";
+                    Runtime.PythonDLL = pythondllPath;
+                    PythonEngine.Initialize();
+                    PythonEngine.BeginAllowThreads();
+                    
+                    try
+                    {
+                        using (Py.GIL())// 初始化Python环境的Global Interpreter Lock)
+                        {
+                            dynamic modelscope = Py.Import("modelscope");
+                            //dynamic model_dir = modelscope.snapshot_download("AI-ModelScope/bge-large-zh-v1.5", revision: "master");
+                            dynamic model_dir = modelscope.snapshot_download(modelName, revision: "master");
+                            dynamic HuggingFaceBgeEmbeddingstemp = Py.Import("langchain.embeddings");
+                            dynamic HuggingFaceBgeEmbeddings = HuggingFaceBgeEmbeddingstemp.HuggingFaceBgeEmbeddings;
+                            string model_name = model_dir;
+                            dynamic model_kwargs = new PyDict();
+                            model_kwargs["device"] = new PyString("cpu");
+                            dynamic hugginmodel = HuggingFaceBgeEmbeddings(
+                                  model_name: model_dir,
+                                  model_kwargs: model_kwargs
+                              );
+                            model = hugginmodel;
+                            return hugginmodel;
+                        }
+                    }
+                    catch(Exception ex)
+                    {
+                        throw ex;
+                    }
+                }
+                else
+                    return model;
+            }
+        }
+
+        public static Task<float[]> GetEmbedding(string queryStr)
+        {
+            using (Py.GIL())
+            {
+                PyObject queryResult = model.embed_query(queryStr);
+                var floatList = queryResult.As<float[]>();
+                return Task.FromResult(floatList); ;
+            }
+        }
+
+        public static int TokenCount(string queryStr)
+        {
+            using (Py.GIL())
+            {
+                PyObject queryResult = model.client.tokenize(queryStr);
+                int len = (int)(queryResult.Length());
+                return len; 
+            }
+        }
+
+        public static void Dispose()
+        {
+            Console.WriteLine("python dispose");
+        }
+    }
+}

src/AntSK.Domain/Domain/Service/ChatService.cs

@@ -16,6 +16,8 @@ using ChatHistory = Microsoft.SemanticKernel.ChatCompletion.ChatHistory;
 using Microsoft.SemanticKernel.Plugins.Core;
 using Azure.Core;
 using AntSK.Domain.Domain.Model;
+using AntSK.LLM.StableDiffusion;
+using System.Drawing;
 
 namespace AntSK.Domain.Domain.Service
 {
@@ -23,7 +25,8 @@ namespace AntSK.Domain.Domain.Service
     public class ChatService(
         IKernelService _kernelService,
         IKMService _kMService,
-        IKmsDetails_Repositories _kmsDetails_Repositories
+        IKmsDetails_Repositories _kmsDetails_Repositories,
+        IAIModels_Repositories _aIModels_Repositories
         ) : IChatService
     {
         /// <summary>
@@ -122,6 +125,58 @@ namespace AntSK.Domain.Domain.Service
             }
         }
 
+
+        public async Task<string> SendImgByAppAsync(Apps app, string questions)
+        {
+            var imageModel = _aIModels_Repositories.GetFirst(p => p.Id == app.ImageModelID);
+            KernelArguments args = new() {
+                { "input", questions }
+            };
+            var _kernel = _kernelService.GetKernelByApp(app);
+            var temperature = app.Temperature / 100; //存的是0~100需要缩小
+            OpenAIPromptExecutionSettings settings = new() { Temperature = temperature };
+            var func = _kernel.CreateFunctionFromPrompt("你是一个StableDiffusion提示词助手,需要将用户问题转化为StableDiffusion的英文提示词并返回,请注意只返回提示词不要有其他多余内容,用户的问题是：{{$input}}", settings);
+            var chatResult = await _kernel.InvokeAsync(function: func, arguments: args);
+            if (chatResult.IsNotNull())
+            {
+                string prompt = chatResult.GetValue<string>();
+                if (!SDHelper.IsInitialized)
+                {
+                    Structs.ModelParams modelParams = new Structs.ModelParams
+                    {
+                        ModelPath = imageModel.ModelName,
+                        RngType = Structs.RngType.CUDA_RNG,
+                        //VaePath = vaePath,
+                        //KeepVaeOnCpu = keepVaeOnCpu,
+                        //VaeTiling = vaeTiling,
+                        //LoraModelDir = loraModelDir,
+                    };
+                    bool result = SDHelper.Initialize(modelParams);
+                }
+
+                Structs.TextToImageParams textToImageParams = new Structs.TextToImageParams
+                {
+                    Prompt = prompt,
+                    NegativePrompt = "2d, 3d, cartoon, paintings",
+                    SampleMethod = (Structs.SampleMethod)Enum.Parse(typeof(Structs.SampleMethod), "EULER_A"),
+                    Width = 256,
+                    Height = 256,
+                    NormalizeInput = true,
+                    ClipSkip = -1,
+                    CfgScale = 7,
+                    SampleSteps = 20,
+                    Seed = -1,
+                };
+                Bitmap[] outputImages = SDHelper.TextToImage(textToImageParams);
+                var base64 = ImageUtils.BitmapToBase64(outputImages[0]);
+                return base64;
+            }
+            else
+            {
+                return "";
+            }
+        }
+
         public async Task<ChatHistory> GetChatHistory(List<MessageInfo> MessageList)
         {
             ChatHistory history = new ChatHistory();

src/AntSK.Domain/Domain/Service/KMService.cs

@@ -1,5 +1,6 @@
 using AntDesign;
 using AntSK.Domain.Common.DependencyInjection;
+using AntSK.Domain.Common.Embedding;
 using AntSK.Domain.Domain.Interface;
 using AntSK.Domain.Domain.Model.Constant;
 using AntSK.Domain.Domain.Model.Dto;
@@ -147,6 +148,11 @@ namespace AntSK.Domain.Domain.Service
                     var embedder = new LLamaEmbedder(weights, parameters);
                     memory.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGenerator(embedder));
                     break;
+                case Model.Enum.AIType.BgeEmbedding:
+                    string pyDll = embedModel.EndPoint;
+                    string bgeEmbeddingModelName = embedModel.ModelName;
+                    memory.WithBgeTextEmbeddingGeneration(new HuggingfaceTextEmbeddingGenerator(pyDll,bgeEmbeddingModelName));
+                    break;
                 case Model.Enum.AIType.DashScope:
                     memory.WithDashScopeDefaults(embedModel.ModelKey);
                     break;
@@ -183,6 +189,14 @@ namespace AntSK.Domain.Domain.Service
                     var executor = new StatelessExecutor(weights, parameters);
                     memory.WithLLamaSharpTextGeneration(new LlamaSharpTextGenerator(weights, context, executor));
                     break;
+                case Model.Enum.AIType.LLamaFactory:
+
+                    memory.WithOpenAITextGeneration(new OpenAIConfig()
+                    {
+                        APIKey = "123",
+                        TextModel = chatModel.ModelName
+                    }, null, chatHttpClient);
+                    break;
                 case Model.Enum.AIType.DashScope:
                     memory.WithDashScopeTextGeneration(new Cnblogs.KernelMemory.AI.DashScope.DashScopeConfig
                     {

src/AntSK.Domain/Repositories/AI/App/Apps.cs

@@ -44,6 +44,7 @@ namespace AntSK.Domain.Repositories
         /// </summary>
         public string? EmbeddingModelID { get; set; }
 
+        public string? ImageModelID { get; set; }
         /// <summary>
         /// 温度
         /// </summary>

src/AntSK.Domain/Utils/ConvertUtils.cs

@@ -250,5 +250,16 @@ namespace AntSK.Domain.Utils
 
             return nameValueCollection.ToString();
         }
+
+        /// <summary>
+        /// 忽略大小写匹配
+        /// </summary>
+        /// <param name="s"></param>
+        /// <param name="value"></param>
+        /// <returns></returns>
+        public static bool ComparisonIgnoreCase(this string s, string value)
+        {
+            return s.Equals(value, StringComparison.OrdinalIgnoreCase);
+        }
     }
 }

src/AntSK.Domain/Utils/ImageUtils.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing.Imaging;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AntSK.Domain.Utils
+{
+    public class ImageUtils
+    {
+        public static string BitmapToBase64(Bitmap bitmap)
+        {
+            using (MemoryStream memoryStream = new MemoryStream())
+            {
+                // 保存为JPEG格式，也可以选择Png，Gif等等
+                bitmap.Save(memoryStream, ImageFormat.Jpeg);
+
+                // 获取内存流的字节数组
+                byte[] imageBytes = memoryStream.ToArray();
+
+                // 将字节转换为Base64字符串
+                string base64String = Convert.ToBase64String(imageBytes);
+                return base64String;
+            }
+        }
+        public static List<string> BitmapListToBase64(Bitmap[] bitmaps)
+        {
+            List<string> base64Strings = new List<string>();
+
+            foreach (Bitmap bitmap in bitmaps)
+            {
+                base64Strings.Add(BitmapToBase64(bitmap));
+            }
+            return base64Strings;
+        }
+    }
+}

src/AntSK.LLamaFactory/llamafactory/api_start.py

@@ -1,27 +0,0 @@
-import subprocess
-import shlex
-import os
-class Start(object):
-
-    def __init__(self,model_name_or_path):
-        self.model_name_or_path=model_name_or_path
-
-    def StartCommand(self):
-        os.environ['CUDA_VISIBLE_DEVICES'] = '0'
-        os.environ['API_PORT'] = '8000'
-        # 构建要执行的命令
-        command = (
-            'python api_demo.py'
-            ' --model_name_or_path E:/model/Qwen1.5-0.5B-Chat_back'
-            ' --template default '
-        )
-
-        # 使用shlex.split()去安全地分割命令字符串
-        command = shlex.split(command)
-
-        # 执行命令
-        subprocess.run(command, shell=True)
-
-if __name__ == "__main__":
-    star= Start('model_name_or_path')
-    star.StartCommand()

src/AntSK.LLamaFactory/llamafactory/cli_demo.py

@@ -1,49 +0,0 @@
-from llmtuner import ChatModel
-from llmtuner.extras.misc import torch_gc
-
-
-try:
-    import platform
-
-    if platform.system() != "Windows":
-        import readline  # noqa: F401
-except ImportError:
-    print("Install `readline` for a better experience.")
-
-
-def main():
-    chat_model = ChatModel()
-    messages = []
-    print("Welcome to the CLI application, use `clear` to remove the history, use `exit` to exit the application.")
-
-    while True:
-        try:
-            query = input("\nUser: ")
-        except UnicodeDecodeError:
-            print("Detected decoding error at the inputs, please set the terminal encoding to utf-8.")
-            continue
-        except Exception:
-            raise
-
-        if query.strip() == "exit":
-            break
-
-        if query.strip() == "clear":
-            messages = []
-            torch_gc()
-            print("History has been removed.")
-            continue
-
-        messages.append({"role": "user", "content": query})
-        print("Assistant: ", end="", flush=True)
-
-        response = ""
-        for new_text in chat_model.stream_chat(messages):
-            print(new_text, end="", flush=True)
-            response += new_text
-        print()
-        messages.append({"role": "assistant", "content": response})
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/llamafactory/evaluate.py

@@ -1,10 +0,0 @@
-from llmtuner import Evaluator
-
-
-def main():
-    evaluator = Evaluator()
-    evaluator.eval()
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/llamafactory/export_model.py

@@ -1,9 +0,0 @@
-from llmtuner import export_model
-
-
-def main():
-    export_model()
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/llamafactory/train_bash.py

@@ -1,14 +0,0 @@
-from llmtuner import run_exp
-
-
-def main():
-    run_exp()
-
-
-def _mp_fn(index):
-    # For xla_spawn (TPUs)
-    main()
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/llamafactory/train_web.py

@@ -1,11 +0,0 @@
-from llmtuner import create_ui
-
-
-def main():
-    demo = create_ui()
-    demo.queue()
-    demo.launch(server_name="0.0.0.0", share=False, inbrowser=True)
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/llamafactory/web_demo.py

@@ -1,11 +0,0 @@
-from llmtuner import create_web_demo
-
-
-def main():
-    demo = create_web_demo()
-    demo.queue()
-    demo.launch(server_name="0.0.0.0", share=False, inbrowser=True)
-
-
-if __name__ == "__main__":
-    main()

src/AntSK.LLamaFactory/requirements.txt

@@ -1,4 +1,4 @@
-torch>=1.13.1
+torch>=1.13.1  --index-url https://download.pytorch.org/whl/cu121
 transformers>=4.37.2
 datasets>=2.14.3
 accelerate>=0.27.2
@@ -15,4 +15,4 @@ fastapi
 sse-starlette
 matplotlib
 fire
-modelscope
+modelscope

src/AntSK.PyNet/AntSK.PyNet.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/AntSK.PyNet/Class1.cs

@@ -0,0 +1,7 @@
+namespace AntSK.PyNet
+{
+    public class Class1
+    {
+
+    }
+}

src/AntSK/AntSK.csproj

@@ -7,6 +7,13 @@
     <DocumentationFile>AntSK.xml</DocumentationFile>
     <NoWarn>CA1050,CA1707,CA2007,VSTHRD111,CS1591,RCS1110,CA5394,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,,SKEXP0012,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0040,SKEXP0041,SKEXP0042,SKEXP0050,SKEXP0051,SKEXP0052,SKEXP0053,SKEXP0054,SKEXP0055,SKEXP0060,SKEXP0061,SKEXP0101,SKEXP0102</NoWarn>
   </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="llamafactory\**" />
+    <Content Remove="llamafactory\**" />
+    <EmbeddedResource Remove="llamafactory\**" />
+    <None Remove="llamafactory\**" />
+  </ItemGroup>
   <ItemGroup>
 
     

src/AntSK/Controllers/KMSController.cs

@@ -35,7 +35,6 @@ namespace AntSK.Controllers
         [HttpPost]
         public async Task<IActionResult> ImportKMSTask(ImportKMSTaskDTO model)
         {
-            Console.WriteLine("api/kms/ImportKMSTask  开始");
             ImportKMSTaskReq req = model.ToDTO<ImportKMSTaskReq>();
             KmsDetails detail = new KmsDetails()
             {
@@ -49,7 +48,6 @@ namespace AntSK.Controllers
             await _kmsDetailsRepositories.InsertAsync(detail);
             req.KmsDetail = detail;
             _taskBroker.QueueWorkItem(req);
-            Console.WriteLine("api/kms/ImportKMSTask  结束");
             return Ok();
         }
     }

src/AntSK/Pages/AppPage/AddApp.razor

@@ -26,30 +26,45 @@
                 <FormItem Label="类型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
                     <RadioGroup @bind-Value="context.Type">
                         <Radio RadioButton Value="@AppType.chat.ToString()">会话应用</Radio>
-                         <Radio RadioButton Value="@AppType.kms.ToString()">知识库</Radio>
-                     </RadioGroup>
-                 </FormItem>
-                 <FormItem Label="描述" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                     <Input Placeholder="请输入描述" @bind-Value="@context.Describe" />
-                 </FormItem>
+                        <Radio RadioButton Value="@AppType.kms.ToString()">知识库</Radio>
+                        <Radio RadioButton Value="@AppType.img.ToString()">做图应用</Radio>
+                    </RadioGroup>
+                </FormItem>
+                <FormItem Label="描述" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                    <Input Placeholder="请输入描述" @bind-Value="@context.Describe" />
+                </FormItem>
 
-                 <FormItem Label="会话模型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                     <Select DataSource="@_chatList"
-                             @bind-Value="@context.ChatModelID"
-                             ValueProperty="c=>c.Id"
-                             LabelProperty="c=>'【'+c.AIType.ToString()+'】'+c.ModelDescription">
-                     </Select>
-                     <Button Type="@ButtonType.Link" OnClick="NavigateModelList">去创建</Button>
-                 </FormItem>
-                <FormItem Label="向量模型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                    <Select DataSource="@_embedignList"
-                            @bind-Value="@context.EmbeddingModelID"
+                <FormItem Label="会话模型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                    <Select DataSource="@_chatList"
+                            @bind-Value="@context.ChatModelID"
                             ValueProperty="c=>c.Id"
                             LabelProperty="c=>'【'+c.AIType.ToString()+'】'+c.ModelDescription">
                     </Select>
                     <Button Type="@ButtonType.Link" OnClick="NavigateModelList">去创建</Button>
                 </FormItem>
-                 @if (@context.Type == AppType.chat.ToString())
+                @if (@context.Type != AppType.img.ToString())
+                {
+                    <FormItem Label="向量模型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Select DataSource="@_embedingList"
+                                @bind-Value="@context.EmbeddingModelID"
+                                ValueProperty="c=>c.Id"
+                                LabelProperty="c=>'【'+c.AIType.ToString()+'】'+c.ModelDescription">
+                        </Select>
+                        <Button Type="@ButtonType.Link" OnClick="NavigateModelList">去创建</Button>
+                    </FormItem>
+                }
+                else              
+                {
+                    <FormItem Label="图片模型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Select DataSource="@_imageList"
+                                @bind-Value="@context.ImageModelID"
+                                ValueProperty="c=>c.Id"
+                                LabelProperty="c=>'【'+c.AIType.ToString()+'】'+c.ModelDescription">
+                        </Select>
+                        <Button Type="@ButtonType.Link" OnClick="NavigateModelList">去创建</Button>
+                    </FormItem>
+                }
+                @if (@context.Type == AppType.chat.ToString())
                 {
 
                     <FormItem Label="提示词" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
@@ -61,8 +76,8 @@
                         <Slider TValue="double" Style="display: inline-block;width: 300px; " Min="0" Max="100" DefaultValue="70" @bind-Value="@context.Temperature" />
                         <span>更发散</span>
                     </FormItem>
-                  
-                    <FormItem Label="API插件列表" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">                      
+
+                    <FormItem Label="API插件列表" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
                         <Select Mode="multiple"
                                 @bind-Values="apiIds"
                                 Placeholder="选择API插件, 选择后会开启自动调用"

src/AntSK/Pages/AppPage/AddApp.razor.cs

@@ -45,7 +45,8 @@ namespace AntSK.Pages.AppPage
         public Dictionary<string, string> _funList = new Dictionary<string, string>();
 
         private List<AIModels> _chatList;
-        private List<AIModels> _embedignList;
+        private List<AIModels> _embedingList;
+        private List<AIModels> _imageList;
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
@@ -53,7 +54,8 @@ namespace AntSK.Pages.AppPage
             _apiList = _apis_Repositories.GetList();
             var models=_aimodels_Repositories.GetList();
             _chatList = models.Where(p => p.AIModelType == AIModelType.Chat).ToList();
-            _embedignList = models.Where(p => p.AIModelType == AIModelType.Embedding).ToList();
+            _embedingList = models.Where(p => p.AIModelType == AIModelType.Embedding).ToList();
+            _imageList = models.Where(p => p.AIModelType == AIModelType.Image).ToList();
 
             _functionService.SearchMarkedMethods();
             foreach (var func in _functionService.Functions)

src/AntSK/Pages/AppPage/AppList.razor

@@ -58,6 +58,10 @@
                                         <Tag Color="@PresetColor.Green.ToString()">知识库</Tag>
                                         
                                     }
+                                    else if (context.Type == AppType.img.ToString())
+                                    {
+                                        <Tag Color="@PresetColor.Lime.ToString()">做图应用</Tag>
+                                    }
                                 </DescriptionTemplate>
                             </CardMeta>
                         </Card>

src/AntSK/Pages/ChatPage/Components/ChatView.razor.cs

@@ -2,13 +2,17 @@
 using AntSK.Domain.Domain.Interface;
 using AntSK.Domain.Domain.Model;
 using AntSK.Domain.Domain.Model.Dto;
+using AntSK.Domain.Domain.Model.Enum;
 using AntSK.Domain.Repositories;
 using AntSK.Domain.Utils;
+using AntSK.LLM.StableDiffusion;
 using Blazored.LocalStorage;
 using Markdig;
 using Microsoft.AspNetCore.Components;
 using Microsoft.JSInterop;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
 using Newtonsoft.Json;
 
 namespace AntSK.Pages.ChatPage.Components
@@ -76,7 +80,9 @@ namespace AntSK.Pages.ChatPage.Components
                 {
                     MessageList.Clear();
                     await _localStorage.SetItemAsync<List<MessageInfo>>("msgs", MessageList);
+                    await InvokeAsync(StateHasChanged);
                     _ = Message.Info("清理成功");
+            
                 }
             }
             else
@@ -107,9 +113,7 @@ namespace AntSK.Pages.ChatPage.Components
                 Sendding = true;
                 await SendAsync(_messageInput,filePath);
                 _messageInput = "";
-                Sendding = false;
-
-                await _localStorage.SetItemAsync<List<MessageInfo>>("msgs", MessageList);
+                Sendding = false; 
             }
             catch (System.Exception ex)
             {
@@ -144,23 +148,48 @@ namespace AntSK.Pages.ChatPage.Components
             {
                 history = await _chatService.GetChatHistory(MessageList);
             }
-            
-            switch (app.Type)
-            {
-                case "chat" when filePath == null||app.EmbeddingModelID.IsNull():
-                    //普通会话
-                    await SendChat(questions, history, app);
-                    break;
 
-                default:
-                    //知识库问答
-                    await SendKms(questions, history,  app, filePath);
-                    break;
+            if (app.Type == AppType.chat.ToString() && (filePath == null || app.EmbeddingModelID.IsNull()))
+            {
+                await SendChat(questions, history, app);
+            }
+            else if (app.Type == AppType.kms.ToString() || filePath != null || app.EmbeddingModelID.IsNotNull())
+            {
+                await SendKms(questions, history, app, filePath);
+               
+            }
+            else if (app.Type == AppType.img.ToString())
+            {
+                await SendImg(questions,app);
             }
 
+            //缓存消息记录
+            if (app.Type != AppType.img.ToString())
+            {
+                await _localStorage.SetItemAsync<List<MessageInfo>>("msgs", MessageList);
+            }
+
+
             return await Task.FromResult(true);
         }
 
+        private async Task SendImg(string questions,Apps app)
+        {
+            MessageInfo info = new MessageInfo();
+            info.ID = Guid.NewGuid().ToString();
+            info.CreateTime = DateTime.Now;
+            var base64= await _chatService.SendImgByAppAsync(app, questions);
+            if (string.IsNullOrEmpty(base64))
+            {
+                info.HtmlAnswers = "生成失败";
+            }
+            else 
+            {
+                info.HtmlAnswers = $"<img src=\"data:image/jpeg;base64,{base64}\" alt=\"Base64 Image\" />";
+            }
+            MessageList.Add(info);
+        }
+
         /// <summary>
         /// 发送知识库问答
         /// </summary>

src/AntSK/Pages/Setting/AIModel/AddModel.razor

@@ -24,18 +24,32 @@
                 </FormItem>
 
                 <FormItem Label="AI类型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                    <EnumRadioGroup @bind-Value="context.AIType"></EnumRadioGroup>
+                    <EnumRadioGroup @bind-Value="context.AIType" ButtonStyle="RadioButtonStyle.Solid"> </EnumRadioGroup>
                 </FormItem>
                 <FormItem Label="模型类型" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
                     <RadioGroup @bind-Value="context.AIModelType">
-                        <Radio RadioButton Value="@(AIModelType.Chat)">会话模型</Radio>
-                        <Radio RadioButton Value="@(AIModelType.Embedding)">向量模型</Radio>
+                    
+                        @if (context.AIType == AIType.StableDiffusion)
+                        {
+                            <Radio RadioButton Value="@(AIModelType.Image)">图片模型</Radio>
+                        }
+                        else  
+                        {
+                            @if (context.AIType != AIType.BgeEmbedding)
+                            {
+                                <Radio RadioButton Value="@(AIModelType.Chat)">会话模型</Radio>
+                            }
+                            @if (context.AIType != AIType.LLamaFactory && context.AIType != AIType.Mock)
+                            {
+                                <Radio RadioButton Value="@(AIModelType.Embedding)">向量模型</Radio>
+                            }
+                        }
                     </RadioGroup>
                 </FormItem>
                 @if (context.AIModelType == AIModelType.Embedding)
                 {
                     <FormItem Label="注意事项" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                        <b>请不要使用不同维度的向量模型，否则会导致无法向量存储</b>
+                        <span style="color:red"><b>请不要使用不同维度的向量模型，否则会导致无法向量存储</b></span>
                     </FormItem>
                 }
 
@@ -84,7 +98,7 @@
                         <Input Placeholder="请输入模型名称" @bind-Value="@context.ModelName" />
                     </FormItem>
                 }
-                @if (context.AIType == AIType.LLamaSharp)
+                @if (context.AIType == AIType.LLamaSharp || context.AIType == AIType.StableDiffusion)
                 {
                     <FormItem Label="模型路径" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
                         <InputGroup>
@@ -125,10 +139,30 @@
                             }
                         </InputGroup>
                     </FormItem>
-                     <FormItem Label="环境安装" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
-                        <Button Type="primary" OnClick="PipInstall" >初始化</Button>
+                    <FormItem Label="环境安装" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Button Type="primary" OnClick="PipInstall">初始化</Button>
                     </FormItem>
                 }
+
+                @if (context.AIType == AIType.BgeEmbedding)
+                {
+                    <FormItem Label="模型名称" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Select DataSource="@bgeEmbeddingList"
+                                @bind-Value="@context.ModelName"
+                                ValueProperty="c=>c"
+                                LabelProperty="c=>c">
+                        </Select>
+                    </FormItem>
+                    <FormItem Label="PythonDll路径" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Input Placeholder="D:\Programs\Python\Python311\python311.dll" @bind-Value="@context.EndPoint" />
+                    </FormItem>
+                    <FormItem Label="下载并初始化" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                        <Spin Tip="请等待..." Spinning="@(BgeIsStart)" >
+                            <Button Type="primary" Disabled="@(BgeIsStart)" OnClick="BgeDownload">@BgeBtnText</Button>
+                        </Spin>
+                    </FormItem>
+                }
+
                 @if (context.AIType == AIType.Mock)
                 {
                 }
@@ -168,3 +202,7 @@
 </Modal>
 
 
+@code
+{
+
+}

src/AntSK/Pages/Setting/AIModel/AddModel.razor.cs

@@ -3,6 +3,7 @@ using AntDesign.ProLayout;
 using AntSK.Domain.Domain.Interface;
 using AntSK.Domain.Domain.Model.Constant;
 using AntSK.Domain.Domain.Model.Enum;
+using AntSK.Domain.Domain.Other;
 using AntSK.Domain.Domain.Service;
 using AntSK.Domain.Options;
 using AntSK.Domain.Repositories;
@@ -12,6 +13,7 @@ using BlazorComponents.Terminal;
 using DocumentFormat.OpenXml.Office2010.Excel;
 using Downloader;
 using Microsoft.AspNetCore.Components;
+using NRedisStack.Search;
 using System.ComponentModel;
 
 namespace AntSK.Pages.Setting.AIModel
@@ -58,6 +60,10 @@ namespace AntSK.Pages.Setting.AIModel
         private TerminalParagraph para;
         private bool _logModalVisible;
 
+        private List<string> bgeEmbeddingList = new List<string>() { "AI-ModelScope/bge-small-zh-v1.5", "AI-ModelScope/bge-base-zh-v1.5", "AI-ModelScope/bge-large-zh-v1.5" };
+        private bool BgeIsStart = false;
+        private string BgeBtnText = "初始化";
+
         protected override async Task OnInitializedAsync()
         {
             try
@@ -75,12 +81,25 @@ namespace AntSK.Pages.Setting.AIModel
                     llamaFactoryIsStart = llamaFactoryDic.Value == "false" ? false : true;
                 }
 
+               
                 //目前只支持gguf的 所以筛选一下
-                _modelFiles = Directory.GetFiles(Path.Combine(Directory.GetCurrentDirectory(), LLamaSharpOption.FileDirectory)).Where(p=>p.Contains(".gguf")).ToArray();
+                _modelFiles = Directory.GetFiles(Path.Combine(Directory.GetCurrentDirectory(), LLamaSharpOption.FileDirectory)).Where(p=> p.Contains(".gguf")||p.Contains(".ckpt")|| p.Contains(".safetensors")).ToArray();
                 if (!string.IsNullOrEmpty(ModelPath))
                 {
+                    string extension = Path.GetExtension(ModelPath);
+                    switch (extension)
+                    {
+                        case ".gguf":
+                            _aiModel.AIType = AIType.LLamaSharp;
+                            break;
+                        case ".safetensors":
+                        case ".ckpt":
+                            _aiModel.AIType = AIType.StableDiffusion;
+                            break;
+
+                    }
                     //下载页跳入
-                    _aiModel.AIType = AIType.LLamaSharp;
+                 
                     _downloadModalVisible = true;
 
                     _downloadUrl = $"https://hf-mirror.com{ModelPath.Replace("---","/")}";
@@ -245,9 +264,40 @@ namespace AntSK.Pages.Setting.AIModel
             {
                 _logModalVisible = true;
                 _ILLamaFactoryService.LogMessageReceived += CmdLogHandler;
-                _ILLamaFactoryService.PipInstall();
+                await _ILLamaFactoryService.PipInstall();
             }
         }
+
+        private async Task BgeDownload()
+        {
+            if (string.IsNullOrEmpty(_aiModel.ModelName))
+            {
+                _ = Message.Error("请输入模型名称！", 2);
+                return;
+            }
+            if (string.IsNullOrEmpty(_aiModel.EndPoint))
+            {
+                _ = Message.Error("请输入正确的Python dll路径！", 2);
+                return;
+            }
+
+            BgeIsStart = true;
+            BgeBtnText = "正在初始化...";
+            await Task.Run(() =>
+            {
+                try
+                {
+                    EmbeddingConfig.LoadModel(_aiModel.EndPoint, _aiModel.ModelName);
+                    BgeBtnText = "初始化完成";
+                    BgeIsStart = false;
+                }
+                catch (System.Exception ex)
+                {
+                    _ = Message.Error(ex.Message, 2);
+                    BgeIsStart = false;
+                }
+            });   
+        }
         private async Task CmdLogHandler(string message)
         {
             await InvokeAsync(() =>
@@ -263,5 +313,10 @@ namespace AntSK.Pages.Setting.AIModel
         private void OnCancelLog() {
             _logModalVisible = false;
         }
+
+        private void AITypeModelChange() 
+        { 
+        
+        }
     }
 }

src/AntSK/Pages/Setting/AIModel/ModelDown.razor

@@ -10,12 +10,19 @@
 
 <PageContainer Title="模型列表">
     <Content>
+        <RadioGroup @bind-Value="@_modelType">
+            <Radio Value="@("gguf")" DefaultChecked=true>LLama本地模型(gguf)</Radio>
+            <Radio Value="@("safetensors")">StableDiffusion(safetensors)</Radio>
+            <Radio Value="@("ckpt")">StableDiffusion2(ckpt)</Radio>
+        </RadioGroup>
         <div style="text-align: center;">
+            
             <Search Placeholder="输入回车"
                     EnterButton="@("搜索")"
                     Size="large"
                     Style="max-width: 522px; width: 100%;"
                     OnSearch="Search" />
+          
         </div>
     </Content>
     <ChildContent>

src/AntSK/Pages/Setting/AIModel/ModelDown.razor.cs

@@ -16,7 +16,7 @@ namespace AntSK.Pages.Setting.AIModel
         private readonly IList<string> _selectCategories = new List<string>();
 
         private List<HfModels> _modelList = new List<HfModels>();
-
+        private string _modelType;
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
@@ -27,7 +27,7 @@ namespace AntSK.Pages.Setting.AIModel
         {
             var param = searchKey.ConvertToString().Split(" ");
 
-            string urlBase = "https://hf-mirror.com/models-json?sort=trending&search=gguf";
+            string urlBase = $"https://hf-mirror.com/models-json?sort=trending&search={_modelType}";
             if (param.Count() > 0)
             {
                 urlBase += "+" + string.Join("+", param);

src/AntSK/Pages/Setting/AIModel/ModelList.razor

@@ -70,6 +70,14 @@
                                             {
                                                 <Tag Color="@PresetColor.Cyan.ToString()">LLamaFactory</Tag>
                                             }
+                                            else if (context.AIType == AIType.BgeEmbedding)
+                                            {
+                                                <Tag Color="@PresetColor.Gold.ToString()">BgeEmbedding</Tag>
+                                            }
+                                            else if (context.AIType == AIType.StableDiffusion)
+                                            {
+                                                <Tag Color="@PresetColor.Lime.ToString()">StableDiffusion</Tag>
+                                            }
                                         </p>
                                     </div>
 
@@ -85,6 +93,10 @@
                                             {
                                                 <Tag Color="@PresetColor.Green.ToString()">向量模型</Tag>
                                             }
+                                            else if (context.AIModelType == AIModelType.Image)
+                                            {
+                                                <Tag Color="@PresetColor.Lime.ToString()">图片模型</Tag>
+                                            }
                                         </p>
                                     </div>
                                     <div class="listContentItem" style="width:20%">

src/AntSK/Services/OpenApi/OpenApiService.cs

@@ -236,15 +236,15 @@ namespace AntSK.Services.OpenApi
             for (int i = 0; i < model.messages.Count() - 1; i++)
             {
                 var item = model.messages[i];
-                if (item.role.ToLower() == "user")
+                if (item.role.ComparisonIgnoreCase("user"))
                 {
                     history.AddUserMessage(item.content);
                 }
-                else if (item.role.ToLower() == "assistant")
+                else if (item.role.ComparisonIgnoreCase("assistant"))
                 {
                     history.AddAssistantMessage(item.content);
                 }
-                else if (item.role.ToLower() == "system")
+                else if (item.role.ComparisonIgnoreCase("system"))
                 {
                     history.AddSystemMessage(item.content);
                 }

src/AntSK/wwwroot/data/menu.json

@@ -57,5 +57,11 @@
         "key": "setting.modeldown"
       }
     ]
+  },
+  {
+    "path": "http://antsk.cn/",
+    "name": "文档",
+    "key": "antskdoc",
+    "icon": "question-circle"
   }
 ]

src/AntSk.LLM/AntSK.LLM.csproj

@@ -13,6 +13,13 @@
     <PackageReference Include="Cnblogs.SemanticKernel.Connectors.DashScope" Version="0.3.2" />
     <PackageReference Include="Microsoft.SemanticKernel" Version="1.6.3" />
     <PackageReference Include="Sdcb.SparkDesk" Version="3.0.0" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="stable-diffusion.dll">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>

src/AntSk.LLM/StableDiffusion/Native.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace AntSK.LLM.StableDiffusion
+{
+    using static AntSK.LLM.StableDiffusion.Structs;
+    using int32_t = Int32;
+    using int64_t = Int64;
+    using SdContext = IntPtr;
+    using SDImagePtr = IntPtr;
+    using UpscalerContext = IntPtr;
+
+    internal class Native
+    {
+        const string DllName = "stable-diffusion";
+
+        internal delegate void SdLogCallback(SdLogLevel level, [MarshalAs(UnmanagedType.LPStr)] string text, IntPtr data);
+        internal delegate void SdProgressCallback(int step, int steps, float time, IntPtr data);
+
+        [DllImport(DllName, EntryPoint = "new_sd_ctx", CallingConvention = CallingConvention.Cdecl)]
+        internal extern static SdContext new_sd_ctx(string model_path,
+                                                     string vae_path,
+                                                     string taesd_path,
+                                                     string control_net_path_c_str,
+                                                     string lora_model_dir,
+                                                     string embed_dir_c_str,
+                                                     string stacked_id_embed_dir_c_str,
+                                                     bool vae_decode_only,
+                                                     bool vae_tiling,
+                                                     bool free_params_immediately,
+                                                     int n_threads,
+                                                     WeightType weightType,
+                                                     RngType rng_type,
+                                                     ScheduleType s,
+                                                     bool keep_clip_on_cpu,
+                                                     bool keep_control_net_cpu,
+                                                     bool keep_vae_on_cpu);
+
+
+        [DllImport(DllName, EntryPoint = "txt2img", CallingConvention = CallingConvention.Cdecl)]
+        internal static extern SDImagePtr txt2img(SdContext sd_ctx,
+                           string prompt,
+                           string negative_prompt,
+                           int clip_skip,
+                           float cfg_scale,
+                           int width,
+                           int height,
+                           SampleMethod sample_method,
+                           int sample_steps,
+                           int64_t seed,
+                           int batch_count,
+                           SDImagePtr control_cond,
+                           float control_strength,
+                           float style_strength,
+                           bool normalize_input,
+                           string input_id_images_path);
+
+        [DllImport(DllName, EntryPoint = "img2img", CallingConvention = CallingConvention.Cdecl)]
+        internal static extern SDImagePtr img2img(SdContext sd_ctx,
+                    SDImage init_image,
+                    string prompt_c_str,
+                    string negative_prompt_c_str,
+                    int clip_skip,
+                    float cfg_scale,
+                    int width,
+                    int height,
+                    SampleMethod sample_method,
+                    int sample_steps,
+                    float strength,
+                    int64_t seed,
+                    int batch_count);
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern IntPtr preprocess_canny(IntPtr imgData,
+                                 int width,
+                                 int height,
+                                 float high_threshold,
+                                 float low_threshold,
+                                 float weak,
+                                 float strong,
+                                 bool inverse);
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern UpscalerContext new_upscaler_ctx(string esrgan_path,
+                                        int n_threads,
+                                        WeightType wtype);
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern int32_t get_num_physical_cores();
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern void free_sd_ctx(SdContext sd_ctx);
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern void free_upscaler_ctx(UpscalerContext upscaler_ctx);
+
+        [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern SDImage upscale(UpscalerContext upscaler_ctx, SDImage input_image, int upscale_factor);
+
+        [DllImport(DllName, EntryPoint = "sd_set_log_callback", CallingConvention = CallingConvention.Cdecl)]
+        internal static extern void sd_set_log_callback(SdLogCallback cb, IntPtr data);
+
+        [DllImport(DllName, EntryPoint = "sd_set_progress_callback", CallingConvention = CallingConvention.Cdecl)]
+        internal static extern void sd_set_progress_callback(SdProgressCallback cb, IntPtr data);
+
+    }
+}
+

src/AntSk.LLM/StableDiffusion/SDHelper.cs

@@ -0,0 +1,217 @@
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Runtime.InteropServices;
+
+namespace AntSK.LLM.StableDiffusion
+{
+    using static AntSK.LLM.StableDiffusion.Structs;
+    using SdContext = IntPtr;
+    using SDImagePtr = IntPtr;
+    using UpscalerContext = IntPtr;
+
+    public static class SDHelper
+    {
+        public static bool IsInitialized => SdContext.Zero != sd_ctx;
+        public static bool IsUpscalerInitialized => UpscalerContext.Zero != upscaler_ctx;
+
+        private static SdContext sd_ctx = new SdContext();
+        private static UpscalerContext upscaler_ctx = new UpscalerContext();
+
+        public static event EventHandler<StableDiffusionEventArgs.StableDiffusionLogEventArgs> Log;
+        public static event EventHandler<StableDiffusionEventArgs.StableDiffusionProgressEventArgs> Progress;
+        static readonly Native.SdLogCallback sd_Log_Cb;
+        static readonly Native.SdProgressCallback sd_Progress_Cb;
+
+        static SDHelper()
+        {
+            sd_Log_Cb = new Native.SdLogCallback(OnNativeLog);
+            Native.sd_set_log_callback(sd_Log_Cb, IntPtr.Zero);
+
+            sd_Progress_Cb = new Native.SdProgressCallback(OnProgressRunning);
+            Native.sd_set_progress_callback(sd_Progress_Cb, IntPtr.Zero);
+
+        }
+
+        public static bool Initialize(ModelParams modelParams)
+        {
+            sd_ctx = Native.new_sd_ctx(modelParams.ModelPath,
+                                        modelParams.VaePath,
+                                        modelParams.TaesdPath,
+                                        modelParams.ControlnetPath,
+                                        modelParams.LoraModelDir,
+                                        modelParams.EmbeddingsPath,
+                                        modelParams.StackedIdEmbeddingsPath,
+                                        modelParams.VaeDecodeOnly,
+                                        modelParams.VaeTiling,
+                                        modelParams.FreeParamsImmediately,
+                                        modelParams.Threads,
+                                        modelParams.SdType,
+                                        modelParams.RngType,
+                                        modelParams.Schedule,
+                                        modelParams.KeepClipOnCpu,
+                                        modelParams.KeepControlNetOnCpu,
+                                        modelParams.KeepVaeOnCpu);
+            return SdContext.Zero != sd_ctx;
+        }
+
+        public static bool InitializeUpscaler(UpscalerParams @params)
+        {
+            upscaler_ctx = Native.new_upscaler_ctx(@params.ESRGANPath, @params.Threads, @params.SdType);
+            return UpscalerContext.Zero != upscaler_ctx;
+        }
+
+        public static void FreeSD()
+        {
+            if (SdContext.Zero != sd_ctx)
+            {
+                Native.free_sd_ctx(sd_ctx);
+                sd_ctx = SdContext.Zero;
+            }
+        }
+
+        public static void FreeUpscaler()
+        {
+            if (UpscalerContext.Zero != upscaler_ctx)
+            {
+                Native.free_upscaler_ctx(upscaler_ctx);
+                upscaler_ctx = UpscalerContext.Zero;
+            }
+        }
+
+        public static Bitmap[] TextToImage(TextToImageParams textToImageParams)
+        {
+            if (!IsInitialized) throw new ArgumentNullException("Model not loaded!");
+
+            SDImagePtr sd_Image_ptr = Native.txt2img(sd_ctx,
+                          textToImageParams.Prompt,
+                          textToImageParams.NegativePrompt,
+                          textToImageParams.ClipSkip,
+                          textToImageParams.CfgScale,
+                          textToImageParams.Width,
+                          textToImageParams.Height,
+                          textToImageParams.SampleMethod,
+                          textToImageParams.SampleSteps,
+                          textToImageParams.Seed,
+                          textToImageParams.BatchCount,
+                          SDImagePtr.Zero,
+                          textToImageParams.ControlStrength,
+                          textToImageParams.StyleStrength,
+                          textToImageParams.NormalizeInput,
+                          textToImageParams.InputIdImagesPath);
+
+            Bitmap[] images = new Bitmap[textToImageParams.BatchCount];
+            for (int i = 0; i < textToImageParams.BatchCount; i++)
+            {
+                SDImage sd_image = Marshal.PtrToStructure<SDImage>(sd_Image_ptr + i * Marshal.SizeOf<SDImage>());
+                images[i] = GetBitmapFromSdImage(sd_image);
+            }
+            return images;
+        }
+
+
+        public static Bitmap ImageToImage(ImageToImageParams imageToImageParams)
+        {
+            if (!IsInitialized) throw new ArgumentNullException("Model not loaded!");
+            SDImage input_sd_image = GetSDImageFromBitmap(imageToImageParams.InputImage);
+
+            SDImagePtr sdImgPtr = Native.img2img(sd_ctx,
+                  input_sd_image,
+                  imageToImageParams.Prompt,
+                  imageToImageParams.NegativePrompt,
+                  imageToImageParams.ClipSkip,
+                  imageToImageParams.CfgScale,
+                  imageToImageParams.Width,
+                  imageToImageParams.Height,
+                  imageToImageParams.SampleMethod,
+                  imageToImageParams.SampleSteps,
+                  imageToImageParams.Strength,
+                  imageToImageParams.Seed,
+                  imageToImageParams.BatchCount);
+            SDImage sdImg = Marshal.PtrToStructure<SDImage>(sdImgPtr);
+
+            return GetBitmapFromSdImage(sdImg);
+        }
+
+        public static Bitmap UpscaleImage(Bitmap image, int upscaleFactor)
+        {
+            if (!IsUpscalerInitialized) throw new ArgumentNullException("Upscaler not loaded!");
+            SDImage inputSDImg = GetSDImageFromBitmap(image);
+            SDImage result = Native.upscale(upscaler_ctx, inputSDImg, upscaleFactor);
+            return GetBitmapFromSdImage(result);
+        }
+
+        private static Bitmap GetBitmapFromSdImage(SDImage sd_Image)
+        {
+            int width = (int)sd_Image.Width;
+            int height = (int)sd_Image.Height;
+            int channel = (int)sd_Image.Channel;
+            byte[] bytes = new byte[width * height * channel];
+            Marshal.Copy(sd_Image.Data, bytes, 0, bytes.Length);
+            Bitmap bmp = new Bitmap(width, height, PixelFormat.Format24bppRgb);
+            int stride = bmp.Width * channel;
+            byte[] des = new byte[bytes.Length];
+            for (int i = 0; i < height; i++)
+            {
+                for (int j = 0; j < width; j++)
+                {
+                    des[stride * i + channel * j + 0] = bytes[stride * i + channel * j + 2];
+                    des[stride * i + channel * j + 1] = bytes[stride * i + channel * j + 1];
+                    des[stride * i + channel * j + 2] = bytes[stride * i + channel * j + 0];
+                }
+            }
+            BitmapData bitmapData = bmp.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.WriteOnly, bmp.PixelFormat);
+            Marshal.Copy(des, 0, bitmapData.Scan0, bytes.Length);
+            bmp.UnlockBits(bitmapData);
+
+            return bmp;
+        }
+
+        private static SDImage GetSDImageFromBitmap(Bitmap bmp)
+        {
+            int width = bmp.Width;
+            int height = bmp.Height;
+            int channel = Bitmap.GetPixelFormatSize(bmp.PixelFormat) / 8;
+            int stride = width * channel;
+            byte[] bytes = new byte[width * height * channel];
+            BitmapData bitmapData = bmp.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, bmp.PixelFormat);
+            Marshal.Copy(bitmapData.Scan0, bytes, 0, bytes.Length);
+            bmp.UnlockBits(bitmapData);
+
+            byte[] sdImageBytes = new byte[bytes.Length];
+            for (int i = 0; i < height; i++)
+            {
+                for (int j = 0; j < width; j++)
+                {
+                    sdImageBytes[stride * i + j * 3 + 0] = bytes[stride * i + j * 3 + 2];
+                    sdImageBytes[stride * i + j * 3 + 1] = bytes[stride * i + j * 3 + 1];
+                    sdImageBytes[stride * i + j * 3 + 2] = bytes[stride * i + j * 3 + 0];
+                }
+            }
+
+            SDImage sd_Image = new SDImage
+            {
+                Width = (uint)width,
+                Height = (uint)height,
+                Channel = 3,
+                Data = Marshal.UnsafeAddrOfPinnedArrayElement(sdImageBytes, 0),
+            };
+
+            return sd_Image;
+        }
+
+        private static void OnNativeLog(SdLogLevel level, string text, IntPtr data)
+        {
+            Log?.Invoke(null, new StableDiffusionEventArgs.StableDiffusionLogEventArgs { Level = level, Text = text });
+        }
+
+        private static void OnProgressRunning(int step, int steps, float time, IntPtr data)
+        {
+            Progress?.Invoke(null, new StableDiffusionEventArgs.StableDiffusionProgressEventArgs { Step = step, Steps = steps, Time = time });
+        }
+
+
+    }
+}
+
+

src/AntSk.LLM/StableDiffusion/StableDiffusionEventArgs.cs

@@ -0,0 +1,33 @@
+using System;
+using static AntSK.LLM.StableDiffusion.Structs;
+
+namespace AntSK.LLM.StableDiffusion
+{
+    public class StableDiffusionEventArgs
+    {
+        public class StableDiffusionProgressEventArgs : EventArgs
+        {
+            #region Properties & Fields
+
+            public int Step { get; set; }
+            public int Steps { get; set; }
+            public float Time { get; set; }
+            public IntPtr Data { get; set; }
+
+            public double Progress => (double)Step / Steps;
+            public float IterationsPerSecond => 1.0f / Time;
+
+            #endregion
+        }
+
+        public class StableDiffusionLogEventArgs : EventArgs
+        {
+            #region Properties & Fields
+
+            public SdLogLevel Level { get; set; }
+            public string Text { get; set; }
+
+            #endregion
+        }
+    }
+}

src/AntSk.LLM/StableDiffusion/StableDiffusionService.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AntSK.LLM.StableDiffusion
+{
+    public static class StableDiffusionService
+    {
+
+    }
+}

src/AntSk.LLM/StableDiffusion/Structs.cs

@@ -0,0 +1,154 @@
+using System;
+using System.Drawing;
+using System.Runtime.InteropServices;
+
+
+namespace AntSK.LLM.StableDiffusion
+{
+    using int64_t = Int64;
+    using uint32_t = UInt32;
+
+    public class Structs
+    {
+        public class ModelParams
+        {
+            public string ModelPath = string.Empty;
+            public string VaePath = string.Empty;
+            public string TaesdPath = string.Empty;
+            public string ControlnetPath = string.Empty;
+            public string LoraModelDir = string.Empty;
+            public string EmbeddingsPath = string.Empty;
+            public string StackedIdEmbeddingsPath = string.Empty;
+            public bool VaeDecodeOnly = false;
+            public bool VaeTiling = true;
+            public bool FreeParamsImmediately = false;
+            public int Threads = Native.get_num_physical_cores();
+            public WeightType SdType = WeightType.SD_TYPE_COUNT;
+            public RngType RngType = RngType.CUDA_RNG;
+            public ScheduleType Schedule = ScheduleType.DEFAULT;
+            public bool KeepClipOnCpu = false;
+            public bool KeepControlNetOnCpu = false;
+            public bool KeepVaeOnCpu = false;
+        }
+
+        public class TextToImageParams
+        {
+            public string Prompt = string.Empty;
+            public string NegativePrompt = string.Empty;
+            public int ClipSkip = 0;
+            public float CfgScale = 7;
+            public int Width = 512;
+            public int Height = 512;
+            public SampleMethod SampleMethod = SampleMethod.EULER_A;
+            public int SampleSteps = 20;
+            public int64_t Seed = -1;
+            public int BatchCount = 1;
+            public Bitmap ControlCond = new Bitmap(1, 1);
+            public float ControlStrength = 0.9f;
+            public float StyleStrength = 0.75f;
+            public bool NormalizeInput = false;
+            public string InputIdImagesPath = string.Empty;
+        }
+
+        public class ImageToImageParams
+        {
+            public Bitmap InputImage;
+            public string Prompt = string.Empty;
+            public string NegativePrompt  = string.Empty;
+            public int ClipSkip = -1;
+            public float CfgScale  = 7.0f;
+            public int Width  = 512;
+            public int Height = 512;
+            public SampleMethod SampleMethod = SampleMethod.EULER_A;
+            public int SampleSteps  = 20;
+            public float Strength = 0.75f;
+            public int64_t Seed = 42;
+            public int BatchCount  = 1;
+        }
+
+        public class UpscalerParams
+        {
+            public string ESRGANPath = string.Empty;
+            public int Threads = Native.get_num_physical_cores();
+            public WeightType SdType = WeightType.SD_TYPE_COUNT;
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct SDImage
+        {
+            public uint32_t Width;
+            public uint32_t Height;
+            public uint32_t Channel;
+            public IntPtr Data;
+        }
+
+        public enum WeightType
+        {
+            SD_TYPE_F32 = 0,
+            SD_TYPE_F16 = 1,
+            SD_TYPE_Q4_0 = 2,
+            SD_TYPE_Q4_1 = 3,
+            // SD_TYPE_Q4_2 = 4, support has been removed
+            // SD_TYPE_Q4_3 (5) support has been removed
+            SD_TYPE_Q5_0 = 6,
+            SD_TYPE_Q5_1 = 7,
+            SD_TYPE_Q8_0 = 8,
+            SD_TYPE_Q8_1 = 9,
+            // k-quantizations
+            SD_TYPE_Q2_K = 10,
+            SD_TYPE_Q3_K = 11,
+            SD_TYPE_Q4_K = 12,
+            SD_TYPE_Q5_K = 13,
+            SD_TYPE_Q6_K = 14,
+            SD_TYPE_Q8_K = 15,
+            SD_TYPE_IQ2_XXS = 16,
+            SD_TYPE_IQ2_XS = 17,
+            SD_TYPE_IQ3_XXS = 18,
+            SD_TYPE_IQ1_S = 19,
+            SD_TYPE_IQ4_NL = 20,
+            SD_TYPE_IQ3_S = 21,
+            SD_TYPE_IQ2_S = 22,
+            SD_TYPE_IQ4_XS = 23,
+            SD_TYPE_I8,
+            SD_TYPE_I16,
+            SD_TYPE_I32,
+            SD_TYPE_COUNT,
+        };
+
+        public enum RngType
+        {
+            STD_DEFAULT_RNG,
+            CUDA_RNG
+        };
+
+        public enum ScheduleType
+        {
+            DEFAULT,
+            DISCRETE,
+            KARRAS,
+            N_SCHEDULES
+        };
+
+        public enum SampleMethod
+        {
+            EULER_A,
+            EULER,
+            HEUN,
+            DPM2,
+            DPMPP2S_A,
+            DPMPP2M,
+            DPMPP2Mv2,
+            LCM,
+            N_SAMPLE_METHODS
+        };
+
+        public enum SdLogLevel
+        {
+            Debug,
+            Info,
+            Warn,
+            Error
+        }
+
+    }
+}