Merge pull request #71 from AIDotNet/feature_qa

Feature qa

.gitignore

@@ -324,10 +324,6 @@ ASALocalRun/
 # MSBuild Binary and Structured Log
 *.binlog
 
-# NVidia Nsight GPU debugger configuration file
-*.nvuser
-*.dll
-*.pdb
 # MFractors (Xamarin productivity tool) working folder 
 .mfractor/
 **/bin/

Dockerfile

@@ -22,4 +22,5 @@ WORKDIR /app
 COPY --from=build /app/publish .
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 RUN echo 'Asia/Shanghai' >/etc/timezone
+RUN apt update && apt install -y libpugixml-dev libtbb-dev
 ENTRYPOINT ["dotnet", "AntSK.dll"]

Dockerfile-py

@@ -25,4 +25,5 @@ ENV PATH="/app:/opt/conda/bin:/usr/local/bin:${PATH}"
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 RUN echo 'Asia/Shanghai' >/etc/timezone
 RUN pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-ENTRYPOINT ["dotnet", "AntSK.dll"]
+RUN apt update && apt install -y libpugixml-dev libtbb-dev
+ENTRYPOINT ["dotnet", "AntSK.dll"]

README.md

@@ -89,7 +89,7 @@ version: '3.8'
 services:
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/AIDotNet/antsk:v0.2.3
+    image: registry.cn-hangzhou.aliyuncs.com/AIDotNet/antsk:v0.2.7
     ports:
       - 5000:5000
     networks:
@@ -102,6 +102,7 @@ services:
     volumes:
       - ./appsettings.json:/app/appsettings.json # 本地配置文件 需要放在同级目录
       - D://model:/app/model
+      - D://model:/root/.cache/modelscope/hub/AI-ModelScope #使用Llamafactory时需要挂载 否则初始化的环境重启后会丢失
 networks:
   antsk:
 ```

docker-compose.simple.yml

@@ -3,9 +3,9 @@ version: '3.8'
 services:
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.6
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.7
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.6
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.7
     ports:
       - 5000:5000
     networks:

docker-compose.yml

@@ -18,9 +18,9 @@ services:
       - ./pg/data:/var/lib/postgresql/data
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.6
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.7
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.6
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.7
     ports:
       - 5000:5000
     networks:

src/AntSK.Domain/AntSK.Domain.csproj

@@ -24,28 +24,29 @@
     <PackageReference Include="SqlSugarCore" Version="5.1.4.151" />
     <PackageReference Include="System.Data.SQLite.Core" Version="1.0.118" />
     <PackageReference Include="RestSharp" Version="110.2.0" />
-    <PackageReference Include="NPOI" Version="2.5.5" />
+    <PackageReference Include="NPOI" Version="2.7.0" />
     
-    <PackageReference Include="Microsoft.SemanticKernel" Version="1.6.3" />
-    <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.6.3" />
-    <PackageReference Include="Microsoft.SemanticKernel.Plugins.Core" Version="1.6.3-alpha" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.7.1" />
+    <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.7.1" />
+    <PackageReference Include="Microsoft.SemanticKernel.Plugins.Core" Version="1.7.1-alpha" />
     <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240321.1" />
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Postgres" Version="0.35.240321.1" />
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240321.1" />
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Redis" Version="0.35.240321.1" />
     <PackageReference Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="0.35.240321.1" />
 
-    <PackageReference Include="LLamaSharp" Version="0.11.1" />
-    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.11.1" />
-    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.11.1" />
-    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.11.1" />
-    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.11.1" />
+    <PackageReference Include="LLamaSharp" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.11.2" />
     
     
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\AntSK.LLamaFactory\AntSK.LLamaFactory.csproj" />
     <ProjectReference Include="..\AntSk.LLM\AntSK.LLM.csproj" />
+    <ProjectReference Include="..\AntSK.OCR\AntSK.OCR.csproj" />
     <ProjectReference Include="..\MiddleWare\AntSK.BackgroundTask\AntSK.BackgroundTask.csproj" />
   </ItemGroup>
 

src/AntSK.Domain/AntSK.Domain.xml

@@ -193,6 +193,12 @@
             避免模型重复加载，本地缓存
             </summary>
         </member>
+        <member name="P:AntSK.Domain.Domain.Other.QAHandler.StepName">
+            <inheritdoc />
+        </member>
+        <member name="M:AntSK.Domain.Domain.Other.QAHandler.InvokeAsync(Microsoft.KernelMemory.Pipeline.DataPipeline,System.Threading.CancellationToken)">
+            <inheritdoc />
+        </member>
         <member name="M:AntSK.Domain.Domain.Service.ChatService.SendChatByAppAsync(AntSK.Domain.Repositories.Apps,System.String,Microsoft.SemanticKernel.ChatCompletion.ChatHistory)">
             <summary>
             发送消息

src/AntSK.Domain/Common/DependencyInjection/InitExtensions.cs

@@ -50,6 +50,8 @@ namespace AntSK.Domain.Common.DependencyInjection
                         _repository.GetDB().CodeFirst.InitTables(type);
                     }
                 }
+                //安装向量插件
+                _repository.GetDB().Ado.ExecuteCommandAsync($"CREATE EXTENSION IF NOT EXISTS vector;");
             }
             return app;
         }

src/AntSK.Domain/Domain/Interface/IKernelService.cs

@@ -6,6 +6,8 @@ namespace AntSK.Domain.Domain.Interface
     public interface IKernelService
     {
         Kernel GetKernelByApp(Apps app);
+
+        Kernel GetKernelByAIModelID(string modelid);
         void ImportFunctionsByApp(Apps app, Kernel _kernel);
         Task<string> HistorySummarize(Kernel _kernel, string questions, string history);
     }

src/AntSK.Domain/Domain/Model/ImportKMSTaskReq.cs

@@ -17,11 +17,14 @@ namespace AntSK.Domain.Domain.Model
         public string FilePath { get; set; } = "";
 
         public string FileName { get; set; } = "";
+
+        public bool IsQA { get; set; } = false;
     }
 
 
     public class ImportKMSTaskReq : ImportKMSTaskDTO
     {
+        public bool IsQA { get; set; }=false;
         public KmsDetails KmsDetail { get; set; } = new KmsDetails();
     }
 
@@ -32,4 +35,10 @@ namespace AntSK.Domain.Domain.Model
         Text = 3,
         Excel=4
     }
+
+    public class QAModel
+    {
+        public string ChatModelId { get; set; }
+        public string Context { get; set; }
+    }
 }

src/AntSK.Domain/Domain/Other/QAHandler.cs

@@ -0,0 +1,154 @@
+using AntSK.Domain.Domain.Model;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Configuration;
+using Microsoft.KernelMemory.DataFormats.Text;
+using Microsoft.KernelMemory.Diagnostics;
+using Microsoft.KernelMemory.Extensions;
+using Microsoft.KernelMemory.Pipeline;
+using Newtonsoft.Json;
+using RestSharp;
+using System.Security.Policy;
+using System.Text;
+
+namespace AntSK.Domain.Domain.Other
+{
+    public class QAHandler : IPipelineStepHandler
+    {
+        private readonly TextPartitioningOptions _options;
+        private readonly IPipelineOrchestrator _orchestrator;
+        private readonly ILogger<QAHandler> _log;
+        private readonly TextChunker.TokenCounter _tokenCounter;
+        public QAHandler(
+            string stepName,
+            IPipelineOrchestrator orchestrator,
+            TextPartitioningOptions? options = null,
+            ILogger<QAHandler>? log = null
+            )
+        {
+            this.StepName = stepName;
+            this._orchestrator = orchestrator;
+            this._options = options ?? new TextPartitioningOptions();
+            this._options.Validate();
+
+            this._log = log ?? DefaultLogger<QAHandler>.Instance;
+            this._tokenCounter = DefaultGPTTokenizer.StaticCountTokens;
+        }
+
+        /// <inheritdoc />
+        public string StepName { get; }
+
+        /// <inheritdoc />
+        public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(
+            DataPipeline pipeline, CancellationToken cancellationToken = default)
+        {
+            this._log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId);
+
+            if (pipeline.Files.Count == 0)
+            {
+                this._log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId);
+                return (true, pipeline);
+            }
+
+            foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)
+            {
+                // Track new files being generated (cannot edit originalFile.GeneratedFiles while looping it)
+                Dictionary<string, DataPipeline.GeneratedFileDetails> newFiles = new();
+
+                foreach (KeyValuePair<string, DataPipeline.GeneratedFileDetails> generatedFile in uploadedFile.GeneratedFiles)
+                {
+                    var file = generatedFile.Value;
+                    if (file.AlreadyProcessedBy(this))
+                    {
+                        this._log.LogTrace("File {0} already processed by this handler", file.Name);
+                        continue;
+                    }
+
+                    // Partition only the original text
+                    if (file.ArtifactType != DataPipeline.ArtifactTypes.ExtractedText)
+                    {
+                        this._log.LogTrace("Skipping file {0} (not original text)", file.Name);
+                        continue;
+                    }
+
+                    // Use a different partitioning strategy depending on the file type
+                    List<string> partitions;
+                    List<string> sentences;
+                    BinaryData partitionContent = await this._orchestrator.ReadFileAsync(pipeline, file.Name, cancellationToken).ConfigureAwait(false);
+
+                    // Skip empty partitions. Also: partitionContent.ToString() throws an exception if there are no bytes.
+                    if (partitionContent.ToArray().Length == 0) { continue; }
+
+                    switch (file.MimeType)
+                    {
+                        case MimeTypes.PlainText:
+                        case MimeTypes.MarkDown:
+                            {   
+                                this._log.LogDebug("Partitioning text file {0}", file.Name);
+                                string content = partitionContent.ToString();
+
+                                using (HttpClient httpclient = new HttpClient())
+                                {
+                                    httpclient.Timeout = TimeSpan.FromMinutes(10);
+                                    StringContent scontent = new StringContent(JsonConvert.SerializeObject(new QAModel() { ChatModelId = StepName, Context = content }), Encoding.UTF8, "application/json");
+                                    HttpResponseMessage response = await httpclient.PostAsync("http://localhost:5000/api/KMS/QA", scontent);
+                                    List<string> qaList = JsonConvert.DeserializeObject<List<string>>( await response.Content.ReadAsStringAsync());
+                                    sentences = qaList;
+                                    partitions = qaList;
+                                }
+                                break;
+                            }
+                        default:
+                            this._log.LogWarning("File {0} cannot be partitioned, type '{1}' not supported", file.Name, file.MimeType);
+                            // Don't partition other files
+                            continue;
+                    }
+
+                    if (partitions.Count == 0) { continue; }
+
+                    this._log.LogDebug("Saving {0} file partitions", partitions.Count);
+                    for (int partitionNumber = 0; partitionNumber < partitions.Count; partitionNumber++)
+                    {
+                        // TODO: turn partitions in objects with more details, e.g. page number
+                        string text = partitions[partitionNumber];
+                        int sectionNumber = 0; // TODO: use this to store the page number (if any)
+                        BinaryData textData = new(text);
+
+                        int tokenCount = this._tokenCounter(text);
+                        this._log.LogDebug("Partition size: {0} tokens", tokenCount);
+
+                        var destFile = uploadedFile.GetPartitionFileName(partitionNumber);
+                        await this._orchestrator.WriteFileAsync(pipeline, destFile, textData, cancellationToken).ConfigureAwait(false);
+
+                        var destFileDetails = new DataPipeline.GeneratedFileDetails
+                        {
+                            Id = Guid.NewGuid().ToString("N"),
+                            ParentId = uploadedFile.Id,
+                            Name = destFile,
+                            Size = text.Length,
+                            MimeType = MimeTypes.PlainText,
+                            ArtifactType = DataPipeline.ArtifactTypes.TextPartition,
+                            PartitionNumber = partitionNumber,
+                            SectionNumber = sectionNumber,
+                            Tags = pipeline.Tags,
+                            ContentSHA256 = textData.CalculateSHA256(),
+                        };
+                        newFiles.Add(destFile, destFileDetails);
+                        destFileDetails.MarkProcessedBy(this);
+                    }
+
+                    file.MarkProcessedBy(this);
+                }
+
+                // Add new files to pipeline status
+                foreach (var file in newFiles)
+                {
+                    uploadedFile.GeneratedFiles.Add(file.Key, file.Value);
+                }
+            }
+
+            return (true, pipeline);
+        }
+    }
+}

src/AntSK.Domain/Domain/Service/ChatService.cs

@@ -1,23 +1,21 @@
 using AntSK.Domain.Common.DependencyInjection;
 using AntSK.Domain.Domain.Interface;
-using AntSK.Domain.Repositories;
-using Microsoft.SemanticKernel.Connectors.OpenAI;
-using Microsoft.SemanticKernel;
-using System.Text;
-using AntSK.Domain.Utils;
-using AntSK.Domain.Domain.Model.Dto;
-using AntSK.Domain.Domain.Model.Constant;
-using DocumentFormat.OpenXml.Drawing;
-using System.Reflection.Metadata;
-using Microsoft.KernelMemory;
-using System.Collections.Generic;
-using Markdig;
-using ChatHistory = Microsoft.SemanticKernel.ChatCompletion.ChatHistory;
-using Microsoft.SemanticKernel.Plugins.Core;
-using Azure.Core;
 using AntSK.Domain.Domain.Model;
+using AntSK.Domain.Domain.Model.Constant;
+using AntSK.Domain.Domain.Model.Dto;
+using AntSK.Domain.Repositories;
+using AntSK.Domain.Utils;
 using AntSK.LLM.StableDiffusion;
+using Markdig;
+using Microsoft.KernelMemory;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
+using System.Diagnostics;
 using System.Drawing;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using ChatHistory = Microsoft.SemanticKernel.ChatCompletion.ChatHistory;
 
 namespace AntSK.Domain.Domain.Service
 {
@@ -44,7 +42,7 @@ namespace AntSK.Domain.Domain.Service
                 //如果模板为空，给默认提示词
                 app.Prompt = app.Prompt.ConvertToString() + "{{$input}}";
             }
-            KernelArguments args =new KernelArguments();
+            KernelArguments args = new KernelArguments();
             if (history.Count > 10)
             {
                 app.Prompt = @"${{ConversationSummaryPlugin.SummarizeConversation $history}}" + app.Prompt;
@@ -53,14 +51,14 @@ namespace AntSK.Domain.Domain.Service
                 { "input", questions }
                 };
             }
-            else 
+            else
             {
-                args=new()
+                args = new()
                 {
                 { "input", $"{string.Join("\n", history.Select(x => x.Role + ": " + x.Content))}{Environment.NewLine} user:{questions}" }
                 };
             }
-    
+
             var _kernel = _kernelService.GetKernelByApp(app);
             var temperature = app.Temperature / 100;//存的是0~100需要缩小
             OpenAIPromptExecutionSettings settings = new() { Temperature = temperature };
@@ -70,7 +68,7 @@ namespace AntSK.Domain.Domain.Service
                 settings.ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions;
             }
             var func = _kernel.CreateFunctionFromPrompt(app.Prompt, settings);
-            var chatResult = _kernel.InvokeStreamingAsync(function: func, 
+            var chatResult = _kernel.InvokeStreamingAsync(function: func,
                 arguments: args);
             await foreach (var content in chatResult)
             {
@@ -105,14 +103,14 @@ namespace AntSK.Domain.Domain.Service
             var dataMsg = new StringBuilder();
             if (relevantSourceList.Any())
             {
-                bool isSearch=false;
+                bool isSearch = false;
                 foreach (var item in relevantSourceList)
                 {
                     //匹配相似度
-                    if (item.Relevance >= app.Relevance/100)
+                    if (item.Relevance >= app.Relevance / 100)
                     {
                         dataMsg.AppendLine(item.ToString());
-                        isSearch=true;
+                        isSearch = true;
                     }
                 }
 
@@ -138,10 +136,10 @@ namespace AntSK.Domain.Domain.Service
                         yield return content;
                     }
                 }
-                else 
+                else
                 {
                     yield return new StreamingTextContent(KmsConstantcs.KmsSearchNull);
-                }             
+                }
             }
             else
             {
@@ -163,6 +161,59 @@ namespace AntSK.Domain.Domain.Service
             var chatResult = await _kernel.InvokeAsync(function: func, arguments: args);
             if (chatResult.IsNotNull())
             {
+                //Can Load stable-diffusion library in diffenert environment
+
+                //SDHelper.LoadLibrary()
+                string versionString = string.Empty;
+                string extensionString = string.Empty;
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                {
+                    extensionString = ".dll";
+                }
+                else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                {
+                    extensionString = ".so";
+                }
+                else
+                {
+                    throw new InvalidOperationException("OS Platform no support");
+                }
+
+                ProcessStartInfo startInfo = new ProcessStartInfo("nvcc", "--version");
+                startInfo.RedirectStandardOutput = true;
+                startInfo.UseShellExecute = false;
+                startInfo.CreateNoWindow = true;
+                using (Process process = Process.Start(startInfo))
+                {
+                    if (process != null)
+                    {
+                        string result = process.StandardOutput.ReadToEnd();
+                        Regex regex = new Regex(@"release (\d+).[\d]");
+                        Match match = regex.Match(result);
+                        if (match.Success)
+                        {
+                            switch (match.Groups[1].Value.ToString())
+                            {
+                                case "11":
+                                    versionString = "Cuda11";
+                                    break;
+                                case "12":
+                                    versionString = "Cuda12";
+                                    break;
+                                default:
+                                    versionString = "CPU";
+                                    break;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        throw new Exception("nvcc get an error");
+                    }
+                }
+
+                string libraryPath = System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "StableDiffusion", "Backend", versionString, "stable-diffusion" + extensionString);
+                NativeLibrary.TryLoad(libraryPath, out _);
                 string prompt = chatResult.GetValue<string>();
                 if (!SDHelper.IsInitialized)
                 {
@@ -172,7 +223,8 @@ namespace AntSK.Domain.Domain.Service
                         RngType = Structs.RngType.CUDA_RNG,
                         //VaePath = vaePath,
                         //KeepVaeOnCpu = keepVaeOnCpu,
-                        //VaeTiling = vaeTiling,
+                        //set false can get a better image, otherwise can use lower vram
+                        VaeTiling = false,
                         //LoraModelDir = loraModelDir,
                     };
                     bool result = SDHelper.Initialize(modelParams);
@@ -181,10 +233,11 @@ namespace AntSK.Domain.Domain.Service
                 Structs.TextToImageParams textToImageParams = new Structs.TextToImageParams
                 {
                     Prompt = prompt,
-                    NegativePrompt = "2d, 3d, cartoon, paintings",
+                    NegativePrompt = "bad quality, wrong image, worst quality",
                     SampleMethod = (Structs.SampleMethod)Enum.Parse(typeof(Structs.SampleMethod), "EULER_A"),
-                    Width = 256,
-                    Height = 256,
+                    //the base image size in SD1.5 is 512x512
+                    Width = 512,
+                    Height = 512,
                     NormalizeInput = true,
                     ClipSkip = -1,
                     CfgScale = 7,

src/AntSK.Domain/Domain/Service/ImportKMSService.cs

@@ -24,18 +24,40 @@ namespace AntSK.Domain.Domain.Service
             try
             {
                 var km = _kmss_Repositories.GetFirst(p => p.Id == req.KmsId);
-
                 var _memory = _kMService.GetMemoryByKMS(km.Id);
                 string fileid = req.KmsDetail.Id;
+                List<string> step = new List<string>();
+                if (req.IsQA)
+                {
+                    _memory.Orchestrator.AddHandler<TextExtractionHandler>("extract_text");
+                    _memory.Orchestrator.AddHandler<QAHandler>(km.ChatModelID);
+                    _memory.Orchestrator.AddHandler<GenerateEmbeddingsHandler>("generate_embeddings");
+                    _memory.Orchestrator.AddHandler<SaveRecordsHandler>("save_memory_records");
+                    step.Add("extract_text");
+                    step.Add(km.ChatModelID);
+                    step.Add("generate_embeddings");
+                    step.Add("save_memory_records");
+                }
+
                 switch (req.ImportType)
                 {
                     case ImportType.File:
-                        //导入文件
                         {
-                            var importResult = _memory.ImportDocumentAsync(new Document(fileid)
-                          .AddFile(req.FilePath)
-                          .AddTag(KmsConstantcs.KmsIdTag, req.KmsId)
-                          , index: KmsConstantcs.KmsIndex).Result;
+                            //导入文件
+                            if (req.IsQA)
+                            {
+                                var importResult = _memory.ImportDocumentAsync(new Document(fileid)
+                                .AddFile(req.FilePath)
+                                .AddTag(KmsConstantcs.KmsIdTag, req.KmsId)
+                                ,index: KmsConstantcs.KmsIndex ,steps: step.ToArray()).Result;
+                            }
+                            else 
+                            {
+                                var importResult = _memory.ImportDocumentAsync(new Document(fileid)
+                                 .AddFile(req.FilePath)
+                                 .AddTag(KmsConstantcs.KmsIdTag, req.KmsId)
+                             , index: KmsConstantcs.KmsIndex).Result;
+                            }
                             //查询文档数量
                             var docTextList = _kMService.GetDocumentByFileID(km.Id, fileid).Result;
                             string fileGuidName = Path.GetFileName(req.FilePath);
@@ -48,8 +70,16 @@ namespace AntSK.Domain.Domain.Service
                     case ImportType.Url:
                         {
                             //导入url                  
-                            var importResult = _memory.ImportWebPageAsync(req.Url, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
-                                 , index: KmsConstantcs.KmsIndex).Result;
+                            if (req.IsQA)
+                            {
+                                var importResult = _memory.ImportWebPageAsync(req.Url, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
+                                , index: KmsConstantcs.KmsIndex, steps: step.ToArray()).Result;
+                            }
+                            else 
+                            {
+                                var importResult = _memory.ImportWebPageAsync(req.Url, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
+                                , index: KmsConstantcs.KmsIndex).Result;
+                            }  
                             //查询文档数量
                             var docTextList = _kMService.GetDocumentByFileID(km.Id, fileid).Result;
                             req.KmsDetail.Url = req.Url;
@@ -59,8 +89,16 @@ namespace AntSK.Domain.Domain.Service
                     case ImportType.Text:
                         //导入文本
                         {
-                            var importResult = _memory.ImportTextAsync(req.Text, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
-                                , index: KmsConstantcs.KmsIndex).Result;
+                            if (req.IsQA)
+                            {
+                                var importResult = _memory.ImportTextAsync(req.Text, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
+                                , index: KmsConstantcs.KmsIndex, steps: step.ToArray()).Result;
+                            }
+                            else 
+                            {
+                                var importResult = _memory.ImportTextAsync(req.Text, fileid, new TagCollection() { { KmsConstantcs.KmsIdTag, req.KmsId } }
+                                   , index: KmsConstantcs.KmsIndex).Result;
+                            }                  
                             //查询文档数量
                             var docTextList = _kMService.GetDocumentByFileID(km.Id, fileid).Result;
                             req.KmsDetail.Url = req.Url;
@@ -71,8 +109,7 @@ namespace AntSK.Domain.Domain.Service
                     case ImportType.Excel:
                         using (var fs = File.OpenRead(req.FilePath))
                         {
-                            var excelList= ExeclHelper.ExcelToList<KMSExcelModel>(fs);
-                            
+                            var excelList= ExeclHelper.ExcelToList<KMSExcelModel>(fs);           
                             _memory.Orchestrator.AddHandler<TextExtractionHandler>("extract_text");
                             _memory.Orchestrator.AddHandler<KMExcelHandler>("antsk_excel_split");
                             _memory.Orchestrator.AddHandler<GenerateEmbeddingsHandler>("generate_embeddings");

src/AntSK.Domain/Domain/Service/KMService.cs

@@ -8,6 +8,7 @@ using AntSK.Domain.Domain.Other;
 using AntSK.Domain.Options;
 using AntSK.Domain.Repositories;
 using AntSK.Domain.Utils;
+using AntSK.OCR;
 using DocumentFormat.OpenXml.Drawing.Diagrams;
 using LLama;
 using LLamaSharp.KernelMemory;
@@ -16,6 +17,7 @@ using Microsoft.AspNetCore.Components;
 using Microsoft.Extensions.Configuration;
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.Configuration;
+using Microsoft.KernelMemory.DataFormats;
 using Microsoft.KernelMemory.FileSystem.DevTools;
 using Microsoft.KernelMemory.MemoryStorage;
 using Microsoft.KernelMemory.MemoryStorage.DevTools;
@@ -104,13 +106,15 @@ namespace AntSK.Domain.Domain.Service
                         MaxTokensPerParagraph = kms.MaxTokensPerParagraph,
                         OverlappingTokens = kms.OverlappingTokens
                     });
+                //加载OCR
+                WithOcr(memoryBuild, kms);
                 //加载会话模型
                 WithTextGenerationByAIType(memoryBuild, chatModel, chatHttpClient);
                 //加载向量模型
                 WithTextEmbeddingGenerationByAIType(memoryBuild, embedModel, embeddingHttpClient);
                 //加载向量库
                 WithMemoryDbByVectorDB(memoryBuild);
-
+              
                 _memory = memoryBuild.Build<MemoryServerless>();
                 return _memory;
             }
@@ -119,6 +123,14 @@ namespace AntSK.Domain.Domain.Service
             //}
         }
 
+        private static void WithOcr(IKernelMemoryBuilder memoryBuild, Kmss kms)
+        {
+            if (kms.IsOCR == 1)
+            {
+                memoryBuild.WithCustomImageOcr(new AntSKOcrEngine());
+            }
+        }
+
         private void WithTextEmbeddingGenerationByAIType(IKernelMemoryBuilder memory, AIModels embedModel,
             HttpClient embeddingHttpClient)
         {
@@ -193,7 +205,7 @@ namespace AntSK.Domain.Domain.Service
 
                     memory.WithOpenAITextGeneration(new OpenAIConfig()
                     {
-                        APIKey = "123",
+                        APIKey = "NotNull",
                         TextModel = chatModel.ModelName
                     }, null, chatHttpClient);
                     break;
@@ -262,7 +274,7 @@ namespace AntSK.Domain.Domain.Service
             {
                 foreach (var memoryDb in memoryDbs)
                 {
-                    var items = await memoryDb.GetListAsync(memoryIndex.Name, new List<MemoryFilter>() { new MemoryFilter().ByDocument(fileId) }, 100, true).ToListAsync();
+                    var items = await memoryDb.GetListAsync(memoryIndex.Name, new List<MemoryFilter>() { new MemoryFilter().ByDocument(fileId) }, 1000, true).ToListAsync();
                     docTextList.AddRange(items.Select(item => new KMFile()
                     {
                         DocumentId = item.GetDocumentId(),
@@ -319,7 +331,10 @@ namespace AntSK.Domain.Domain.Service
                 "application/pdf",
                 "application/json",
                 "text/x-markdown",
-                "text/markdown"
+                "text/markdown",
+                "image/jpeg",
+                "image/png",
+                "image/tiff"
             };
 
             string[] exceptExts = [".md", ".pdf"];

src/AntSK.Domain/Domain/Service/KernelService.cs

@@ -18,6 +18,8 @@ using AntSK.Domain.Domain.Model.Enum;
 using AntSK.LLM.LLamaFactory;
 using System.Reflection;
 using DocumentFormat.OpenXml.Drawing;
+using Microsoft.KernelMemory;
+using OpenCvSharp.ML;
 
 namespace AntSK.Domain.Domain.Service
 {
@@ -57,7 +59,7 @@ namespace AntSK.Domain.Domain.Service
                 var chatHttpClient = OpenAIHttpClientHandlerUtil.GetHttpClient(chatModel.EndPoint);
 
                 var builder = Kernel.CreateBuilder();
-                WithTextGenerationByAIType(builder, app, chatModel, chatHttpClient);
+                WithTextGenerationByAIType(builder, chatModel, chatHttpClient);
 
                 _kernel = builder.Build();
                 RegisterPluginsWithKernel(_kernel);
@@ -69,7 +71,18 @@ namespace AntSK.Domain.Domain.Service
             //}
         }
 
-        private void WithTextGenerationByAIType(IKernelBuilder builder, Apps app, AIModels chatModel, HttpClient chatHttpClient)
+        public Kernel GetKernelByAIModelID(string modelid)
+        {
+            var chatModel = _aIModels_Repositories.GetById(modelid);
+            var chatHttpClient = OpenAIHttpClientHandlerUtil.GetHttpClient(chatModel.EndPoint);
+            var builder = Kernel.CreateBuilder();
+            WithTextGenerationByAIType(builder, chatModel, chatHttpClient);
+            _kernel = builder.Build();
+            RegisterPluginsWithKernel(_kernel);
+            return _kernel;
+        }
+
+        private void WithTextGenerationByAIType(IKernelBuilder builder,AIModels chatModel, HttpClient chatHttpClient)
         {
             switch (chatModel.AIType)
             {
@@ -96,7 +109,7 @@ namespace AntSK.Domain.Domain.Service
 
                 case Model.Enum.AIType.SparkDesk:
                     var options = new SparkDeskOptions { AppId = chatModel.EndPoint, ApiSecret = chatModel.ModelKey, ApiKey = chatModel.ModelName, ModelVersion = Sdcb.SparkDesk.ModelVersion.V3_5 };
-                    builder.Services.AddKeyedSingleton<ITextGenerationService>("spark-desk", new SparkDeskTextCompletion(options, app.Id));
+                    builder.Services.AddKeyedSingleton<ITextGenerationService>("spark-desk", new SparkDeskTextCompletion(options, chatModel.Id));
                     break;
 
                 case Model.Enum.AIType.DashScope:

src/AntSK.Domain/Repositories/AI/App/Apps.cs

@@ -83,7 +83,7 @@ namespace AntSK.Domain.Repositories
         /// 相似度
         /// </summary>
         [SugarColumn(DefaultValue = "70")]
-        public double Relevance { get; set; } = 70;
+        public double Relevance { get; set; } = 70f;
 
         /// <summary>
         /// 提问最大token数

src/AntSK.Domain/Repositories/AI/Kms/Kmss.cs

@@ -55,6 +55,7 @@ namespace AntSK.Domain.Repositories
         [SugarColumn(DefaultValue = "49")]
         public int OverlappingTokens { get; set; } = 49;
 
-
+        [SugarColumn(DefaultValue = "0")]
+        public int IsOCR { get; set; } = 0;
     }
 }

src/AntSK.OCR/AntSK.OCR.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240321.1" />
+    <PackageReference Include="OpenCvSharp4.runtime.win" Version="4.9.0.20240103" />
+    <PackageReference Include="Sdcb.OpenCvSharp4.mini.runtime.debian.12-x64" Version="4.8.0.20231125" />
+    <PackageReference Include="Sdcb.OpenVINO" Version="0.6.4" />
+    <PackageReference Include="Sdcb.OpenVINO.PaddleOCR.Models.Online" Version="0.6.2" />
+    <PackageReference Include="Sdcb.OpenVINO.PaddleOCR" Version="0.6.3" />
+    <PackageReference Include="Sdcb.OpenVINO.runtime.ubuntu.22.04-x64" Version="2024.0.0" />
+    <PackageReference Include="Sdcb.OpenVINO.runtime.win-x64" Version="2024.0.0" />
+    
+  </ItemGroup>
+</Project>

src/AntSK.OCR/AntSKOcrEngine.cs

@@ -0,0 +1,56 @@
+using Microsoft.KernelMemory.DataFormats;
+using Sdcb.OpenVINO.PaddleOCR;
+using Sdcb.OpenVINO.PaddleOCR.Models.Online;
+using Sdcb.OpenVINO.PaddleOCR.Models;
+using OpenCvSharp;
+
+namespace AntSK.OCR
+{
+    /// <summary>
+    /// OCR
+    /// </summary>
+    public class AntSKOcrEngine : IOcrEngine
+    {
+        FullOcrModel model;
+        public Task<string> ExtractTextFromImageAsync(Stream imageContent, CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                if (model == null)
+                {
+                    model = OnlineFullModels.ChineseV4.DownloadAsync().Result;
+                }
+                using (PaddleOcrAll all = new(model)
+                {
+                    AllowRotateDetection = true,
+                    Enable180Classification = true,
+                })
+                {
+                    Mat src = Cv2.ImDecode(StreamToByte(imageContent), ImreadModes.Color);
+                    PaddleOcrResult result = all.Run(src);
+                    return Task.FromResult(result.Text);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.Message);
+                return Task.FromResult("");
+            }
+        }
+
+        private byte[] StreamToByte(Stream stream)
+        {
+            using (var memoryStream = new MemoryStream())
+            {
+                byte[] buffer = new byte[1024]; //自定义大小，例如 1024
+                int bytesRead;
+                while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
+                {
+                    memoryStream.Write(buffer, 0, bytesRead);
+                }
+                byte[] bytes = memoryStream.ToArray();
+                return bytes;
+            }
+        }
+    }
+}

src/AntSK.PyNet/AntSK.PyNet.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>

src/AntSK.PyNet/Class1.cs

@@ -1,7 +0,0 @@
-namespace AntSK.PyNet
-{
-    public class Class1
-    {
-
-    }
-}

src/AntSK.sln

@@ -12,6 +12,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docker", "Docker", "{9F2E19
 		..\docker-compose.simple.yml = ..\docker-compose.simple.yml
 		..\docker-compose.yml = ..\docker-compose.yml
 		..\Dockerfile = ..\Dockerfile
+		..\Dockerfile-py = ..\Dockerfile-py
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MiddleWare", "MiddleWare", "{40DDB1DC-571B-4A95-9F34-47F52981C511}"
@@ -24,6 +25,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AntSK.Test", "AntSK.Test\An
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AntSK.LLamaFactory", "AntSK.LLamaFactory\AntSK.LLamaFactory.csproj", "{664DFA1F-68B7-49C7-B889-FA14D1756D3D}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AntSK.OCR", "AntSK.OCR\AntSK.OCR.csproj", "{6195F7AA-18C2-4372-85CA-11FC4B522686}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Settings", "Settings", "{EB227FAB-6060-4271-9A0A-6C99C7965126}"
+	ProjectSection(SolutionItems) = preProject
+		NuGet.config = NuGet.config
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -54,6 +62,10 @@ Global
 		{664DFA1F-68B7-49C7-B889-FA14D1756D3D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{664DFA1F-68B7-49C7-B889-FA14D1756D3D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{664DFA1F-68B7-49C7-B889-FA14D1756D3D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6195F7AA-18C2-4372-85CA-11FC4B522686}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6195F7AA-18C2-4372-85CA-11FC4B522686}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6195F7AA-18C2-4372-85CA-11FC4B522686}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6195F7AA-18C2-4372-85CA-11FC4B522686}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/AntSK/AntSK.csproj

@@ -15,13 +15,11 @@
     <None Remove="llamafactory\**" />
   </ItemGroup>
   <ItemGroup>
-
-    
     <PackageReference Include="Microsoft.AspNetCore.Authorization" Version="8.0.2" />
     <PackageReference Include="Microsoft.AspNetCore.Components.Authorization" Version="8.0.2" />
     <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Authentication" Version="8.0.2" />
     <PackageReference Include="System.Net.Http.Json" Version="8.0.0" />
-	<PackageReference Include="Downloader" Version="3.0.6" />
+    <PackageReference Include="Downloader" Version="3.0.6" />
   </ItemGroup>
 
   <ItemGroup>
@@ -29,11 +27,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="plugins\KMSPlugin\Ask1\skprompt.txt">
+    <None Update="plugins\**">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="plugins\KMSPlugin\Ask\skprompt.txt">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
+    </None>  
   </ItemGroup>
 </Project>

src/AntSK/Controllers/InitController.cs

@@ -1,35 +0,0 @@
-using AntSK.Domain.Repositories;
-using Microsoft.AspNetCore.Mvc;
-
-namespace AntSK.Controllers
-{
-    [Route("api/[controller]/[action]")]
-    [ApiController]
-    public class InitController : ControllerBase
-    {
-        private readonly IApps_Repositories _repository;
-
-        public InitController(IApps_Repositories repository)
-        {
-            _repository = repository;
-        }
-        /// <summary>
-        /// 初始化DB 和表
-        /// </summary>
-        /// <returns></returns>
-        [HttpPost]
-        public IActionResult InitTable()
-        {
-            _repository.GetDB().DbMaintenance.CreateDatabase();
-            _repository.GetDB().CodeFirst.InitTables(typeof(Apps));
-            _repository.GetDB().CodeFirst.InitTables(typeof(Kmss));
-            _repository.GetDB().CodeFirst.InitTables(typeof(KmsDetails));
-            _repository.GetDB().CodeFirst.InitTables(typeof(Users));
-            _repository.GetDB().CodeFirst.InitTables(typeof(Apis));
-            _repository.GetDB().CodeFirst.InitTables(typeof(AIModels));
-            //创建vector插件如果数据库没有则需要提供支持向量的数据库
-            _repository.GetDB().Ado.ExecuteCommandAsync($"CREATE EXTENSION IF NOT EXISTS vector;");
-            return Ok();
-        }
-    }
-}

src/AntSK/Controllers/KMSController.cs

@@ -1,10 +1,17 @@
-using AntSK.BackgroundTask;
+using AntDesign;
+using AntSK.BackgroundTask;
 using AntSK.Domain.Common.Map;
 using AntSK.Domain.Domain.Interface;
 using AntSK.Domain.Domain.Model;
 using AntSK.Domain.Domain.Model.Enum;
+using AntSK.Domain.Domain.Service;
 using AntSK.Domain.Repositories;
+using AntSK.Domain.Utils;
+using Microsoft.AspNetCore.Components.Forms;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Text;
+using System.Text.RegularExpressions;
 
 namespace AntSK.Controllers
 {
@@ -17,14 +24,17 @@ namespace AntSK.Controllers
     {
         private readonly IKmsDetails_Repositories _kmsDetailsRepositories;
         private readonly BackgroundTaskBroker<ImportKMSTaskReq> _taskBroker;
+        private readonly IKernelService _kernelService;
 
         public KMSController(
             IKmsDetails_Repositories kmsDetailsRepositories,
-            BackgroundTaskBroker<ImportKMSTaskReq> taskBroker
+            BackgroundTaskBroker<ImportKMSTaskReq> taskBroker,
+            IKernelService kernelService
         )
         {
             _kmsDetailsRepositories = kmsDetailsRepositories;
             _taskBroker = taskBroker;
+            _kernelService = kernelService;
         }
 
         /// <summary>
@@ -47,8 +57,36 @@ namespace AntSK.Controllers
 
             await _kmsDetailsRepositories.InsertAsync(detail);
             req.KmsDetail = detail;
+            req.IsQA=model.IsQA;
             _taskBroker.QueueWorkItem(req);
             return Ok();
         }
+
+        [HttpPost]
+        public async Task<IActionResult> QA(QAModel model)
+        {
+            var kernel = _kernelService.GetKernelByAIModelID(model.ChatModelId);
+            var lines = TextChunker.SplitPlainTextLines(model.Context, 299);
+            var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 4000);
+            KernelFunction jsonFun = kernel.Plugins.GetFunction("KMSPlugin", "QA");
+
+            List<string> qaList = new List<string>();
+            foreach (var para in paragraphs)
+            {
+                var qaresult = await kernel.InvokeAsync(function: jsonFun, new KernelArguments() { ["input"] = para });
+                var qaListStr = qaresult.GetValue<string>().ConvertToString();
+
+                string pattern = @"Q\d+:.*?A\d+:.*?(?=(Q\d+:|$))";
+                RegexOptions options = RegexOptions.Singleline;
+
+                foreach (Match match in Regex.Matches(qaListStr, pattern, options))
+                {
+                    qaList.Add(match.Value.Trim()); // Trim用于删除可能的首尾空格
+                }
+
+            }
+
+            return Ok(qaList);
+        }
     }
 }

src/AntSK/Pages/ChatPage/Chat.razor

@@ -9,7 +9,7 @@
 @inherits AuthComponentBase
 
 <GridRow Gutter="(16, 16)">
-    <GridCol Span="12">
+    <GridCol Span="14">
         <Card Style="height:75vh;overflow: auto;">
             <TitleTemplate>
                 <Icon Type="setting" /> 选择应用
@@ -30,7 +30,7 @@
             </Body>
         </Card>
     </GridCol>
-    <GridCol Span="12">
+    <GridCol Span="10">
         <Card Style="height: 75vh;overflow: auto;">
             <TitleTemplate>
                 <Icon Type="search" /> 调试结果

src/AntSK/Pages/ChatPage/Components/ChatView.razor.cs

@@ -51,7 +51,7 @@ namespace AntSK.Pages.ChatPage.Components
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
-            await LoadData();
+            LoadData();
             var msgs = await _localStorage.GetItemAsync<List<MessageInfo>>("msgs");
             if (msgs != null && msgs.Count > 0)
             {
@@ -61,12 +61,12 @@ namespace AntSK.Pages.ChatPage.Components
 
         protected override async Task OnParametersSetAsync()
         {
-           await LoadData();
+            LoadData();
         }
 
-        private async Task LoadData()
+        private void LoadData()
         {
-            app =await _apps_Repositories.GetFirstAsync(p => p.Id == AppId);
+            app = _apps_Repositories.GetFirst(p => p.Id == AppId);
         }
 
         protected async Task OnClearAsync()

src/AntSK/Pages/KmsPage/AddKms.razor

@@ -50,6 +50,9 @@
                 <FormItem Label="重叠部分(token)" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
                     <AntDesign.InputNumber @bind-Value="context.OverlappingTokens" PlaceHolder="重叠部分"></AntDesign.InputNumber>
                 </FormItem>
+                <FormItem Label="是否开启OCR" LabelCol="LayoutModel._formItemLayout.LabelCol" WrapperCol="LayoutModel._formItemLayout.WrapperCol">
+                    <Switch @bind-Value="@isOcr" />
+                </FormItem>
 
                 <FormItem Label=" " Style="margin-top:32px" WrapperCol="LayoutModel._submitFormLayout.WrapperCol">
                     <Button Type="primary" HtmlType="submit">

src/AntSK/Pages/KmsPage/AddKms.razor.cs

@@ -25,6 +25,8 @@ namespace AntSK.Pages.KmsPage
 
         private List<AIModels> _chatList { get; set; }
         private List<AIModels> _embeddingList { get; set; }
+
+        private bool isOcr { get; set; }
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
@@ -36,6 +38,7 @@ namespace AntSK.Pages.KmsPage
             {
                 //查看
                 _kmsModel = await _kmss_Repositories.GetFirstAsync(p => p.Id == KmsId);
+                isOcr = _kmsModel.IsOCR == 1;
             }
         }
         private void HandleSubmit()
@@ -51,6 +54,10 @@ namespace AntSK.Pages.KmsPage
                 _ = Message.Error("行切片需小于段落切片！", 2);
                 return;
             }
+            if (isOcr)
+            {
+                _kmsModel.IsOCR = 1;
+            }
 
             if (string.IsNullOrEmpty(KmsId))
             {

src/AntSK/Pages/KmsPage/KmsDetail.razor

@@ -15,8 +15,7 @@
                 <ChildContent>
                     <div class="standardList">
                         <Card Class="listCard"
-                              Title="知识库文档"
-                              >
+                              Title="知识库文档">
 
                             <Extra>
                                 <Button Type="@ButtonType.Primary" Style="position: absolute; right:360px; margin-bottom: 8px;" OnClick="Refresh">刷新 </Button>
@@ -106,7 +105,7 @@
             <TabPane Key="2">
                 <TabTemplate>搜索测试</TabTemplate>
                 <ChildContent>
-                   <KmsTest KmsId="@KmsId"></KmsTest>
+                    <KmsTest KmsId="@KmsId"></KmsTest>
                 </ChildContent>
             </TabPane>
         </Tabs>
@@ -125,6 +124,12 @@
         <FormItem Label="URL地址">
             <Input @bind-Value="@context.Url" />
         </FormItem>
+        <FormItem Label="切分方式">
+            <RadioGroup @bind-Value="@_isQa">
+                <Radio Value="false">直接切分</Radio>
+                <Radio Value="true">QA切分</Radio>
+            </RadioGroup>
+        </FormItem>
     </Form>
 </Modal>
 
@@ -136,10 +141,16 @@
     <Form Model="@textModel"
           LabelColSpan="8"
           WrapperColSpan="16"
-          @ref="@_textForm">
+          @ref="@_textForm">  
         <FormItem Label="文本内容">
             <TextArea @bind-Value="@context.Text" Rows="5" />
         </FormItem>
+        <FormItem Label="切分方式">
+            <RadioGroup @bind-Value="@_isQa">
+                <Radio Value="false">直接切分</Radio>
+                <Radio Value="true">QA切分</Radio>
+            </RadioGroup>
+        </FormItem>
     </Form>
 </Modal>
 
@@ -147,7 +158,7 @@
        Visible="@_fileVisible"
        OnOk="@FileHandleOk"
        OnCancel="@FileHandleCancel"
-       ConfirmLoading="@_fileConfirmLoading">
+       ConfirmLoading="@_fileConfirmLoading">   
     <Upload Action="@("api/File/UploadFile")"
             Name="file"
             Drag
@@ -159,10 +170,24 @@
             <Icon Type="inbox" />
         </p>
         <p class="ant-upload-text">单击或拖动文件到此区域进行上传</p>
-        <p class="ant-upload-hint">
-            支持txt、word、pdf、md、excel、ppt等文件。
-        </p>
+        @if (km.IsOCR == 1)
+        {
+            <p class="ant-upload-hint">
+                支持 txt、word、pdf、md、excel、ppt、jpeg、png、tiff 等文件。
+            </p>
+        }
+        else
+        {
+            <p class="ant-upload-hint">
+                支持 txt、word、pdf、md、excel、ppt 等文件。
+            </p>
+        }
+
     </Upload>
+    <RadioGroup @bind-Value="@_isQa" Style="margin-top:5px;">
+        <Radio Value="false">直接切分</Radio>
+        <Radio Value="true">QA切分</Radio>
+    </RadioGroup>
 </Modal>
 
 <Modal Title="Excel导入"

src/AntSK/Pages/KmsPage/KmsDetail.razor.cs

@@ -19,6 +19,31 @@ namespace AntSK.Pages.KmsPage
         [Parameter]
         public string KmsId { get; set; }
 
+        [Inject]
+        protected IConfirmService _confirmService { get; set; }
+
+        [Inject]
+        protected IKmsDetails_Repositories _kmsDetails_Repositories { get; set; }
+
+        [Inject]
+        protected IKmss_Repositories _kmss_Repositories { get; set; }
+
+        private MemoryServerless _memory { get; set; }
+
+        [Inject]
+        protected IKMService iKMService { get; set; }
+
+        [Inject]
+        protected MessageService? _message { get; set; }
+
+        [Inject]
+        protected BackgroundTaskBroker<ImportKMSTaskReq> _taskBroker { get; set; }
+
+        [Inject]
+        protected IHttpService _httpService { get; set; }
+
+        private Kmss km;
+
         private readonly KmsDetails _model = new KmsDetails();
 
         private bool _urlVisible = false;
@@ -51,34 +76,15 @@ namespace AntSK.Pages.KmsPage
 
         private List<KmsDetails> _data = new List<KmsDetails>();
 
-        [Inject]
-        protected IConfirmService _confirmService { get; set; }
 
-        [Inject]
-        protected IKmsDetails_Repositories _kmsDetails_Repositories { get; set; }
+        private bool _isQa { get; set; } = false;
 
-        [Inject]
-        protected IKmss_Repositories _kmss_Repositories { get; set; }
-
-        private MemoryServerless _memory { get; set; }
-
-        [Inject]
-        protected IKMService iKMService { get; set; }
-
-        [Inject]
-        protected MessageService? _message { get; set; }
-
-        [Inject]
-        protected BackgroundTaskBroker<ImportKMSTaskReq> _taskBroker { get; set; }
-
-        [Inject]
-        protected IHttpService _httpService { get; set; }
 
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
             _data = await _kmsDetails_Repositories.GetListAsync(p => p.KmsId == KmsId);
-            var km = _kmss_Repositories.GetFirst(p => p.Id == KmsId);
+            km = _kmss_Repositories.GetFirst(p => p.Id == KmsId);
             //使用知识库设置的参数，
             _memory = iKMService.GetMemoryByKMS(km.Id);
         }
@@ -112,6 +118,7 @@ namespace AntSK.Pages.KmsPage
                     ImportType = ImportType.Url,
                     KmsId = KmsId,
                     Url = urlModel.Url,
+                    IsQA = _isQa
                 });
                 _data = await _kmsDetails_Repositories.GetListAsync(p => p.KmsId == KmsId);
                 _urlVisible = false;
@@ -152,7 +159,8 @@ namespace AntSK.Pages.KmsPage
                 {
                     ImportType = ImportType.Text,
                     KmsId = KmsId,
-                    Text = textModel.Text
+                    Text = textModel.Text,
+                    IsQA = _isQa
                 });
                 _data = await _kmsDetails_Repositories.GetListAsync(p => p.KmsId == KmsId);
                 _textVisible = false;
@@ -190,7 +198,8 @@ namespace AntSK.Pages.KmsPage
                         ImportType = ImportType.File,
                         KmsId = KmsId,
                         FilePath = item.Url,
-                        FileName = item.FileName
+                        FileName = item.FileName,
+                        IsQA=_isQa
                     });
                 }
                 _data = await _kmsDetails_Repositories.GetListAsync(p => p.KmsId == KmsId);
@@ -229,7 +238,8 @@ namespace AntSK.Pages.KmsPage
                         ImportType = ImportType.Excel,
                         KmsId = KmsId,
                         FilePath = item.Url,
-                        FileName = item.FileName
+                        FileName = item.FileName,
+                        IsQA = false
                     });
                 }
                 _data = await _kmsDetails_Repositories.GetListAsync(p => p.KmsId == KmsId);

src/AntSK/Pages/KmsPage/KmsDetailList.razor

@@ -15,14 +15,8 @@
                             info(()=> Info(context.Text)) ,
                              })">
             <CardMeta>
-                <AvatarTemplate>
-
-                </AvatarTemplate>
-                <TitleTemplate>
-                    <a>@context.LastUpdate</a>
-                </TitleTemplate>
                 <DescriptionTemplate>
-                    <Paragraph class="item" Ellipsis Style=" white-space: nowrap; ">
+                    <Paragraph class="item" Ellipsis Style="display: -webkit-box; -webkit-line-clamp: 3;-webkit-box-orient: vertical;overflow: hidden; text-overflow: ellipsis;white-space: normal;height:65px;">
                         <!--todo: Ellipsis not working-->
                         @context.Text
                     </Paragraph>

src/AntSK/Pages/Setting/AIModel/AddModel.razor.cs

@@ -319,7 +319,7 @@ namespace AntSK.Pages.Setting.AIModel
             switch (aiType)
             { 
                 case AIType.LLamaFactory:
-                    _aiModel.EndPoint = "http://localhost:8080/";
+                    _aiModel.EndPoint = "http://localhost:8000/";
                     _aiModel.AIModelType=AIModelType.Chat;
                     break;
                 case AIType.StableDiffusion:

src/AntSK/Pages/Setting/AIModel/ModelDown.razor

@@ -27,6 +27,7 @@
     </Content>
     <ChildContent>
         <div class="filterCardList">
+            <Spin Tip="加载中..." Spinning="@(loaddding)">
             <AntList TItem="HfModels"
                      Grid="LayoutModel._listGridType"
                      DataSource="_modelList">
@@ -59,6 +60,7 @@
                     </Card>
                 </ListItem>
             </AntList>
+            </Spin>
         </div>
     </ChildContent>
 </PageContainer>

src/AntSK/Pages/Setting/AIModel/ModelDown.razor.cs

@@ -1,12 +1,9 @@
-using AntDesign;
+using AntSK.Domain.Domain.Model.hfmirror;
+using AntSK.Domain.Utils;
 using AntSK.Models;
-using AntSK.Services;
-using DocumentFormat.OpenXml.Office2010.Excel;
 using Microsoft.AspNetCore.Components;
 using Newtonsoft.Json;
 using RestSharp;
-using AntSK.Domain.Utils;
-using AntSK.Domain.Domain.Model.hfmirror;
 
 namespace AntSK.Pages.Setting.AIModel
 {
@@ -16,27 +13,55 @@ namespace AntSK.Pages.Setting.AIModel
         private readonly IList<string> _selectCategories = new List<string>();
 
         private List<HfModels> _modelList = new List<HfModels>();
-        private string _modelType;
+        private string _modelType="gguf";
+        private bool loaddding = false;
         protected override async Task OnInitializedAsync()
         {
             await base.OnInitializedAsync();
             InitData("");
         }
 
-        private void InitData(string searchKey)
+        private async Task InitData(string searchKey)
         {
-            var param = searchKey.ConvertToString().Split(" ");
-
-            string urlBase = $"https://hf-mirror.com/models-json?sort=trending&search={_modelType}";
-            if (param.Count() > 0)
+            loaddding = true;
+            if (_modelType.Contains("safetensors"))
             {
-                urlBase += "+" + string.Join("+", param);
+                _modelList.Clear();
+                var param = searchKey.ConvertToString().Split(" ");
+                string[] lines = File.ReadAllLines(Path.Combine(AppContext.BaseDirectory, "StableDiffusionModelList.txt"));
+                foreach (string line in lines)
+                {
+                    string urlBase = $"https://hf-mirror.com/models-json?sort=trending&search={line}";
+                    if (param.Count() > 0)
+                    {
+                        urlBase += "+" + string.Join("+", param);
+                    }
+                    RestClient client = new RestClient();
+                    RestRequest request = new RestRequest(urlBase, Method.Get);
+                    var response =await client.ExecuteAsync(request);
+                    var model = JsonConvert.DeserializeObject<HfModel>(response.Content);
+                    _modelList.AddRange(model.models);
+                }
+
             }
-            RestClient client = new RestClient();
-            RestRequest request = new RestRequest(urlBase, Method.Get);
-            var response = client.Execute(request);
-            var model = JsonConvert.DeserializeObject<HfModel>(response.Content);
-            _modelList = model.models;
+            else
+            {
+                var param = searchKey.ConvertToString().Split(" ");
+
+                string urlBase = $"https://hf-mirror.com/models-json?sort=trending&search={_modelType}";
+                if (param.Count() > 0)
+                {
+                    urlBase += "+" + string.Join("+", param);
+                }
+                RestClient client = new RestClient();
+                RestRequest request = new RestRequest(urlBase, Method.Get);
+                var response = await client.ExecuteAsync(request);
+                var model = JsonConvert.DeserializeObject<HfModel>(response.Content);
+                _modelList = model.models;
+            }
+
+            loaddding = false;
+            InvokeAsync(StateHasChanged);
         }
 
         private async Task Search(string searchKey)

src/AntSK/Program.cs

@@ -65,15 +65,14 @@ builder.Services.AddBackgroundTaskBroker().AddHandler<ImportKMSTaskReq, BackGrou
     {
         NativeLibraryConfig
            .Instance
-           .WithCuda(false)
-           .WithLogs(true);
+           .WithCuda(false);
     }
     else if (LLamaSharpOption.RunType.ToUpper() == "GPU")
     {
         NativeLibraryConfig
         .Instance
         .WithCuda(true)
-        .WithLogs(true);
+        .WithAvx(NativeLibraryConfig.AvxLevel.Avx);
     }
 }
 

src/AntSK/plugins/KMSPlugin/QA/config.json

@@ -0,0 +1,12 @@
+{
+  "schema": 1,
+  "description": "知识库问答",
+  "execution_settings": {
+    "default": {
+      "temperature": 0.0,
+      "top_p": 0.0,
+      "presence_penalty": 0.0,
+      "frequency_penalty": 0.0
+    }
+  }
+}

src/AntSK/plugins/KMSPlugin/QA/skprompt.txt

@@ -0,0 +1,13 @@
+我会给你一段文本，学习它们，并整理学习成果，要求为：
+1. 提出最多 25 个问题。
+2. 给出每个问题的答案。
+3. 答案要详细完整，答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 markdown 元素。
+4. 按格式返回多个问题和答案:
+
+Q1: 问题。
+A1: 答案。
+Q2:
+A2:
+……
+
+我的文本："""{{$input}}"""

src/AntSk.LLM/AntSK.LLM.csproj

@@ -17,8 +17,32 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="stable-diffusion.dll">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    <None Update="StableDiffusion\Backend\CPU\stable-diffusion.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\CPU\stable-diffusion.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\Cuda11\stable-diffusion.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\Cuda11\stable-diffusion.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\Cuda12\stable-diffusion.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\Cuda12\stable-diffusion.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\ROCm\stable-diffusion.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusion\Backend\ROCm\stable-diffusion.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="StableDiffusionModelList.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
 

src/AntSk.LLM/StableDiffusion/SDHelper.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Drawing;
+using System.Drawing;
 using System.Drawing.Imaging;
 using System.Runtime.InteropServices;
 
@@ -23,15 +22,16 @@ namespace AntSK.LLM.StableDiffusion
         static readonly Native.SdLogCallback sd_Log_Cb;
         static readonly Native.SdProgressCallback sd_Progress_Cb;
 
-        static SDHelper()
-        {
-            sd_Log_Cb = new Native.SdLogCallback(OnNativeLog);
-            Native.sd_set_log_callback(sd_Log_Cb, IntPtr.Zero);
+        //Hide the code below so that the process can be seen in console.
+        //static SDHelper()
+        //{
+        //    sd_Log_Cb = new Native.SdLogCallback(OnNativeLog);
+        //    Native.sd_set_log_callback(sd_Log_Cb, IntPtr.Zero);
 
-            sd_Progress_Cb = new Native.SdProgressCallback(OnProgressRunning);
-            Native.sd_set_progress_callback(sd_Progress_Cb, IntPtr.Zero);
+        //    sd_Progress_Cb = new Native.SdProgressCallback(OnProgressRunning);
+        //    Native.sd_set_progress_callback(sd_Progress_Cb, IntPtr.Zero);
 
-        }
+        //}
 
         public static bool Initialize(ModelParams modelParams)
         {
@@ -83,6 +83,16 @@ namespace AntSK.LLM.StableDiffusion
         {
             if (!IsInitialized) throw new ArgumentNullException("Model not loaded!");
 
+            IntPtr cnPtr = IntPtr.Zero;
+            if (textToImageParams.ControlCond != null)
+            {
+                if (textToImageParams.ControlCond.Width > 1)
+                {
+                    SDImage cnImg = GetSDImageFromBitmap(textToImageParams.ControlCond);
+                    cnPtr = GetPtrFromImage(cnImg);
+                }
+            }
+
             SDImagePtr sd_Image_ptr = Native.txt2img(sd_ctx,
                           textToImageParams.Prompt,
                           textToImageParams.NegativePrompt,
@@ -94,7 +104,7 @@ namespace AntSK.LLM.StableDiffusion
                           textToImageParams.SampleSteps,
                           textToImageParams.Seed,
                           textToImageParams.BatchCount,
-                          SDImagePtr.Zero,
+                          cnPtr,
                           textToImageParams.ControlStrength,
                           textToImageParams.StyleStrength,
                           textToImageParams.NormalizeInput,
@@ -200,6 +210,13 @@ namespace AntSK.LLM.StableDiffusion
             return sd_Image;
         }
 
+        private static IntPtr GetPtrFromImage(SDImage sdImg)
+        {
+            IntPtr imgPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(SDImage)));
+            Marshal.StructureToPtr(sdImg, imgPtr, false);
+            return imgPtr;
+        }
+
         private static void OnNativeLog(SdLogLevel level, string text, IntPtr data)
         {
             Log?.Invoke(null, new StableDiffusionEventArgs.StableDiffusionLogEventArgs { Level = level, Text = text });

src/AntSk.LLM/StableDiffusionModelList.txt

@@ -0,0 +1,6 @@
+AsAHuman/chilloutmix
+GraMpa7/dreamsharper
+Airic/Anything-V4.5
+liqira/anythingv3
+wind1/MoYou
+Reuploadingfromcivitai/DosMix