Merge pull request #73 from AIDotNet/feature_deldimensions

add DelDimensions

README.md

@@ -51,6 +51,8 @@ https://antsk.ai-dotnet.com/
 默认密码：test
 
 由于云服务器配置较低，无法运行本地模型，所以把系统设置权限关闭了，大家看看界面即可，要使用本地模型，请下载自行使用
+
+请勿在演示站点上传敏感信息
 ```
 
 ### 其他功能示例
@@ -89,7 +91,7 @@ version: '3.8'
 services:
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/AIDotNet/antsk:v0.2.7
+    image: registry.cn-hangzhou.aliyuncs.com/AIDotNet/antsk:v0.2.9
     ports:
       - 5000:5000
     networks:

docker-compose.simple.yml

@@ -3,9 +3,9 @@ version: '3.8'
 services:
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.7
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.9
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.7
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.9
     ports:
       - 5000:5000
     networks:

docker-compose.yml

@@ -18,9 +18,9 @@ services:
       - ./pg/data:/var/lib/postgresql/data
   antsk:
     container_name: antsk
-    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.7
+    image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:v0.2.9
     # 如果需要pytorch环境需要使用下面这个镜像，镜像比较大
-    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.7
+    # image: registry.cn-hangzhou.aliyuncs.com/xuzeyu91/antsk:p0.2.9
     ports:
       - 5000:5000
     networks:

src/AntSK.Domain/AntSK - Backup.Domain.csproj

@@ -0,0 +1,53 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <DocumentationFile>AntSK.Domain.xml</DocumentationFile>
+    <NoWarn>CA1050,CA1707,CA2007,VSTHRD111,CS1591,RCS1110,CA5394,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,,SKEXP0012,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0040,SKEXP0041,SKEXP0042,SKEXP0050,SKEXP0051,SKEXP0052,SKEXP0053,SKEXP0054,SKEXP0055,SKEXP0060,SKEXP0061,SKEXP0101,SKEXP0102</NoWarn>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="AntDesign.Charts" Version="0.5.1" />
+    <PackageReference Include="AntDesign.ProLayout" Version="0.18.2" />
+    <PackageReference Include="BlazorComponents.Terminal" Version="0.6.0" />
+    <PackageReference Include="Blazored.LocalStorage" Version="4.5.0" />
+  
+    <PackageReference Include="pythonnet" Version="3.0.3" />
+    
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
+    
+    <PackageReference Include="AutoMapper" Version="8.1.0" />
+    <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
+    <PackageReference Include="Markdig" Version="0.37.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="SqlSugarCore" Version="5.1.4.151" />
+    <PackageReference Include="System.Data.SQLite.Core" Version="1.0.118" />
+    <PackageReference Include="RestSharp" Version="110.2.0" />
+    <PackageReference Include="NPOI" Version="2.7.0" />
+    
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.7.1" />
+    <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.7.1" />
+    <PackageReference Include="Microsoft.SemanticKernel.Plugins.Core" Version="1.7.1-alpha" />
+    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.36.240415.2" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Postgres" Version="0.36.240415.2" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.36.240415.2" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Redis" Version="0.36.240415.2" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="0.36.240415.2" />
+
+    <PackageReference Include="LLamaSharp" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.11.2" />
+    
+    
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\AntSK.LLamaFactory\AntSK.LLamaFactory.csproj" />
+    <ProjectReference Include="..\AntSk.LLM\AntSK.LLM.csproj" />
+    <ProjectReference Include="..\AntSK.OCR\AntSK.OCR.csproj" />
+    <ProjectReference Include="..\MiddleWare\AntSK.BackgroundTask\AntSK.BackgroundTask.csproj" />
+  </ItemGroup>
+
+</Project>

src/AntSK.Domain/AntSK.Domain.csproj

@@ -19,9 +19,9 @@
     
     <PackageReference Include="AutoMapper" Version="8.1.0" />
     <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
-    <PackageReference Include="Markdig" Version="0.36.2" />
+    <PackageReference Include="Markdig" Version="0.37.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="SqlSugarCore" Version="5.1.4.151" />
+    <PackageReference Include="SqlSugarCore" Version="5.1.4.152" />
     <PackageReference Include="System.Data.SQLite.Core" Version="1.0.118" />
     <PackageReference Include="RestSharp" Version="110.2.0" />
     <PackageReference Include="NPOI" Version="2.7.0" />
@@ -29,17 +29,17 @@
     <PackageReference Include="Microsoft.SemanticKernel" Version="1.7.1" />
     <PackageReference Include="Microsoft.SemanticKernel.Core" Version="1.7.1" />
     <PackageReference Include="Microsoft.SemanticKernel.Plugins.Core" Version="1.7.1-alpha" />
-    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240321.1" />
-    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Postgres" Version="0.35.240321.1" />
-    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240321.1" />
-    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Redis" Version="0.35.240321.1" />
-    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="0.35.240321.1" />
+    <PackageReference Include="Microsoft.KernelMemory.Core" Version="$(KMVersion)" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Postgres" Version="$(KMVersion)" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="$(KMVersion)" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.Redis" Version="$(KMVersion)" />
+    <PackageReference Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="$(KMVersion)" />
 
-    <PackageReference Include="LLamaSharp" Version="0.11.2" />
-    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="0.11.2" />
-    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.11.2" />
-    <PackageReference Include="LLamaSharp.kernel-memory" Version="0.11.2" />
-    <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.11.2" />
+    <PackageReference Include="LLamaSharp" Version="$(LLamaSharpVersion)" />
+    <PackageReference Include="LLamaSharp.Backend.Cpu" Version="$(LLamaSharpVersion)" />
+    <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="$(LLamaSharpVersion)" />
+    <PackageReference Include="LLamaSharp.kernel-memory" Version="$(LLamaSharpVersion)" />
+    <PackageReference Include="LLamaSharp.semantic-kernel" Version="$(LLamaSharpVersion)" />
     
     
   </ItemGroup>

src/AntSK.Domain/Directory.Build.props

@@ -0,0 +1,8 @@
+<Project>
+  <!-- See https://aka.ms/dotnet/msbuild/customize for more details on customizing your build -->
+  <PropertyGroup>
+
+    <KMVersion>0.36.240416.1</KMVersion>
+    <LLamaSharpVersion>0.11.2</LLamaSharpVersion>
+  </PropertyGroup>
+</Project>

src/AntSK.Domain/Domain/Other/QAHandler.cs

@@ -1,4 +1,6 @@
-using AntSK.Domain.Domain.Model;
+using AntSK.Domain.Domain.Interface;
+using AntSK.Domain.Domain.Model;
+using AntSK.Domain.Utils;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI.OpenAI;
@@ -7,10 +9,12 @@ using Microsoft.KernelMemory.DataFormats.Text;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Extensions;
 using Microsoft.KernelMemory.Pipeline;
+using Microsoft.SemanticKernel;
 using Newtonsoft.Json;
 using RestSharp;
 using System.Security.Policy;
 using System.Text;
+using System.Text.RegularExpressions;
 
 namespace AntSK.Domain.Domain.Other
 {
@@ -20,9 +24,11 @@ namespace AntSK.Domain.Domain.Other
         private readonly IPipelineOrchestrator _orchestrator;
         private readonly ILogger<QAHandler> _log;
         private readonly TextChunker.TokenCounter _tokenCounter;
+        private readonly IKernelService _kernelService;
         public QAHandler(
             string stepName,
             IPipelineOrchestrator orchestrator,
+            IKernelService kernelService,
             TextPartitioningOptions? options = null,
             ILogger<QAHandler>? log = null
             )
@@ -34,6 +40,7 @@ namespace AntSK.Domain.Domain.Other
 
             this._log = log ?? DefaultLogger<QAHandler>.Instance;
             this._tokenCounter = DefaultGPTTokenizer.StaticCountTokens;
+            this._kernelService = kernelService;
         }
 
         /// <inheritdoc />
@@ -88,15 +95,27 @@ namespace AntSK.Domain.Domain.Other
                                 this._log.LogDebug("Partitioning text file {0}", file.Name);
                                 string content = partitionContent.ToString();
 
-                                using (HttpClient httpclient = new HttpClient())
+                                var kernel = _kernelService.GetKernelByAIModelID(StepName);
+                                var lines = TextChunker.SplitPlainTextLines(content, 299);
+                                var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 3000);
+                                KernelFunction jsonFun = kernel.Plugins.GetFunction("KMSPlugin", "QA");
+
+                                List<string> qaList = new List<string>();
+                                foreach (var para in paragraphs)
                                 {
-                                    httpclient.Timeout = TimeSpan.FromMinutes(10);
-                                    StringContent scontent = new StringContent(JsonConvert.SerializeObject(new QAModel() { ChatModelId = StepName, Context = content }), Encoding.UTF8, "application/json");
-                                    HttpResponseMessage response = await httpclient.PostAsync("http://localhost:5000/api/KMS/QA", scontent);
-                                    List<string> qaList = JsonConvert.DeserializeObject<List<string>>( await response.Content.ReadAsStringAsync());
-                                    sentences = qaList;
-                                    partitions = qaList;
+                                    var qaresult = await kernel.InvokeAsync(function: jsonFun, new KernelArguments() { ["input"] = para });
+                                    var qaListStr = qaresult.GetValue<string>().ConvertToString();
+
+                                    string pattern = @"Q\d+:.*?A\d+:.*?(?=(Q\d+:|$))";
+                                    RegexOptions options = RegexOptions.Singleline;
+
+                                    foreach (Match match in Regex.Matches(qaListStr, pattern, options))
+                                    {
+                                        qaList.Add(match.Value.Trim()); // Trim用于删除可能的首尾空格
+                                    }
                                 }
+                                sentences = qaList;
+                                partitions = qaList;        
                                 break;
                             }
                         default:

src/AntSK.Domain/Domain/Service/KMService.cs

@@ -29,7 +29,8 @@ namespace AntSK.Domain.Domain.Service
     public class KMService(
         IKmss_Repositories _kmss_Repositories,
         IAIModels_Repositories _aIModels_Repositories,
-        IMessageService? _message
+        IMessageService? _message,
+        IKernelService _kernelService
     ) : IKMService
     {
         private MemoryServerless _memory;
@@ -115,7 +116,7 @@ namespace AntSK.Domain.Domain.Service
                 //加载向量库
                 WithMemoryDbByVectorDB(memoryBuild);
               
-                _memory = memoryBuild.Build<MemoryServerless>();
+                _memory = memoryBuild.AddSingleton<IKernelService>(_kernelService).Build<MemoryServerless>();
                 return _memory;
             }
             //else {

src/AntSK.Domain/Domain/Service/LLamaFactoryService.cs

@@ -92,9 +92,9 @@ namespace AntSK.Domain.Domain.Service
                         WorkingDirectory = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location), "llamafactory"),
                     }
                 };
-                process.StartInfo.Environment["CUDA_VISIBLE_DEVICES"] = "0";
+                process.StartInfo.Environment["CUDA_VISIBLE_DEVICES"] = Environment.GetEnvironmentVariable("CUDA_VISIBLE_DEVICES") ?? "0";
                 process.StartInfo.Environment["API_PORT"] = "8000";
-                process.StartInfo.EnvironmentVariables["USE_MODELSCOPE_HUB"] = "1";
+                process.StartInfo.EnvironmentVariables["USE_MODELSCOPE_HUB"] = Environment.GetEnvironmentVariable("USE_MODELSCOPE_HUB") ?? "1";
                 process.OutputDataReceived += (sender, eventArgs) =>
                 {
                     Console.WriteLine($"{eventArgs.Data}");

src/AntSK.OCR/AntSK.OCR.csproj

@@ -6,7 +6,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240321.1" />
+    <PackageReference Include="Microsoft.KernelMemory.Core" Version="$(KMVersion)" />
     <PackageReference Include="OpenCvSharp4.runtime.win" Version="4.9.0.20240103" />
     <PackageReference Include="Sdcb.OpenCvSharp4.mini.runtime.debian.12-x64" Version="4.8.0.20231125" />
     <PackageReference Include="Sdcb.OpenVINO" Version="0.6.4" />

src/AntSK.OCR/Directory.Build.props

@@ -0,0 +1,8 @@
+<Project>
+  <!-- See https://aka.ms/dotnet/msbuild/customize for more details on customizing your build -->
+  <PropertyGroup>
+
+    <KMVersion>0.36.240416.1</KMVersion>
+    <LLamaSharpVersion>0.11.2</LLamaSharpVersion>
+  </PropertyGroup>
+</Project>

src/AntSK.sln

@@ -27,11 +27,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AntSK.LLamaFactory", "AntSK
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AntSK.OCR", "AntSK.OCR\AntSK.OCR.csproj", "{6195F7AA-18C2-4372-85CA-11FC4B522686}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Settings", "Settings", "{EB227FAB-6060-4271-9A0A-6C99C7965126}"
-	ProjectSection(SolutionItems) = preProject
-		NuGet.config = NuGet.config
-	EndProjectSection
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

src/AntSK/Controllers/KMSController.cs

@@ -83,9 +83,7 @@ namespace AntSK.Controllers
                 {
                     qaList.Add(match.Value.Trim()); // Trim用于删除可能的首尾空格
                 }
-
             }
-
             return Ok(qaList);
         }
     }

src/AntSK/Pages/Setting/AIModel/ModelList.razor.cs

@@ -16,6 +16,13 @@ namespace AntSK.Pages.Setting.AIModel
         [Inject]
         protected IAIModels_Repositories _aIModels_Repositories { get; set; }
 
+        [Inject]
+        protected IApps_Repositories _apps_Repositories{ get; set; }
+        [Inject]
+        protected IKmss_Repositories _kmss_Repositories { get; set; }
+
+        [Inject] protected MessageService? Message { get; set; }
+
         [Inject]
         IConfirmService _confirmService { get; set; }
 
@@ -59,6 +66,16 @@ namespace AntSK.Pages.Setting.AIModel
             var result = await _confirmService.Show(content, title, ConfirmButtons.YesNo);
             if (result == ConfirmResult.Yes)
             {
+                if (_apps_Repositories.IsAny(p => p.ChatModelID == modelid || p.EmbeddingModelID == modelid))
+                {
+                    _ = Message.Error("该模型有应用在使用，请先删除应用后才允许删除该模型");
+                    return;
+                }
+                if (_kmss_Repositories.IsAny(p => p.ChatModelID == modelid || p.EmbeddingModelID == modelid))
+                {
+                    _ = Message.Error("该模型有知识库在使用，请先删除知识库后才允许删除该模型");
+                    return;
+                }
                 await _aIModels_Repositories.DeleteAsync(modelid);
                 await InitData("");
             }

src/AntSK/Pages/Setting/DelKms/DelKms.razor

@@ -0,0 +1,88 @@
+@namespace AntSK.Pages.Setting.AIModel
+@page "/setting/delkms"
+@using AntSK.Services.Auth
+@using AntSK.Domain.Options
+@using AntSK.Domain.Repositories
+@inherits AuthComponentBase
+@using Microsoft.AspNetCore.Authorization
+@attribute [Authorize(Roles = "AntSKAdmin")]
+
+<GridContent>
+    <Card>
+        <Result Status="error"
+                Title="删除向量表"
+                SubTitle="删除向量表将会删除您的所有知识库文件，以及知识库向量。请谨慎操作！"
+                Style="margin-top: 48px; margin-bottom: 16px;">
+            <Extra>
+                <Button Type="primary" OnClick="DelDimensions">删除向量表</Button>
+            </Extra>
+            <ChildContent>
+                <div class="title__b__0">
+                    当您切换不同维度的向量模型时，出现异常:
+                </div>
+                <div style="margin-bottom: 16px;">
+                <Icon Type="close-circle" Theme="outline" Class="error_icon" Style="margin-right: 8px;" />
+                Npgsql.PostgresException (0x80004005): 22000: expected 2048 dimensions, not 1024  
+                </div>
+                <div>
+                <Icon Type="close-circle" Theme="outline" Class="error_icon" Style="margin-right: 8px;" />
+                您需要删除向量表，然后重建向量维度。
+                </div>
+            </ChildContent>
+        </Result>
+    </Card>
+</GridContent>
+
+<Modal @ref="_modal" Visible="_showAdminPwdVisible" Footer="null" Closable Title="请输入管理员密码" OnCancel="OnCancel" DestroyOnClose>
+    <Flex Gap="10" Vertical>
+        <InputGroup>
+            <InputPassword @bind-Value="@adminPwd" Placeholder="管理员密码" Size="@InputSize.Large" />
+        </InputGroup>
+        <InputGroup>
+            <Button Type="primary" OnClick="DelSubmit">确认</Button>
+        </InputGroup>
+    </Flex>
+</Modal>
+@code {
+    [Inject] protected MessageService? Message { get; set; }
+    [Inject] protected IKmsDetails_Repositories kmsDetails_Repositories { get; set; }
+
+    [Inject] protected IConfirmService _confirmService { get; set; }
+
+    private Modal _modal;
+    private bool _showAdminPwdVisible = false;
+    private string adminPwd = "";
+
+    public async Task DelDimensions()
+    {
+        var content = "删除向量表将会删除您的所有知识库文件，以及知识库向量。请谨慎操作！";
+        var title = "删除向量表";
+        var result = await _confirmService.Show(content, title, ConfirmButtons.YesNo);
+        if (result == ConfirmResult.Yes)
+        {
+            _showAdminPwdVisible = true;
+        }
+    }
+
+    public async Task DelSubmit() {
+        if (adminPwd != LoginOption.Password)
+        {
+            _ = Message.Error("管理员密码错误！");
+            return;
+        }
+
+        kmsDetails_Repositories.GetDB().Ado.ExecuteCommand("DROP TABLE IF EXISTS \"km-kms\"");    
+        var kmsDetails = await kmsDetails_Repositories.GetListAsync();
+        foreach (var detail in kmsDetails)
+        {
+            kmsDetails_Repositories.Delete(detail);
+        }     
+        _showAdminPwdVisible = false;
+        _ = Message.Info("删除成功！");
+    }
+
+    private void OnCancel()
+    {
+        _showAdminPwdVisible = false;
+    }
+}

src/AntSK/Program.cs

@@ -65,14 +65,20 @@ builder.Services.AddBackgroundTaskBroker().AddHandler<ImportKMSTaskReq, BackGrou
     {
         NativeLibraryConfig
            .Instance
-           .WithCuda(false);
+           .WithCuda(false)
+           .WithLogCallback((level, message) => {
+               Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
+           });
     }
     else if (LLamaSharpOption.RunType.ToUpper() == "GPU")
     {
         NativeLibraryConfig
         .Instance
         .WithCuda(true)
-        .WithAvx(NativeLibraryConfig.AvxLevel.Avx);
+        .WithAvx(NativeLibraryConfig.AvxLevel.Avx)
+        .WithLogCallback((level, message) => {
+            Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
+        });
     }
 }
 

src/AntSK/wwwroot/data/menu.json

@@ -55,6 +55,11 @@
         "path": "/setting/modeldown",
         "name": "模型下载",
         "key": "setting.modeldown"
+      },
+      {
+        "path": "/setting/delkms",
+        "name": "删除向量表",
+        "key": "setting.delkms"
       }
     ]
   },