From add4964a65bd322b178c670162c317537f0a5aa4 Mon Sep 17 00:00:00 2001 From: token <239573049@qq.com> Date: Tue, 16 Apr 2024 14:08:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81QA=E9=97=AE=E7=AD=94=E8=A7=A3?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Wikis/Dto/CreateWikiDetailsInput.cs | 2 + .../Application/Wikis/WikiCommandHandler.cs | 53 +- .../Backgrounds/QuantizeBackgroundService.cs | 56 +- .../Domain/Wikis/Aggregates/WikiDetail.cs | 35 +- .../Infrastructure/KM/QAHandler.cs | 172 +++++ .../20240416051957_UpdateDetaiis.Designer.cs | 725 ++++++++++++++++++ .../20240416051957_UpdateDetaiis.cs | 144 ++++ .../Migrations/WikiDbContextModelSnapshot.cs | 42 +- src/Service/FastWiki.Service/Program.cs | 6 +- .../FastWiki.Service/Service/OpenAIService.cs | 6 +- .../Service/WikiMemoryService.cs | 2 + web/src/models/index.d.ts | 6 +- .../wiki-detail/features/UploadWikiFile.tsx | 26 +- .../wiki-detail/features/UploadWikiWeb.tsx | 26 +- 14 files changed, 1223 insertions(+), 78 deletions(-) create mode 100644 src/Service/FastWiki.Service/Infrastructure/KM/QAHandler.cs create mode 100644 src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.Designer.cs create mode 100644 src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.cs diff --git a/src/Contracts/FastWiki.Service.Contracts/Wikis/Dto/CreateWikiDetailsInput.cs b/src/Contracts/FastWiki.Service.Contracts/Wikis/Dto/CreateWikiDetailsInput.cs index e6df879a..d580debf 100644 --- a/src/Contracts/FastWiki.Service.Contracts/Wikis/Dto/CreateWikiDetailsInput.cs +++ b/src/Contracts/FastWiki.Service.Contracts/Wikis/Dto/CreateWikiDetailsInput.cs @@ -19,4 +19,6 @@ public sealed class CreateWikiDetailsInput public ProcessMode Mode { get; set; } = ProcessMode.Auto; public TrainingPattern TrainingPattern { get; set; } = TrainingPattern.Subsection; + + public string? QAPromptTemplate { get; set; } } \ No newline at end of file diff --git a/src/Service/FastWiki.Service/Application/Wikis/WikiCommandHandler.cs b/src/Service/FastWiki.Service/Application/Wikis/WikiCommandHandler.cs index 107082bf..2a998b06 100644 --- a/src/Service/FastWiki.Service/Application/Wikis/WikiCommandHandler.cs +++ b/src/Service/FastWiki.Service/Application/Wikis/WikiCommandHandler.cs @@ -48,17 +48,17 @@ public async Task CreateWikiDetailsAsync(CreateWikiDetailsCommand command) { var wikiDetail = new WikiDetail(command.Input.WikiId, command.Input.Name, command.Input.FilePath, command.Input.FileId, 0, "file"); + wikiDetail.TrainingPattern = command.Input.TrainingPattern; + wikiDetail.Mode = command.Input.Mode; + wikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; + wikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; + wikiDetail.OverlappingTokens = command.Input.OverlappingTokens; + wikiDetail.QAPromptTemplate = command.Input.QAPromptTemplate; wikiDetail = await wikiRepository.AddDetailsAsync(wikiDetail); - QuantizeWikiDetail quantizeWikiDetail = mapper.Map(wikiDetail); - quantizeWikiDetail.OverlappingTokens = command.Input.OverlappingTokens; - quantizeWikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; - quantizeWikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; - quantizeWikiDetail.Mode = command.Input.Mode; - quantizeWikiDetail.TrainingPattern = command.Input.TrainingPattern; - await QuantizeBackgroundService.AddWikiDetailAsync(quantizeWikiDetail); + await QuantizeBackgroundService.AddWikiDetailAsync(wikiDetail); } [EventHandler] @@ -66,15 +66,14 @@ public async Task CreateWikiDetailWebPageAsync(CreateWikiDetailWebPageCommand co { var wikiDetail = new WikiDetail(command.Input.WikiId, command.Input.Name, command.Input.Path, -1, 0, "web"); + wikiDetail.OverlappingTokens = command.Input.OverlappingTokens; + wikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; + wikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; + wikiDetail.Mode = command.Input.Mode; + wikiDetail.TrainingPattern = command.Input.TrainingPattern; wikiDetail = await wikiRepository.AddDetailsAsync(wikiDetail); - - var quantizeWikiDetail = mapper.Map(wikiDetail); - quantizeWikiDetail.OverlappingTokens = command.Input.OverlappingTokens; - quantizeWikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; - quantizeWikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; - quantizeWikiDetail.Mode = command.Input.Mode; - quantizeWikiDetail.TrainingPattern = command.Input.TrainingPattern; + var quantizeWikiDetail = mapper.Map(wikiDetail); await QuantizeBackgroundService.AddWikiDetailAsync(quantizeWikiDetail); } @@ -85,14 +84,15 @@ public async Task CreateWikiDetailDataAsync(CreateWikiDetailDataCommand command) var wikiDetail = new WikiDetail(command.Input.WikiId, command.Input.Name, command.Input.FilePath, command.Input.FileId, 0, "data"); + wikiDetail.OverlappingTokens = command.Input.OverlappingTokens; + wikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; + wikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; + wikiDetail.Mode = command.Input.Mode; + wikiDetail.TrainingPattern = command.Input.TrainingPattern; + wikiDetail = await wikiRepository.AddDetailsAsync(wikiDetail); - var quantizeWikiDetail = mapper.Map(wikiDetail); - quantizeWikiDetail.OverlappingTokens = command.Input.OverlappingTokens; - quantizeWikiDetail.MaxTokensPerLine = command.Input.MaxTokensPerLine; - quantizeWikiDetail.MaxTokensPerParagraph = command.Input.MaxTokensPerParagraph; - quantizeWikiDetail.Mode = command.Input.Mode; - quantizeWikiDetail.TrainingPattern = command.Input.TrainingPattern; + var quantizeWikiDetail = mapper.Map(wikiDetail); await QuantizeBackgroundService.AddWikiDetailAsync(quantizeWikiDetail); } @@ -143,18 +143,7 @@ public async Task RetryVectorDetailAsync(RetryVectorDetailCommand command) throw new UserFriendlyException("未找到数据"); } - await QuantizeBackgroundService.AddWikiDetailAsync(new QuantizeWikiDetail() - { - Path = wikiDetail.Path, - WikiId = wikiDetail.WikiId, - TrainingPattern = TrainingPattern.Subsection, - FileName = wikiDetail.FileName, - Type = wikiDetail.Type, - MaxTokensPerParagraph = 1000, - MaxTokensPerLine = 300, - OverlappingTokens = 100, - FileId = wikiDetail.FileId, - }); + await QuantizeBackgroundService.AddWikiDetailAsync(wikiDetail); } [EventHandler] diff --git a/src/Service/FastWiki.Service/Backgrounds/QuantizeBackgroundService.cs b/src/Service/FastWiki.Service/Backgrounds/QuantizeBackgroundService.cs index 5c18eafb..7f4deef9 100644 --- a/src/Service/FastWiki.Service/Backgrounds/QuantizeBackgroundService.cs +++ b/src/Service/FastWiki.Service/Backgrounds/QuantizeBackgroundService.cs @@ -1,5 +1,8 @@ +using System.Collections.Concurrent; using System.Threading.Channels; +using FastWiki.Service.Infrastructure.KM; using FastWiki.Service.Service; +using Microsoft.KernelMemory.Handlers; namespace FastWiki.Service.Backgrounds; @@ -8,6 +11,11 @@ namespace FastWiki.Service.Backgrounds; /// public sealed class QuantizeBackgroundService : BackgroundService { + /// + /// 线程安全字典 + /// + public static ConcurrentDictionary CacheWikiDetails { get; } = new(); + private readonly IServiceProvider _serviceProvider; /// @@ -20,7 +28,7 @@ public sealed class QuantizeBackgroundService : BackgroundService /// private static int _maxTask = 1; - private static readonly Channel WikiDetails = Channel.CreateBounded( + private static readonly Channel WikiDetails = Channel.CreateBounded( new BoundedChannelOptions(1000) { SingleReader = true, @@ -69,7 +77,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// /// /// - public static async Task AddWikiDetailAsync(QuantizeWikiDetail wikiDetail) + public static async Task AddWikiDetailAsync(WikiDetail wikiDetail) { await WikiDetails.Writer.WriteAsync(wikiDetail); } @@ -92,13 +100,16 @@ private async Task WikiDetailHandlerAsync() /// /// /// - private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServiceProvider service) + private static async ValueTask HandlerAsync(WikiDetail wikiDetail, IServiceProvider service) { var fileStorageRepository = service.GetRequiredService(); var wikiRepository = service.GetRequiredService(); var wikiMemoryService = service.GetRequiredService(); var wiki = await wikiRepository.FindAsync(x => x.Id == wikiDetail.WikiId); + + CacheWikiDetails.TryAdd(wikiDetail.Id.ToString(), new ValueTuple(wikiDetail, wiki)); + if (wikiDetail.Mode == ProcessMode.Auto) { wikiDetail.MaxTokensPerLine = 300; @@ -113,6 +124,19 @@ private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServ try { Console.WriteLine($"开始量化：ʼ{wikiDetail.FileName} {wikiDetail.Path} {wikiDetail.FileId}"); + List step = new List(); + if (wikiDetail.TrainingPattern == TrainingPattern.QA) + { + var stepName = wikiDetail.Id.ToString(); + serverless.Orchestrator.AddHandler("extract_text"); + serverless.Orchestrator.AddHandler(stepName); + serverless.Orchestrator.AddHandler("generate_embeddings"); + serverless.Orchestrator.AddHandler("save_memory_records"); + step.Add("extract_text"); + step.Add(stepName); + step.Add("generate_embeddings"); + step.Add("save_memory_records"); + } string result = string.Empty; if (wikiDetail.Type == "file") @@ -132,7 +156,7 @@ private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServ { "wikiDetailId", wikiDetail.Id.ToString() } - }, "wiki"); + }, "wiki", steps: step.ToArray()); } else if (wikiDetail.Type == "web") { @@ -146,7 +170,7 @@ private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServ { "wikiDetailId", wikiDetail.Id.ToString() } - }, "wiki"); + }, "wiki", steps: step.ToArray()); } else if (wikiDetail.Type == "data") { @@ -160,7 +184,7 @@ private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServ { "wikiDetailId", wikiDetail.Id.ToString() } - }, "wiki"); + }, "wiki", steps: step.ToArray()); } await wikiRepository.UpdateDetailsState(wikiDetail.Id, WikiQuantizationState.Accomplish); @@ -176,8 +200,13 @@ private static async ValueTask HandlerAsync(QuantizeWikiDetail wikiDetail, IServ await wikiRepository.UpdateDetailsState(wikiDetail.Id, WikiQuantizationState.Fail); } } + finally + { + CacheWikiDetails.Remove(wikiDetail.Id.ToString(), out _); + } } + private async Task LoadingWikiDetailAsync() { using var asyncServiceScope = _serviceProvider.CreateScope(); @@ -186,20 +215,7 @@ private async Task LoadingWikiDetailAsync() var mapper = asyncServiceScope.ServiceProvider.GetRequiredService(); foreach (var wikiDetail in await wikiRepository.GetFailedDetailsAsync()) { - await AddWikiDetailAsync(mapper.Map(wikiDetail)); + await AddWikiDetailAsync(wikiDetail); } } -} - -public sealed class QuantizeWikiDetail : WikiDetail -{ - public int MaxTokensPerParagraph { get; set; } - - public int MaxTokensPerLine { get; set; } - - public int OverlappingTokens { get; set; } - - public ProcessMode Mode { get; set; } = ProcessMode.Auto; - - public TrainingPattern TrainingPattern { get; set; } = TrainingPattern.Subsection; } \ No newline at end of file diff --git a/src/Service/FastWiki.Service/Domain/Wikis/Aggregates/WikiDetail.cs b/src/Service/FastWiki.Service/Domain/Wikis/Aggregates/WikiDetail.cs index 776d29fd..690646b7 100644 --- a/src/Service/FastWiki.Service/Domain/Wikis/Aggregates/WikiDetail.cs +++ b/src/Service/FastWiki.Service/Domain/Wikis/Aggregates/WikiDetail.cs @@ -37,6 +37,9 @@ public class WikiDetail : Entity, IAuditEntity /// public string Type { get; set; } + /// + /// 知识库状态 + /// public WikiQuantizationState State { get; set; } public long Creator { get; set; } @@ -47,6 +50,37 @@ public class WikiDetail : Entity, IAuditEntity public DateTime ModificationTime { get; set; } + + public int MaxTokensPerParagraph { get; set; } + + public int MaxTokensPerLine { get; set; } + + public int OverlappingTokens { get; set; } + + public ProcessMode Mode { get; set; } = ProcessMode.Auto; + + public TrainingPattern TrainingPattern { get; set; } = TrainingPattern.Subsection; + + /// + /// QAPrompt模板 + /// + public string? QAPromptTemplate { get; set; } = + """" + 我会给你一段文本，学习它们，并整理学习成果，要求为： + 1. 提出最多 20 个问题。 + 2. 给出每个问题的答案。 + 3. 答案要详细完整，答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 markdown 元素。 + 4. 按格式返回多个问题和答案: + + Q1: 问题。 + A1: 答案。 + Q2: + A2: + …… + + 我的文本："""{{$input}}""" + """"; + /// public WikiDetail(long wikiId, string fileName, string path, long fileId, int dataCount, string type) { @@ -61,6 +95,5 @@ public WikiDetail(long wikiId, string fileName, string path, long fileId, int da protected WikiDetail() { - } } \ No newline at end of file diff --git a/src/Service/FastWiki.Service/Infrastructure/KM/QAHandler.cs b/src/Service/FastWiki.Service/Infrastructure/KM/QAHandler.cs new file mode 100644 index 00000000..b74c3108 --- /dev/null +++ b/src/Service/FastWiki.Service/Infrastructure/KM/QAHandler.cs @@ -0,0 +1,172 @@ +using FastWiki.Service.Backgrounds; +using FastWiki.Service.Service; +using Microsoft.KernelMemory.AI.OpenAI; +using Microsoft.KernelMemory.Configuration; +using Microsoft.KernelMemory.DataFormats.Text; +using Microsoft.KernelMemory.Diagnostics; +using Microsoft.KernelMemory.Extensions; +using Microsoft.KernelMemory.Pipeline; + +namespace FastWiki.Service.Infrastructure.KM; + +/// +/// QA问答处理器 +/// +public class QAHandler : IPipelineStepHandler +{ + private readonly TextPartitioningOptions _options; + private readonly IPipelineOrchestrator _orchestrator; + private readonly OpenAIService _openAiService; + private readonly WikiMemoryService _wikiMemoryService; + private readonly ILogger _log; + private readonly TextChunker.TokenCounter _tokenCounter; + + /// + public QAHandler( + string stepName, + IPipelineOrchestrator orchestrator, OpenAIService openAiService, WikiMemoryService wikiMemoryService, + TextPartitioningOptions? options = null, + ILogger? log = null + ) + { + this.StepName = stepName; + this._orchestrator = orchestrator; + _openAiService = openAiService; + _wikiMemoryService = wikiMemoryService; + this._options = options ?? new TextPartitioningOptions(); + this._options.Validate(); + + this._log = log ?? DefaultLogger.Instance; + this._tokenCounter = DefaultGPTTokenizer.StaticCountTokens; + } + + /// + public string StepName { get; } + + /// + public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync( + DataPipeline pipeline, CancellationToken cancellationToken = default) + { + this._log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId); + + if (pipeline.Files.Count == 0) + { + this._log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", + pipeline.Index, pipeline.DocumentId); + return (true, pipeline); + } + + foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files) + { + // Track new files being generated (cannot edit originalFile.GeneratedFiles while looping it) + Dictionary newFiles = new(); + + foreach (KeyValuePair generatedFile in uploadedFile + .GeneratedFiles) + { + var file = generatedFile.Value; + if (file.AlreadyProcessedBy(this)) + { + this._log.LogTrace("File {0} already processed by this handler", file.Name); + continue; + } + + // Partition only the original text + if (file.ArtifactType != DataPipeline.ArtifactTypes.ExtractedText) + { + this._log.LogTrace("Skipping file {0} (not original text)", file.Name); + continue; + } + + // Use a different partitioning strategy depending on the file type + List partitions = new(); + List sentences = new(); + BinaryData partitionContent = await this._orchestrator + .ReadFileAsync(pipeline, file.Name, cancellationToken).ConfigureAwait(false); + + // Skip empty partitions. Also: partitionContent.ToString() throws an exception if there are no bytes. + if (partitionContent.ToArray().Length == 0) + { + continue; + } + + switch (file.MimeType) + { + case MimeTypes.PlainText: + case MimeTypes.MarkDown: + { + _log.LogDebug("Partitioning text file {0}", file.Name); + string content = partitionContent.ToString(); + + if (QuantizeBackgroundService.CacheWikiDetails.TryGetValue(StepName, out var wikiDetail)) + { + await foreach (var item in _openAiService + .QAAsync(wikiDetail.Item1.QAPromptTemplate, content, wikiDetail.Item2.Model, OpenAIOption.ChatToken, + OpenAIOption.ChatEndpoint, _wikiMemoryService) + .WithCancellation(cancellationToken)) + { + partitions.Add(item); + sentences.Add(item); + } + } + + + break; + } + default: + this._log.LogWarning("File {0} cannot be partitioned, type '{1}' not supported", file.Name, + file.MimeType); + // Don't partition other files + continue; + } + + if (partitions.Count == 0) + { + continue; + } + + this._log.LogDebug("Saving {0} file partitions", partitions.Count); + for (int partitionNumber = 0; partitionNumber < partitions.Count; partitionNumber++) + { + // TODO: turn partitions in objects with more details, e.g. page number + string text = partitions[partitionNumber]; + int sectionNumber = 0; // TODO: use this to store the page number (if any) + BinaryData textData = new(text); + + int tokenCount = this._tokenCounter(text); + this._log.LogDebug("Partition size: {0} tokens", tokenCount); + + var destFile = uploadedFile.GetPartitionFileName(partitionNumber); + await this._orchestrator.WriteFileAsync(pipeline, destFile, textData, cancellationToken) + .ConfigureAwait(false); + + var destFileDetails = new DataPipeline.GeneratedFileDetails + { + Id = Guid.NewGuid().ToString("N"), + ParentId = uploadedFile.Id, + Name = destFile, + Size = text.Length, + MimeType = MimeTypes.PlainText, + ArtifactType = DataPipeline.ArtifactTypes.TextPartition, + PartitionNumber = partitionNumber, + SectionNumber = sectionNumber, + Tags = pipeline.Tags, + ContentSHA256 = textData.CalculateSHA256(), + }; + newFiles.Add(destFile, destFileDetails); + destFileDetails.MarkProcessedBy(this); + } + + file.MarkProcessedBy(this); + } + + // Add new files to pipeline status + foreach (var file in newFiles) + { + uploadedFile.GeneratedFiles.Add(file.Key, file.Value); + } + } + + return (true, pipeline); + } +} \ No newline at end of file diff --git a/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.Designer.cs b/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.Designer.cs new file mode 100644 index 00000000..c2a4c1df --- /dev/null +++ b/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.Designer.cs @@ -0,0 +1,725 @@ +// +using System; +using FastWiki.Service.DataAccess; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FastWiki.Service.Migrations +{ + [DbContext(typeof(WikiDbContext))] + [Migration("20240416051957_UpdateDetaiis")] + partial class UpdateDetaiis + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FastWiki.Service.Domain.ChatApplications.Aggregates.ChatApplication", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ChatModel") + .IsRequired() + .HasColumnType("text"); + + b.Property("ChatType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Extend") + .IsRequired() + .HasColumnType("text"); + + b.Property("FunctionIds") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("MaxResponseToken") + .HasColumnType("integer"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("NoReplyFoundTemplate") + .HasColumnType("text"); + + b.Property("Opener") + .IsRequired() + .HasColumnType("text"); + + b.Property("Parameter") + .IsRequired() + .HasColumnType("text"); + + b.Property("Prompt") + .IsRequired() + .HasColumnType("text"); + + b.Property("ReferenceUpperLimit") + .HasColumnType("integer"); + + b.Property("Relevancy") + .HasColumnType("double precision"); + + b.Property("ShowSourceFile") + .HasColumnType("boolean"); + + b.Property("Temperature") + .HasColumnType("double precision"); + + b.Property("Template") + .IsRequired() + .HasColumnType("text"); + + b.Property("WikiIds") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name"); + + b.ToTable("wiki-chat-application", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.ChatApplications.Aggregates.ChatDialog", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ApplicationId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ChatId") + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ChatId"); + + b.ToTable("wiki-chat-dialog", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.ChatApplications.Aggregates.ChatDialogHistory", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ChatDialogId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Content") + .IsRequired() + .HasMaxLength(-1) + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Current") + .HasColumnType("boolean"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("ReferenceFile") + .IsRequired() + .HasColumnType("text"); + + b.Property("TokenConsumption") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ChatDialogId"); + + b.HasIndex("Creator"); + + b.ToTable("wiki-chat-dialog-history", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.ChatApplications.Aggregates.ChatShare", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("APIKey") + .HasColumnType("text"); + + b.Property("AvailableQuantity") + .HasColumnType("integer"); + + b.Property("AvailableToken") + .HasColumnType("bigint"); + + b.Property("ChatApplicationId") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Expires") + .HasColumnType("timestamp without time zone"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UsedToken") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("ChatApplicationId"); + + b.ToTable("wiki-chat-share", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Function.Aggregates.FastWikiFunctionCall", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Enable") + .HasColumnType("boolean"); + + b.Property("Imports") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("Items") + .IsRequired() + .HasColumnType("text"); + + b.Property("Main") + .IsRequired() + .HasColumnType("text"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Parameters") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("CreationTime"); + + b.ToTable("wiki-function-calls", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Model.Aggregates.FastModel", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ApiKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Description") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Enable") + .HasColumnType("boolean"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("Models") + .IsRequired() + .HasMaxLength(-1) + .HasColumnType("text"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("TestTime") + .HasColumnType("bigint"); + + b.Property("Type") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Url") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("UsedQuota") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("Name"); + + b.HasIndex("Type"); + + b.ToTable("wiki-fast-models", (string)null); + + b.HasData( + new + { + Id = "d28cb989669c46fba386666e84a2c72e", + ApiKey = "", + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5311), + Description = "OpenAI", + Enable = true, + IsDeleted = false, + Models = "[\"gpt-3.5-turbo\",\"gpt-3.5-turbo-0125\",\"gpt-3.5-turbo-1106\",\"gpt-3.5-turbo-16k\",\"gpt-3.5-turbo-0613\",\"gpt-3.5-turbo-16k-0613\",\"gpt-4-0125-preview\",\"gpt-4-turbo-preview\",\"gpt-4-1106-preview\",\"gpt-4-vision-preview\",\"gpt-4-1106-vision-preview\",\"gpt-4\",\"gpt-4-0613\",\"gpt-4-32k\",\"gpt-4-32k-0613\"]", + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5312), + Name = "OpenAI", + Order = 1, + Type = "OpenAI", + Url = "http://ai-api.token-ai.cn/", + UsedQuota = 0L + }, + new + { + Id = "0b01f0fddb63483e8174fba06522108a", + ApiKey = "", + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), + Description = "星火大模型", + Enable = true, + IsDeleted = false, + Models = "[\"SparkDesk-v3.5\",\"SparkDesk-v3.1\",\"SparkDesk-v1.5\",\"SparkDesk-v2.1\"]", + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), + Name = "SparkDesk", + Order = 1, + Type = "SparkDesk", + Url = "", + UsedQuota = 0L + }); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Model.Aggregates.ModelLogger", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiKey") + .IsRequired() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("ComplementCount") + .HasColumnType("integer"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("FastModelId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Model") + .IsRequired() + .HasColumnType("text"); + + b.Property("PromptCount") + .HasColumnType("integer"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ApiKey"); + + b.HasIndex("ApplicationId"); + + b.HasIndex("CreationTime"); + + b.HasIndex("FastModelId"); + + b.HasIndex("Type"); + + b.HasIndex("UserId"); + + b.ToTable("wiki-model-logger", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Storage.Aggregates.FileStorage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("FullName") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsCompression") + .HasColumnType("boolean"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("wiki-file-storages", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Users.Aggregates.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Avatar") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("IsDisable") + .HasColumnType("boolean"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Password") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Phone") + .IsRequired() + .HasColumnType("text"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("wiki-users", (string)null); + + b.HasData( + new + { + Id = new Guid("3a2370db-a80f-4bb4-86ba-37c71759c3c5"), + Account = "admin", + Avatar = "https://blog-simple.oss-cn-shenzhen.aliyuncs.com/Avatar.jpg", + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3675), + Email = "239573049@qq.com", + IsDeleted = false, + IsDisable = false, + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3678), + Name = "admin", + Password = "58de2614378ee2067350a70bb789abd5", + Phone = "13049809673", + Role = 2, + Salt = "98d6415225ce4820a571359bd5192b91" + }); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Wikis.Aggregates.Wiki", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("uuid"); + + b.Property("EmbeddingModel") + .IsRequired() + .HasColumnType("text"); + + b.Property("Icon") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("Model") + .IsRequired() + .HasColumnType("text"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.HasKey("Id"); + + b.HasIndex("Name"); + + b.ToTable("wiki-wikis", (string)null); + }); + + modelBuilder.Entity("FastWiki.Service.Domain.Wikis.Aggregates.WikiDetail", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Creator") + .HasColumnType("bigint"); + + b.Property("DataCount") + .HasColumnType("integer"); + + b.Property("FileId") + .HasColumnType("bigint"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("MaxTokensPerLine") + .HasColumnType("integer"); + + b.Property("MaxTokensPerParagraph") + .HasColumnType("integer"); + + b.Property("Mode") + .HasColumnType("integer"); + + b.Property("ModificationTime") + .HasColumnType("timestamp without time zone"); + + b.Property("Modifier") + .HasColumnType("bigint"); + + b.Property("OverlappingTokens") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("QAPromptTemplate") + .HasColumnType("text"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("TrainingPattern") + .HasColumnType("integer"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("WikiId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("wiki-wiki-details", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.cs b/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.cs new file mode 100644 index 00000000..53c88464 --- /dev/null +++ b/src/Service/FastWiki.Service/Migrations/20240416051957_UpdateDetaiis.cs @@ -0,0 +1,144 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +#pragma warning disable CA1814 // Prefer jagged arrays over multidimensional + +namespace FastWiki.Service.Migrations +{ + /// + public partial class UpdateDetaiis : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DeleteData( + table: "wiki-fast-models", + keyColumn: "Id", + keyValue: "d114f3219d8c4398be2b66af4edbe85e"); + + migrationBuilder.DeleteData( + table: "wiki-fast-models", + keyColumn: "Id", + keyValue: "ee2e461ab124432485c4688ca569e174"); + + migrationBuilder.DeleteData( + table: "wiki-users", + keyColumn: "Id", + keyValue: new Guid("c2e43fc3-d870-48ea-a7e2-62586ee6b7ca")); + + migrationBuilder.AddColumn( + name: "MaxTokensPerLine", + table: "wiki-wiki-details", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "MaxTokensPerParagraph", + table: "wiki-wiki-details", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "Mode", + table: "wiki-wiki-details", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "OverlappingTokens", + table: "wiki-wiki-details", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "QAPromptTemplate", + table: "wiki-wiki-details", + type: "text", + nullable: true); + + migrationBuilder.AddColumn( + name: "TrainingPattern", + table: "wiki-wiki-details", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.InsertData( + table: "wiki-fast-models", + columns: new[] { "Id", "ApiKey", "CreationTime", "Creator", "Description", "Enable", "IsDeleted", "Models", "ModificationTime", "Modifier", "Name", "Order", "TestTime", "Type", "Url", "UsedQuota" }, + values: new object[,] + { + { "0b01f0fddb63483e8174fba06522108a", "", new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), null, "星火大模型", true, false, "[\"SparkDesk-v3.5\",\"SparkDesk-v3.1\",\"SparkDesk-v1.5\",\"SparkDesk-v2.1\"]", new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), null, "SparkDesk", 1, null, "SparkDesk", "", 0L }, + { "d28cb989669c46fba386666e84a2c72e", "", new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5311), null, "OpenAI", true, false, "[\"gpt-3.5-turbo\",\"gpt-3.5-turbo-0125\",\"gpt-3.5-turbo-1106\",\"gpt-3.5-turbo-16k\",\"gpt-3.5-turbo-0613\",\"gpt-3.5-turbo-16k-0613\",\"gpt-4-0125-preview\",\"gpt-4-turbo-preview\",\"gpt-4-1106-preview\",\"gpt-4-vision-preview\",\"gpt-4-1106-vision-preview\",\"gpt-4\",\"gpt-4-0613\",\"gpt-4-32k\",\"gpt-4-32k-0613\"]", new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5312), null, "OpenAI", 1, null, "OpenAI", "http://ai-api.token-ai.cn/", 0L } + }); + + migrationBuilder.InsertData( + table: "wiki-users", + columns: new[] { "Id", "Account", "Avatar", "CreationTime", "Creator", "Email", "IsDeleted", "IsDisable", "ModificationTime", "Modifier", "Name", "Password", "Phone", "Role", "Salt" }, + values: new object[] { new Guid("3a2370db-a80f-4bb4-86ba-37c71759c3c5"), "admin", "https://blog-simple.oss-cn-shenzhen.aliyuncs.com/Avatar.jpg", new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3675), null, "239573049@qq.com", false, false, new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3678), null, "admin", "58de2614378ee2067350a70bb789abd5", "13049809673", 2, "98d6415225ce4820a571359bd5192b91" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DeleteData( + table: "wiki-fast-models", + keyColumn: "Id", + keyValue: "0b01f0fddb63483e8174fba06522108a"); + + migrationBuilder.DeleteData( + table: "wiki-fast-models", + keyColumn: "Id", + keyValue: "d28cb989669c46fba386666e84a2c72e"); + + migrationBuilder.DeleteData( + table: "wiki-users", + keyColumn: "Id", + keyValue: new Guid("3a2370db-a80f-4bb4-86ba-37c71759c3c5")); + + migrationBuilder.DropColumn( + name: "MaxTokensPerLine", + table: "wiki-wiki-details"); + + migrationBuilder.DropColumn( + name: "MaxTokensPerParagraph", + table: "wiki-wiki-details"); + + migrationBuilder.DropColumn( + name: "Mode", + table: "wiki-wiki-details"); + + migrationBuilder.DropColumn( + name: "OverlappingTokens", + table: "wiki-wiki-details"); + + migrationBuilder.DropColumn( + name: "QAPromptTemplate", + table: "wiki-wiki-details"); + + migrationBuilder.DropColumn( + name: "TrainingPattern", + table: "wiki-wiki-details"); + + migrationBuilder.InsertData( + table: "wiki-fast-models", + columns: new[] { "Id", "ApiKey", "CreationTime", "Creator", "Description", "Enable", "IsDeleted", "Models", "ModificationTime", "Modifier", "Name", "Order", "TestTime", "Type", "Url", "UsedQuota" }, + values: new object[,] + { + { "d114f3219d8c4398be2b66af4edbe85e", "", new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6618), null, "星火大模型", true, false, "[\"SparkDesk-v3.5\",\"SparkDesk-v3.1\",\"SparkDesk-v1.5\",\"SparkDesk-v2.1\"]", new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6619), null, "SparkDesk", 1, null, "SparkDesk", "", 0L }, + { "ee2e461ab124432485c4688ca569e174", "", new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6603), null, "OpenAI", true, false, "[\"gpt-3.5-turbo\",\"gpt-3.5-turbo-0125\",\"gpt-3.5-turbo-1106\",\"gpt-3.5-turbo-16k\",\"gpt-3.5-turbo-0613\",\"gpt-3.5-turbo-16k-0613\",\"gpt-4-0125-preview\",\"gpt-4-turbo-preview\",\"gpt-4-1106-preview\",\"gpt-4-vision-preview\",\"gpt-4-1106-vision-preview\",\"gpt-4\",\"gpt-4-0613\",\"gpt-4-32k\",\"gpt-4-32k-0613\"]", new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6604), null, "OpenAI", 1, null, "OpenAI", "https://api.openai.com/", 0L } + }); + + migrationBuilder.InsertData( + table: "wiki-users", + columns: new[] { "Id", "Account", "Avatar", "CreationTime", "Creator", "Email", "IsDeleted", "IsDisable", "ModificationTime", "Modifier", "Name", "Password", "Phone", "Role", "Salt" }, + values: new object[] { new Guid("c2e43fc3-d870-48ea-a7e2-62586ee6b7ca"), "admin", "https://blog-simple.oss-cn-shenzhen.aliyuncs.com/Avatar.jpg", new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(5260), null, "239573049@qq.com", false, false, new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(5262), null, "admin", "949de09c9cac12c3f5ceb79f9c49fe3b", "13049809673", 2, "524fcf01d6bd4b1fb9289d3e41ec0044" }); + } + } +} diff --git a/src/Service/FastWiki.Service/Migrations/WikiDbContextModelSnapshot.cs b/src/Service/FastWiki.Service/Migrations/WikiDbContextModelSnapshot.cs index 0bd0bc06..dbaeed9d 100644 --- a/src/Service/FastWiki.Service/Migrations/WikiDbContextModelSnapshot.cs +++ b/src/Service/FastWiki.Service/Migrations/WikiDbContextModelSnapshot.cs @@ -380,30 +380,30 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasData( new { - Id = "ee2e461ab124432485c4688ca569e174", + Id = "d28cb989669c46fba386666e84a2c72e", ApiKey = "", - CreationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6603), + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5311), Description = "OpenAI", Enable = true, IsDeleted = false, Models = "[\"gpt-3.5-turbo\",\"gpt-3.5-turbo-0125\",\"gpt-3.5-turbo-1106\",\"gpt-3.5-turbo-16k\",\"gpt-3.5-turbo-0613\",\"gpt-3.5-turbo-16k-0613\",\"gpt-4-0125-preview\",\"gpt-4-turbo-preview\",\"gpt-4-1106-preview\",\"gpt-4-vision-preview\",\"gpt-4-1106-vision-preview\",\"gpt-4\",\"gpt-4-0613\",\"gpt-4-32k\",\"gpt-4-32k-0613\"]", - ModificationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6604), + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5312), Name = "OpenAI", Order = 1, Type = "OpenAI", - Url = "https://api.openai.com/", + Url = "http://ai-api.token-ai.cn/", UsedQuota = 0L }, new { - Id = "d114f3219d8c4398be2b66af4edbe85e", + Id = "0b01f0fddb63483e8174fba06522108a", ApiKey = "", - CreationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6618), + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), Description = "星火大模型", Enable = true, IsDeleted = false, Models = "[\"SparkDesk-v3.5\",\"SparkDesk-v3.1\",\"SparkDesk-v1.5\",\"SparkDesk-v2.1\"]", - ModificationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(6619), + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(5326), Name = "SparkDesk", Order = 1, Type = "SparkDesk", @@ -584,19 +584,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasData( new { - Id = new Guid("c2e43fc3-d870-48ea-a7e2-62586ee6b7ca"), + Id = new Guid("3a2370db-a80f-4bb4-86ba-37c71759c3c5"), Account = "admin", Avatar = "https://blog-simple.oss-cn-shenzhen.aliyuncs.com/Avatar.jpg", - CreationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(5260), + CreationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3675), Email = "239573049@qq.com", IsDeleted = false, IsDisable = false, - ModificationTime = new DateTime(2024, 4, 5, 10, 24, 24, 310, DateTimeKind.Utc).AddTicks(5262), + ModificationTime = new DateTime(2024, 4, 16, 5, 19, 56, 870, DateTimeKind.Utc).AddTicks(3678), Name = "admin", - Password = "949de09c9cac12c3f5ceb79f9c49fe3b", + Password = "58de2614378ee2067350a70bb789abd5", Phone = "13049809673", Role = 2, - Salt = "524fcf01d6bd4b1fb9289d3e41ec0044" + Salt = "98d6415225ce4820a571359bd5192b91" }); }); @@ -672,20 +672,38 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(100) .HasColumnType("character varying(100)"); + b.Property("MaxTokensPerLine") + .HasColumnType("integer"); + + b.Property("MaxTokensPerParagraph") + .HasColumnType("integer"); + + b.Property("Mode") + .HasColumnType("integer"); + b.Property("ModificationTime") .HasColumnType("timestamp without time zone"); b.Property("Modifier") .HasColumnType("bigint"); + b.Property("OverlappingTokens") + .HasColumnType("integer"); + b.Property("Path") .IsRequired() .HasMaxLength(200) .HasColumnType("character varying(200)"); + b.Property("QAPromptTemplate") + .HasColumnType("text"); + b.Property("State") .HasColumnType("integer"); + b.Property("TrainingPattern") + .HasColumnType("integer"); + b.Property("Type") .IsRequired() .HasMaxLength(100) diff --git a/src/Service/FastWiki.Service/Program.cs b/src/Service/FastWiki.Service/Program.cs index fec66106..9c8a6285 100644 --- a/src/Service/FastWiki.Service/Program.cs +++ b/src/Service/FastWiki.Service/Program.cs @@ -41,7 +41,8 @@ .AddKeyedSingleton(MetaGLMOptions.ServiceName) .AddKeyedSingleton(SparkDeskOptions.ServiceName) .AddKeyedSingleton(ClaudiaOptions.ServiceName) - .AddKeyedSingleton(QiansailOptions.ServiceName); + .AddKeyedSingleton(QiansailOptions.ServiceName) + .AddSingleton(); builder.Services.Configure(builder.Configuration.GetSection("IpRateLimit")); builder.Services.AddSingleton(); @@ -166,7 +167,8 @@ app.UseAuthentication(); app.UseAuthorization(); -app.MapPost("/v1/chat/completions", OpenAIService.Completions) +app.MapPost("/v1/chat/completions", + async (OpenAIService openAiService, HttpContext context) => await openAiService.Completions(context)) .WithTags("OpenAI") .WithGroupName("OpenAI") .WithDescription("OpenAI Completions") diff --git a/src/Service/FastWiki.Service/Service/OpenAIService.cs b/src/Service/FastWiki.Service/Service/OpenAIService.cs index 4ae05f97..261a0c6d 100644 --- a/src/Service/FastWiki.Service/Service/OpenAIService.cs +++ b/src/Service/FastWiki.Service/Service/OpenAIService.cs @@ -20,9 +20,9 @@ namespace FastWiki.Service.Service; -public static class OpenAIService +public class OpenAIService { - public static async Task Completions(HttpContext context) + public async Task Completions(HttpContext context) { using var stream = new StreamReader(context.Request.Body); @@ -500,7 +500,7 @@ await context.WriteOpenAiResultAsync(message, module.model, requestId, } } - public static async IAsyncEnumerable QAAsync(string prompt, string value, string model, string apiKey, + public async IAsyncEnumerable QAAsync(string prompt, string value, string model, string apiKey, string url, WikiMemoryService memoryService) { diff --git a/src/Service/FastWiki.Service/Service/WikiMemoryService.cs b/src/Service/FastWiki.Service/Service/WikiMemoryService.cs index 3b4421f4..dcbf0d16 100644 --- a/src/Service/FastWiki.Service/Service/WikiMemoryService.cs +++ b/src/Service/FastWiki.Service/Service/WikiMemoryService.cs @@ -65,6 +65,8 @@ public MemoryServerless CreateMemoryServerless(SearchClientConfig searchClientCo : OpenAIOption.EmbeddingToken, EmbeddingModel = string.IsNullOrEmpty(embeddingModel) ? OpenAIOption.EmbeddingModel : embeddingModel, }, null, false, new HttpClient(HttpClientHandler)) + .AddSingleton(new OpenAIService()) + .AddSingleton(new WikiMemoryService()) .Build(); return memory; diff --git a/web/src/models/index.d.ts b/web/src/models/index.d.ts index 8b19dd2b..39d4a17a 100644 --- a/web/src/models/index.d.ts +++ b/web/src/models/index.d.ts @@ -10,7 +10,7 @@ export interface ChatApplicationDto { parameter: { [key: string]: string; }; opener: string; wikiIds: number[]; - functionIds:number[]; + functionIds: number[]; referenceUpperLimit: number; noReplyFoundTemplate: string | null; showSourceFile: boolean; @@ -57,7 +57,7 @@ export interface ChatShareDto { availableQuantity: number; apiKey: string; usedToken: number; - + } export interface CompletionsDto { content: string; @@ -166,6 +166,7 @@ export interface CreateWikiDetailWebPageInput { overlappingTokens: number; mode: ProcessMode; trainingPattern: TrainingPattern; + qAPromptTemplate: string; } export interface PaginatedListBase { @@ -237,6 +238,7 @@ export interface CreateWikiDetailWebPageInput { overlappingTokens: number; mode: ProcessMode; trainingPattern: TrainingPattern; + qAPromptTemplate: string; } export interface FastWikiFunctionCallDto { diff --git a/web/src/pages/wiki-detail/features/UploadWikiFile.tsx b/web/src/pages/wiki-detail/features/UploadWikiFile.tsx index ca34f5d5..521d54c1 100644 --- a/web/src/pages/wiki-detail/features/UploadWikiFile.tsx +++ b/web/src/pages/wiki-detail/features/UploadWikiFile.tsx @@ -7,6 +7,7 @@ import { bytesToSize } from '../../../utils/stringHelper'; import { ProcessMode, TrainingPattern } from '../../../models/index.d'; import { UploadFile as FileService } from '../../../services/StorageService'; import { CreateWikiDetails } from '../../../services/WikiService'; +import { TextArea } from '@lobehub/ui'; const FileItem = styled.div` @@ -41,6 +42,20 @@ export default function UploadWikiFile({ id, onChagePath }: IUploadWikiFileProps const [maxTokensPerParagraph, setMaxTokensPerParagraph] = useState(1000); // 每个段落标记的最大数量。当对文档进行分区时，每个分区通常包含一个段落。 const [maxTokensPerLine, setMaxTokensPerLine] = useState(300); // 每行，也就是每个句子的最大标记数。当分割一个文本块时，文本将被分割成句子，然后被分组成段落。注意，这适用于任何文本格式，包括表格、代码、聊天记录、日志文件等。 const [overlappingTokens, setOverlappingTokens] = useState(100); // 重叠标记数。当对文档进行分区时，每个分区的开始和结束部分将重叠。这有助于确保模型在分区之间保持上下文一致性。 + const [qAPromptTemplate, setQAPromptTemplate] = useState(` + 我会给你一段文本，学习它们，并整理学习成果，要求为： + 1. 提出最多 20 个问题。 + 2. 给出每个问题的答案。 + 3. 答案要详细完整，答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 markdown 元素。 + 4. 按格式返回多个问题和答案: + + Q1: 问题。 + A1: 答案。 + Q2: + A2: + …… + + 我的文本："""{{$input}}"""`); // QA问答模板 const props: UploadProps = { name: 'file', multiple: true, @@ -121,6 +136,7 @@ export default function UploadWikiFile({ id, onChagePath }: IUploadWikiFileProps maxTokensPerParagraph: maxTokensPerParagraph, maxTokensPerLine: maxTokensPerLine, overlappingTokens: overlappingTokens, + qAPromptTemplate: qAPromptTemplate, mode: processMode, trainingPattern: trainingPattern }) @@ -287,9 +303,13 @@ export default function UploadWikiFile({ id, onChagePath }: IUploadWikiFileProps } { - trainingPattern === TrainingPattern.QA &&
- 暂时不支持 -
+ trainingPattern === TrainingPattern.QA && <> + QA问答模板： + + } { diff --git a/web/src/pages/wiki-detail/features/UploadWikiWeb.tsx b/web/src/pages/wiki-detail/features/UploadWikiWeb.tsx index 013a86f2..00e7f8ee 100644 --- a/web/src/pages/wiki-detail/features/UploadWikiWeb.tsx +++ b/web/src/pages/wiki-detail/features/UploadWikiWeb.tsx @@ -4,6 +4,7 @@ import { useState } from 'react'; import styled from 'styled-components'; import { CreateWikiDetailWebPageInput, ProcessMode, TrainingPattern } from '../../../models/index.d'; import { CreateWikiDetailWebPage } from '../../../services/WikiService'; +import { TextArea } from '@lobehub/ui'; const FileItem = styled.div` transition: border-color 0.3s linear; @@ -34,6 +35,20 @@ export default function UploadWikiWeb({ id, onChagePath }: IUploadWikiFileProps) const [maxTokensPerLine, setMaxTokensPerLine] = useState(300); // 每行，也就是每个句子的最大标记数。当分割一个文本块时，文本将被分割成句子，然后被分组成段落。注意，这适用于任何文本格式，包括表格、代码、聊天记录、日志文件等。 const [overlappingTokens, setOverlappingTokens] = useState(100); // 重叠标记数。当对文档进行分区时，每个分区的开始和结束部分将重叠。这有助于确保模型在分区之间保持上下文一致性。 const [value, setValue] = useState(''); + const [qAPromptTemplate, setQAPromptTemplate] = useState(` + 我会给你一段文本，学习它们，并整理学习成果，要求为： + 1. 提出最多 20 个问题。 + 2. 给出每个问题的答案。 + 3. 答案要详细完整，答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 markdown 元素。 + 4. 按格式返回多个问题和答案: + + Q1: 问题。 + A1: 答案。 + Q2: + A2: + …… + + 我的文本："""{{$input}}"""`); // QA问答模板 async function upload(){ for (let i = 0; i < webs.length; i++) { @@ -45,6 +60,7 @@ export default function UploadWikiWeb({ id, onChagePath }: IUploadWikiFileProps) maxTokensPerParagraph: maxTokensPerParagraph, maxTokensPerLine: maxTokensPerLine, overlappingTokens: overlappingTokens, + qAPromptTemplate: qAPromptTemplate, mode: processMode, state: '', path: item, @@ -210,9 +226,13 @@ export default function UploadWikiWeb({ id, onChagePath }: IUploadWikiFileProps) } { - trainingPattern === TrainingPattern.QA &&
- 暂时不支持 -
+ trainingPattern === TrainingPattern.QA && <> + QA问答模板： + + }