feat: store processor used during encoding + retry hw conversions usi…

…ng cpu on conversion fault (#46) * feat: track encode processor type + retry conversions without hw acceleration on conversion fault * feat: show processor type used in encode operation widget * chore: update nuget packages
PHILLIPS71 · May 18, 2024 · dae34ec · dae34ec
1 parent c8c8a5c
commit dae34ec
Show file tree

Hide file tree

Showing 25 changed files with 192 additions and 185 deletions.
diff --git a/app/src/components/interfaces/encode/chips/EncodeDuration.tsx b/app/src/components/interfaces/encode/chips/EncodeDuration.tsx
@@ -39,7 +39,7 @@ const EncodeDuration: React.FC<EncodeDurationChipProps> = ({ $key }) => {
   }, [data.cancelled_at, data.completed_at, data.failed_at, data.status])
 
   return (
-    <Chip color="indigo" title={dayjs(date).format('L LT')}>
+    <Chip color="indigo" title={dayjs(date).format('L LTS')}>
       {dayjs.duration(dayjs(date).diff(data.created_at)).format('H[h] m[m] s[s]')}
     </Chip>
   )

diff --git a/app/src/components/interfaces/encode/chips/EncodeStatus.tsx b/app/src/components/interfaces/encode/chips/EncodeStatus.tsx
@@ -41,13 +41,13 @@ const EncodeStatus: React.FC<EncodeStatusChipProps> = ({ $key }) => {
   const title = React.useMemo<string | undefined>(() => {
     switch (data.status) {
       case 'COMPLETED':
-        return dayjs(data.completed_at).format('L LT')
+        return dayjs(data.completed_at).format('L LTS')
 
       case 'CANCELLED':
-        return dayjs(data.cancelled_at).format('L LT')
+        return dayjs(data.cancelled_at).format('L LTS')
 
       case 'FAILED':
-        return dayjs(data.failed_at).format('L LT')
+        return dayjs(data.failed_at).format('L LTS')
 
       default:
         return undefined

diff --git a/app/src/components/interfaces/encode/widgets/EncodeOperationWidget.tsx b/app/src/components/interfaces/encode/widgets/EncodeOperationWidget.tsx
@@ -1,7 +1,9 @@
 import type { EncodeOperationWidgetFragment$key } from '@/__generated__/EncodeOperationWidgetFragment.graphql'
 
 import { Chip, Table, Typography } from '@giantnodes/react'
+import { IconCpu } from '@tabler/icons-react'
 import dayjs from 'dayjs'
+import React from 'react'
 import { graphql, useFragment } from 'react-relay'
 
 import { EncodePercent, EncodeSpeed, EncodeStatus } from '@/components/interfaces/encode'
@@ -13,6 +15,7 @@ const FRAGMENT = graphql`
     machine {
       name
       user_name
+      processor_type
     }
     ...EncodePercentFragment
     ...EncodeStatusFragment
@@ -65,6 +68,9 @@ const EncodeOperationWidget: React.FC<EncodeOperationWidgetProps> = ({ $key }) =
               <>
                 <Chip color="info">{data.machine?.name}</Chip>
                 <Chip color="info">{data.machine?.user_name}</Chip>
+                <Chip color="info">
+                  {data.machine.processor_type === 'CPU' ? <IconCpu size={16} /> : data.machine.processor_type}
+                </Chip>
               </>
             )}
           </Table.Cell>
@@ -86,10 +92,7 @@ const EncodeOperationWidget: React.FC<EncodeOperationWidgetProps> = ({ $key }) =
             </Table.Cell>
             <Table.Cell className="text-right">
               <Typography.Text>
-                <Chip
-                  className="text-pink-500 bg-pink-500/20 border-pink-500"
-                  title={dayjs(data.updated_at).format('L LT')}
-                >
+                <Chip color="pink" title={dayjs(data.updated_at).format('L LTS')}>
                   {dayjs(data.updated_at).fromNow()}
                 </Chip>
               </Typography.Text>

diff --git a/...board/src/Application.Components/Encodes/Sagas/Activities/EncodeOperationBuiltActivity.cs b/...board/src/Application.Components/Encodes/Sagas/Activities/EncodeOperationBuiltActivity.cs
@@ -1,5 +1,6 @@
 using Giantnodes.Infrastructure.Uow.Services;
 using Giantnodes.Service.Dashboard.Domain.Aggregates.Encodes.Repositories;
+using Giantnodes.Service.Dashboard.Domain.Shared.Enums;
 using Giantnodes.Service.Dashboard.Domain.Values;
 using Giantnodes.Service.Dashboard.Persistence.Sagas;
 using Giantnodes.Service.Encoder.Application.Contracts.Encoding.Events;
@@ -35,7 +36,11 @@ public async Task Execute(
         using var uow = await _uow.BeginAsync(context.CancellationToken);
         var encode = await _repository.SingleAsync(x => x.Id == context.Saga.EncodeId);
 
-        var machine = new Machine(context.Message.MachineName, context.Message.MachineUserName);
+        var machine = new Machine(
+            context.Message.MachineName,
+            context.Message.MachineUserName,
+            context.Message.UsingHardwareAcceleration ? ProcessorType.Gpu : ProcessorType.Cpu);
+
         encode.SetFfmpegConversion(machine, context.Message.FFmpegCommand);
 
         await uow.CommitAsync(context.CancellationToken);

diff --git a/...d/src/Application.Components/Encodes/Sagas/Activities/EncodeOperationHeartbeatActivity.cs b/...d/src/Application.Components/Encodes/Sagas/Activities/EncodeOperationHeartbeatActivity.cs
diff --git a/...c/Application.Components/Encodes/Sagas/Activities/EncodeOperationOutputtedDataActivity.cs b/...c/Application.Components/Encodes/Sagas/Activities/EncodeOperationOutputtedDataActivity.cs
@@ -1,5 +1,6 @@
 using Giantnodes.Infrastructure.Uow.Services;
 using Giantnodes.Service.Dashboard.Domain.Aggregates.Encodes.Repositories;
+using Giantnodes.Service.Dashboard.Domain.Aggregates.Encodes.Values;
 using Giantnodes.Service.Dashboard.Persistence.Sagas;
 using Giantnodes.Service.Encoder.Application.Contracts.Encoding.Events;
 using MassTransit;
@@ -34,7 +35,11 @@ public async Task Execute(
         using var uow = await _uow.BeginAsync(context.CancellationToken);
         var encode = await _repository.SingleAsync(x => x.Id == context.Saga.EncodeId);
 
-        encode.AppendOutputLog(context.Message.Data);
+        var speed = context.Message.Speed;
+        if (speed.HasValue)
+            encode.SetSpeed(new EncodeSpeed(speed.Value.Frames, speed.Value.Bitrate, speed.Value.Scale));
+
+        encode.AppendOutputLog(context.Message.Output);
 
         await uow.CommitAsync(context.CancellationToken);
         await next.Execute(context);

diff --git a/src/Service.Dashboard/src/Application.Components/Encodes/Sagas/EncodeStateMachine.cs b/src/Service.Dashboard/src/Application.Components/Encodes/Sagas/EncodeStateMachine.cs
@@ -19,7 +19,6 @@ public EncodeStateMachine()
         Event(() => Created);
         Event(() => Started);
         Event(() => Built);
-        Event(() => Heartbeat);
         Event(() => Progressed);
         Event(() => Outputted);
         Event(() => Completed);
@@ -52,11 +51,13 @@ public EncodeStateMachine()
                 .Activity(context => context.OfType<EncodeOperationStartedActivity>())
                 .TransitionTo(Processing));
 
-        During(Processing,
+        During(Queued, Processing, 
             When(Built)
-                .Activity(context => context.OfType<EncodeOperationBuiltActivity>()),
-            When(Heartbeat)
-                .Activity(context => context.OfType<EncodeOperationHeartbeatActivity>()),
+                .Activity(context => context.OfType<EncodeOperationBuiltActivity>()));
+
+        During(Processing,
+            When(Outputted)
+                .Activity(context => context.OfType<EncodeOperationOutputtedDataActivity>()),
             When(Progressed)
                 .Activity(context => context.OfType<EncodeOperationProgressedActivity>()),
             When(Completed)
@@ -74,8 +75,6 @@ public EncodeStateMachine()
                 .Finalize());
 
         DuringAny(
-            When(Outputted)
-                .Activity(context => context.OfType<EncodeOperationOutputtedDataActivity>()),
             When(Failed)
                 .Then(context => context.Saga.FailedReason = context.Message.Exceptions.Message)
                 .Activity(context => context.OfInstanceType<EncodeFailedActivity>())
@@ -96,7 +95,6 @@ public EncodeStateMachine()
     public required Event<FileProbedEvent> FileProbed { get; set; }
     public required Event<EncodeOperationStartedEvent> Started { get; set; }
     public required Event<EncodeOperationEncodeBuiltEvent> Built { get; set; }
-    public required Event<EncodeOperationEncodeHeartbeatEvent> Heartbeat { get; set; }
     public required Event<EncodeOperationEncodeProgressedEvent> Progressed { get; set; }
     public required Event<EncodeOperationOutputtedEvent> Outputted { get; set; }
     public required Event<EncodeOperationCompletedEvent> Completed { get; set; }

diff --git a/...ervice.Dashboard/src/Application.Components/Encodes/Sagas/EncodeStateMachineDefinition.cs b/...ervice.Dashboard/src/Application.Components/Encodes/Sagas/EncodeStateMachineDefinition.cs
@@ -1,21 +1,31 @@
+using Giantnodes.Service.Dashboard.Application.Contracts.Encodes.Events;
 using Giantnodes.Service.Dashboard.Persistence.Sagas;
 using Giantnodes.Service.Encoder.Application.Contracts.Encoding.Events;
+using Giantnodes.Service.Encoder.Application.Contracts.Probing.Events;
 using MassTransit;
 
 namespace Giantnodes.Service.Dashboard.Application.Components.Encodes.Sagas;
 
 public class EncodeStateMachineDefinition : SagaDefinition<EncodeSagaState>
 {
+    private const int ConcurrencyLimit = 25;
+
     protected override void ConfigureSaga(
         IReceiveEndpointConfigurator endpointConfigurator,
         ISagaConfigurator<EncodeSagaState> sagaConfigurator,
         IRegistrationContext context)
     {
         endpointConfigurator.UseMessageRetry(r => r.Interval(3, TimeSpan.FromSeconds(3)));
 
-        var partition = sagaConfigurator.CreatePartitioner(1);
-        endpointConfigurator.UsePartitioner<EncodeOperationEncodeHeartbeatEvent>(partition, p => p.Message.JobId);
-        endpointConfigurator.UsePartitioner<EncodeOperationEncodeProgressedEvent>(partition, p => p.Message.JobId);
-        endpointConfigurator.UsePartitioner<EncodeOperationOutputtedEvent>(partition, p => p.Message.JobId);
+        var partition = sagaConfigurator.CreatePartitioner(ConcurrencyLimit);
+        endpointConfigurator.UsePartitioner<EncodeCreatedEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<FileProbedEvent>(partition, p => p.Message.JobId);
+        endpointConfigurator.UsePartitioner<EncodeOperationStartedEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeOperationEncodeBuiltEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeOperationEncodeProgressedEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeOperationOutputtedEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeOperationCompletedEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeCancelledEvent>(partition, p => p.Message.CorrelationId);
+        endpointConfigurator.UsePartitioner<EncodeOperationFailedEvent>(partition, p => p.Message.CorrelationId);
     }
 }
diff --git a/...ard/src/Application.Components/Giantnodes.Service.Dashboard.Application.Components.csproj b/...ard/src/Application.Components/Giantnodes.Service.Dashboard.Application.Components.csproj
@@ -13,9 +13,9 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.0" />
-      <PackageReference Include="HotChocolate.Abstractions" Version="13.9.0" />
-      <PackageReference Include="MassTransit" Version="8.2.1" />
+      <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.1" />
+      <PackageReference Include="HotChocolate.Abstractions" Version="13.9.4" />
+      <PackageReference Include="MassTransit" Version="8.2.2" />
     </ItemGroup>
 
 </Project>
diff --git a/src/Service.Dashboard/src/Domain.Shared/Enums/ProcessorType.cs b/src/Service.Dashboard/src/Domain.Shared/Enums/ProcessorType.cs
@@ -0,0 +1,7 @@
+namespace Giantnodes.Service.Dashboard.Domain.Shared.Enums;
+
+public enum ProcessorType
+{
+    Cpu,
+    Gpu
+}
diff --git a/src/Service.Dashboard/src/Domain/Values/Machine.cs b/src/Service.Dashboard/src/Domain/Values/Machine.cs
@@ -1,4 +1,5 @@
 using Giantnodes.Infrastructure.Domain.Values;
+using Giantnodes.Service.Dashboard.Domain.Shared.Enums;
 
 namespace Giantnodes.Service.Dashboard.Domain.Values;
 
@@ -8,19 +9,23 @@ public class Machine : ValueObject
 
     public string UserName { get; init; }
 
+    public ProcessorType ProcessorType { get; init; }
+
     protected Machine()
     {
     }
 
-    public Machine(string name, string username)
+    public Machine(string name, string username, ProcessorType processor)
     {
         Name = name;
         UserName = username;
+        ProcessorType = processor;
     }
 
     protected override IEnumerable<object> GetEqualityComponents()
     {
         yield return Name;
         yield return UserName;
+        yield return ProcessorType;
     }
 }
diff --git a/src/Service.Dashboard/src/HttpApi/Giantnodes.Service.Dashboard.HttpApi.csproj b/src/Service.Dashboard/src/HttpApi/Giantnodes.Service.Dashboard.HttpApi.csproj
@@ -14,11 +14,11 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="HotChocolate.AspNetCore" Version="13.9.0" />
-        <PackageReference Include="HotChocolate.AspNetCore.CommandLine" Version="13.9.0" />
-        <PackageReference Include="HotChocolate.Data.EntityFramework" Version="13.9.0" />
-        <PackageReference Include="HotChocolate.Types.Analyzers" Version="13.9.0" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
+        <PackageReference Include="HotChocolate.AspNetCore" Version="13.9.4" />
+        <PackageReference Include="HotChocolate.AspNetCore.CommandLine" Version="13.9.4" />
+        <PackageReference Include="HotChocolate.Data.EntityFramework" Version="13.9.4" />
+        <PackageReference Include="HotChocolate.Types.Analyzers" Version="13.9.4" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.5">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>

diff --git a/src/Service.Dashboard/src/HttpApi/Types/MachineType.cs b/src/Service.Dashboard/src/HttpApi/Types/MachineType.cs
@@ -11,5 +11,8 @@ protected override void Configure(IObjectTypeDescriptor<Machine> descriptor)
 
         descriptor
             .Field(x => x.UserName);
+
+        descriptor
+            .Field(x => x.ProcessorType);
     }
 }
diff --git a/src/Service.Dashboard/src/Persistence/Configurations/EncodeConfiguration.cs b/src/Service.Dashboard/src/Persistence/Configurations/EncodeConfiguration.cs
@@ -20,6 +20,11 @@ public void Configure(EntityTypeBuilder<Encode> builder)
             .OwnsOne(p => p.Speed);
 
         builder
-            .OwnsOne(p => p.Machine);
+            .OwnsOne(p => p.Machine, machine =>
+            {
+                machine
+                    .Property(p => p.ProcessorType)
+                    .HasConversion<string>();
+            });
     }
 }
diff --git a/src/Service.Dashboard/src/Persistence/Giantnodes.Service.Dashboard.Persistence.csproj b/src/Service.Dashboard/src/Persistence/Giantnodes.Service.Dashboard.Persistence.csproj
@@ -9,14 +9,14 @@
     <ItemGroup>
         <PackageReference Include="EFCore.NamingConventions" Version="8.0.3" />
         <PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />
-        <PackageReference Include="MassTransit.EntityFrameworkCore" Version="8.2.1" />
-        <PackageReference Include="MassTransit.SqlTransport.PostgreSQL" Version="8.2.1" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.4">
+        <PackageReference Include="MassTransit.EntityFrameworkCore" Version="8.2.2" />
+        <PackageReference Include="MassTransit.SqlTransport.PostgreSQL" Version="8.2.2" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.5" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.5">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
+        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/...mponents.Tests.Unit/Giantnodes.Service.Dashboard.Application.Components.Tests.Unit.csproj b/...mponents.Tests.Unit/Giantnodes.Service.Dashboard.Application.Components.Tests.Unit.csproj
@@ -10,17 +10,17 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="MassTransit.TestFramework" Version="8.2.1" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.4" />
+        <PackageReference Include="MassTransit.TestFramework" Version="8.2.2" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.5" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
         <PackageReference Include="coverlet.collector" Version="6.0.2">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
         <PackageReference Include="NSubstitute" Version="5.1.0" />
-        <PackageReference Include="xunit" Version="2.7.1" />
-        <PackageReference Include="xunit.analyzers" Version="1.12.0" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.5.8">
+        <PackageReference Include="xunit" Version="2.8.0" />
+        <PackageReference Include="xunit.analyzers" Version="1.13.0" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>

diff --git a/...Service.Dashboard.Domain.Tests.Unit/Giantnodes.Service.Dashboard.Domain.Tests.Unit.csproj b/...Service.Dashboard.Domain.Tests.Unit/Giantnodes.Service.Dashboard.Domain.Tests.Unit.csproj
@@ -15,9 +15,9 @@
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="xunit" Version="2.7.1" />
-        <PackageReference Include="xunit.analyzers" Version="1.12.0" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.5.8">
+        <PackageReference Include="xunit" Version="2.8.0" />
+        <PackageReference Include="xunit.analyzers" Version="1.13.0" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>