Skip to content

Commit

Permalink
Update to handle SMB enable/disable and move templating to helm (#7)
Browse files Browse the repository at this point in the history
* updating paths

* adding spacefx chart utils

* adding postStart

* adding helm template  util

* adding resourceLimits to pods

* adding appSettings and appSettingsMount

* Adding appsettings

* updating for volume templates

* adding persistentvolumes and persistentvolumeclaims

* updating for persistentvolumeclaim

* updating for integration test

* updating to include service account

* updating to reuse persistentvolumeclaim when available

* updating devcontainer to prod feature
  • Loading branch information
bigtallcampbell authored Jul 19, 2024
1 parent 98e0700 commit 9fbcf33
Show file tree
Hide file tree
Showing 24 changed files with 729 additions and 492 deletions.
21 changes: 17 additions & 4 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
"runArgs": [
"--name=platform-deployment"
],
"workspaceFolder": "/workspaces/platform-deployment",
"workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/platform-deployment,type=bind,consistency=cached",
"workspaceFolder": "/workspace/platform-deployment",
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace/platform-deployment,type=bind,consistency=cached",
"hostRequirements": {
"cpus": 8,
"memory": "8gb"
Expand All @@ -17,7 +17,8 @@
"app_name": "platform-deployment",
"app_type": "sdk-service",
"addl_debug_shim_suffixes": "client",
"debug_shim_post_yaml_file": "/workspaces/platform-deployment/.vscode/debugShim-svcAcct-clusterAdmin.yaml"
"debug_shim_post_yaml_file": "/workspace/platform-deployment/.vscode/debugShim-svcAcct-clusterAdmin.yaml",
"smb_enabled_in_cluster": "true"
}
},
"customizations": {
Expand Down Expand Up @@ -112,10 +113,22 @@
"contents": "read",
"packages": "read"
}
},
"microsoft/azure-orbital-space-sdk-data-generators": {
"permissions": {
"contents": "read",
"packages": "read"
}
}
}
}
},
"remoteEnv": {
"KUBECONFIG": "/workspace/platform-deployment/.git/spacefx-dev/k3s.devcontainer.yaml"
},
"containerEnv": {
"KUBECONFIG": "/workspace/platform-deployment/.git/spacefx-dev/k3s.devcontainer.yaml"
},
"remoteUser": "root",
"postStartCommand": "regctl image export ghcr.io/dapr/samples/pubsub-csharp-subscriber:1.9.0 --name pubsub-csharp-subscriber > /workspaces/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar"
"postStartCommand": "/workspace/platform-deployment/.devcontainer/postStart.sh"
}
1 change: 1 addition & 0 deletions .devcontainer/postStart.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
regctl image export ghcr.io/dapr/samples/pubsub-csharp-subscriber:1.9.0 --name pubsub-csharp-subscriber > /workspace/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar
3 changes: 2 additions & 1 deletion .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
"args": [
"/spacefx-dev/debugShim-deploy.sh",
"--debug_shim",
"${DEBUG_SHIM_HOST}"
"${DEBUG_SHIM_HOST}",
"--disable_plugin_configs"
],
"presentation": {
"echo": true,
Expand Down
34 changes: 34 additions & 0 deletions src/MessageHandlers/MessageHandler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment;

public partial class MessageHandler<T> : Microsoft.Azure.SpaceFx.Core.IMessageHandler<T> where T : notnull {
private readonly ILogger<MessageHandler<T>> _logger;
public static EventHandler<T>? MessageReceivedEvent;
private readonly Microsoft.Azure.SpaceFx.Core.Services.PluginLoader _pluginLoader;
private readonly IServiceProvider _serviceProvider;
private readonly Core.Client _client;
private readonly Models.APP_CONFIG _appConfig;
private readonly PluginDelegates _pluginDelegates;

public MessageHandler(ILogger<MessageHandler<T>> logger, PluginDelegates pluginDelegates, Microsoft.Azure.SpaceFx.Core.Services.PluginLoader pluginLoader, IServiceProvider serviceProvider, Core.Client client) {
_logger = logger;
_pluginDelegates = pluginDelegates;
_pluginLoader = pluginLoader;
_serviceProvider = serviceProvider;
_client = client;

_appConfig = new Models.APP_CONFIG();
}

public void MessageReceived(T message, MessageFormats.Common.DirectToApp fullMessage) => Task.Run(() => {
using (var scope = _serviceProvider.CreateScope()) {
if (message == null || EqualityComparer<T>.Default.Equals(message, default)) {
_logger.LogInformation("Received empty message '{messageType}' from '{appId}'. Discarding message.", typeof(T).Name, fullMessage.SourceAppId);
return;
}
// This function is just a catch all for any messages that come in. They are not processed and no plugins are triggered for security reasons.
// We're catching all messages here to reduce the log warnings for OOTB messages that are flowing
}
});
}
26 changes: 3 additions & 23 deletions src/Models/AppConfig.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
using YamlDotNet.Serialization;

namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment;
public static class Models {
public static partial class Models {
public class APP_CONFIG : Core.APP_CONFIG {
[Flags]
[JsonConverter(typeof(JsonStringEnumConverter))]
Expand Down Expand Up @@ -48,22 +46,14 @@ public PLUG_IN() {
public string CONTAINER_REGISTRY { get; set; }
public string CONTAINER_REGISTRY_INTERNAL { get; set; }
public string SCHEDULE_IMPORT_DIRECTORY { get; set; }
public string DAPR_ANNOTATIONS { get; set; }
public string DEFAULT_LIMIT_MEMORY { get; set; }
public string DEFAULT_LIMIT_CPU { get; set; }
public string DEFAULT_REQUEST_MEMORY { get; set; }
public string DEFAULT_REQUEST_CPU { get; set; }
public string FILESERVER_APP_CRED_NAME { get; set; }
public string FILESERVER_CRED_NAME { get; set; }
public string FILESERVER_CRED_NAMESPACE { get; set; }
public string FILESERVER_PERSISTENT_VOLUMES { get; set; }
public string FILESERVER_PERSISTENT_VOLUMECLAIMS { get; set; }
public string FILESERVER_CLIENT_VOLUME_MOUNTS { get; set; }
public string FILESERVER_CLIENT_VOLUMES { get; set; }
public string PAYLOAD_APP_ANNOTATIONS { get; set; }
public string PAYLOAD_APP_CONFIG { get; set; }
public string PAYLOAD_APP_LABELS { get; set; }
public string PAYLOAD_APP_ENVIRONMENTVARIABLES { get; set; }
public bool FILESERVER_SMB_ENABLED { get; set; }
public TimeSpan DEFAULT_MAX_DURATION { get; set; }

public APP_CONFIG() : base() {
Expand All @@ -90,19 +80,9 @@ public APP_CONFIG() : base() {
FILESERVER_APP_CRED_NAME = Core.GetConfigSetting("fileserverappcredname").Result;
FILESERVER_CRED_NAME = Core.GetConfigSetting("fileservercredname").Result;
FILESERVER_CRED_NAMESPACE = Core.GetConfigSetting("fileservercrednamespace").Result;
FILESERVER_SMB_ENABLED = bool.Parse(Core.GetConfigSetting("fileserversmb").Result);

FILESERVER_PERSISTENT_VOLUMES = Core.GetConfigSetting("fileserverclientpv").Result;
FILESERVER_PERSISTENT_VOLUMECLAIMS = Core.GetConfigSetting("fileserverclientpvc").Result;

FILESERVER_CLIENT_VOLUMES = Core.GetConfigSetting("fileServerclientvolumes").Result;
FILESERVER_CLIENT_VOLUME_MOUNTS = Core.GetConfigSetting("fileServerclientvolumemounts").Result;

DAPR_ANNOTATIONS = Core.GetConfigSetting("daprannotations").Result;

PAYLOAD_APP_ANNOTATIONS = Core.GetConfigSetting("payloadappannotations").Result;
PAYLOAD_APP_CONFIG = Core.GetConfigSetting("payloadappconfig").Result;
PAYLOAD_APP_LABELS = Core.GetConfigSetting("payloadapplabels").Result;
PAYLOAD_APP_ENVIRONMENTVARIABLES = Core.GetConfigSetting("payloadappenvironmentvariables").Result;

if (Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") == "Development" || Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") == "IntegrationTest") {
ENABLE_YAML_DEBUG = true;
Expand Down
73 changes: 73 additions & 0 deletions src/Models/KubernetesObjects.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@

using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;

namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment;
public static partial class Models {
public class KubernetesObjects {
public class ResourceDefinition {
public ResourceSection Resources { get; set; }

public ResourceDefinition() {
Resources = new ResourceSection();
}
}

public class ResourceSection {
public ResourceDetails Limits { get; set; }
public ResourceDetails Requests { get; set; }

public ResourceSection() {
Limits = new ResourceDetails();
Requests = new ResourceDetails();
}
}

public class ResourceDetails {
public string Cpu { get; set; }
public string Memory { get; set; }

public ResourceDetails() {
Cpu = "";
Memory = "";
}
}

public class VolumeMountRoot {
public List<V1VolumeMount> VolumeMounts { get; set; }
public VolumeMountRoot() {
VolumeMounts = new List<V1VolumeMount>();
}
}

public class VolumeRoot {
public List<V1Volume> Volumes { get; set; }
public VolumeRoot() {
Volumes = new List<V1Volume>();
}

public class ConfigMapVolumeSource {
public string Name { get; set; }
public ConfigMapVolumeSource() {
Name = "";
}
}

public class SecretVolumeSource {
public string SecretName { get; set; }
public SecretVolumeSource() {
SecretName = "";
}
}

public class PersistentVolumeClaimVolumeSource {
public string ClaimName { get; set; }
public PersistentVolumeClaimVolumeSource() {
ClaimName = "";
}
}
}
}

}
48 changes: 22 additions & 26 deletions src/Program.cs
Original file line number Diff line number Diff line change
@@ -1,35 +1,12 @@
namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment;

public class Program {
private static void Test() {
MessageFormats.PlatformServices.Deployment.DeployRequest _request = new MessageFormats.PlatformServices.Deployment.DeployRequest();
_request.StartTime = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow);
_request.MaxDuration = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(new TimeSpan(0, 0, 0, 0, 0));
_request.AppContextFile = new MessageFormats.PlatformServices.Deployment.DeployRequest.Types.AppContextFile() {
FileName = "test1.jpg",
Required = false
};

_request.GpuRequirement = MessageFormats.PlatformServices.Deployment.DeployRequest.Types.GpuOptions.Nvidia;
_request.DeployAction = MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Create;




Google.Protobuf.JsonFormatter formatter = new Google.Protobuf.JsonFormatter(Google.Protobuf.JsonFormatter.Settings.Default);
string jsonString = formatter.Format(_request);

Console.WriteLine(jsonString);
Console.WriteLine("Woohoo!");

}
public static void Main(string[] args) {
//Test();
var builder = WebApplication.CreateBuilder(args);

builder.Configuration.AddJsonFile("/workspaces/platform-deployment-config/appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile("/workspaces/platform-deployment/src/appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile("/workspaces/platform-deployment/src/appsettings.{env:DOTNET_ENVIRONMENT}.json", optional: true, reloadOnChange: true).Build();
builder.Configuration.AddJsonFile("/workspace/platform-deployment-config/appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile("/workspace/platform-deployment/src/appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile("/workspace/platform-deployment/src/appsettings.{env:DOTNET_ENVIRONMENT}.json", optional: true, reloadOnChange: true).Build();

builder.WebHost.ConfigureKestrel(options => options.ListenAnyIP(50051, o => o.Protocols = HttpProtocols.Http2))
.ConfigureServices((services) => {
Expand All @@ -42,8 +19,13 @@ public static void Main(string[] args) {
services.AddSingleton<Utils.K8sClient>();
services.AddSingleton<Utils.DownlinkUtil>();
services.AddSingleton<Utils.TimeUtils>();
services.AddSingleton<Utils.TemplateUtil>();
services.AddHostedService<Services.ScheduleProcessor>(p => p.GetRequiredService<Services.ScheduleProcessor>());
services.AddSingleton<Core.IMessageHandler<MessageFormats.HostServices.Link.LinkResponse>, MessageHandler<MessageFormats.HostServices.Link.LinkResponse>>();
services.AddSingleton<Core.IMessageHandler<MessageFormats.Common.LogMessageResponse>, MessageHandler<MessageFormats.Common.LogMessageResponse>>();
}).ConfigureLogging((logging) => {
logging.AddProvider(new Microsoft.Extensions.Logging.SpaceFX.Logger.HostSvcLoggerProvider());
logging.AddConsole();
Expand All @@ -58,6 +40,20 @@ public static void Main(string[] args) {
await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909");
});
});

// Add a middleware to catch exceptions and stop the host gracefully
app.Use(async (context, next) => {
try {
await next.Invoke();
} catch (Exception ex) {
Console.Error.WriteLine($"Exception caught in middleware: {ex.Message}");
// Stop the host gracefully so it triggers the pod to error
var lifetime = context.RequestServices.GetService<IHostApplicationLifetime>();
lifetime?.StopApplication();
}
});

app.Run();
}
}
4 changes: 3 additions & 1 deletion src/Services/DeployRequestProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class DeployRequestProcessor : BackgroundService {
private readonly Utils.TimeUtils _timeUtils;
private string _scheduleImportDirectory;
private string _regctlApp;

private readonly ConcurrentDictionary<Guid, MessageFormats.PlatformServices.Deployment.DeployResponse> _deployRequestCache;
public DeployRequestProcessor(ILogger<DeployRequestProcessor> logger, IServiceProvider serviceProvider, IOptions<Models.APP_CONFIG> appConfig, Core.Services.PluginLoader pluginLoader, Core.Client client, PluginDelegates pluginDelegates, Utils.K8sClient k8sClient, Utils.DownlinkUtil downlinkUtil, Utils.TimeUtils timeUtil) {
_logger = logger;
Expand All @@ -27,12 +28,14 @@ public DeployRequestProcessor(ILogger<DeployRequestProcessor> logger, IServicePr
_k8sClient = k8sClient;
_downlinkUtil = downlinkUtil;
_timeUtils = timeUtil;

_deployRequestCache = new ConcurrentDictionary<Guid, MessageFormats.PlatformServices.Deployment.DeployResponse>();

_scheduleImportDirectory = Path.Combine(_client.GetXFerDirectories().Result.inbox_directory, _appConfig.SCHEDULE_IMPORT_DIRECTORY);

_regctlApp = Path.Combine(_client.GetXFerDirectories().Result.root_directory, "tmp", "regctl", "regctl");


if (File.Exists(_regctlApp)) {
_logger.LogInformation("regctl found at '{regctlApp}'", _regctlApp);
} else {
Expand All @@ -42,7 +45,6 @@ public DeployRequestProcessor(ILogger<DeployRequestProcessor> logger, IServicePr

if (_appConfig.PURGE_SCHEDULE_ON_BOOTUP) {
_client.ClearCache();
if (Directory.Exists(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults"))) Directory.Delete(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults"), true);
}

PopulateCacheFromDisk();
Expand Down
6 changes: 6 additions & 0 deletions src/Services/ScheduleProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) {
File.Move(file, file + ".processed");
downlinkFileName = file + ".processed";
} catch (FileNotFoundException fileEx) {
_logger.LogWarning("Detected a missing file '{file}'. Likely hasn't finished uploaded. Will retry. ", fileEx.FileName);
return deployResponses; // This'll be empty
} catch (Utils.NotAScheduleFileException notAScheduleFileEx) {
_logger.LogInformation("Detected a json file that isn't a schedule file. {exceptionMessage}", notAScheduleFileEx.Message);
return deployResponses; // This'll be empty
Expand Down Expand Up @@ -238,6 +241,9 @@ private void WaitForFileToFinishCopying(string filePath) {

returnDeployItems.Add(_response);
}
} catch (FileNotFoundException fileEx) {
_logger.LogError("File '{file}' found. Likely hasnt finished uploading. Will retry.", fileEx.FileName);
throw;
} catch (DataException dataEx) {
_logger.LogError("Item #'{itemX}' in '{file}' is invalid. Rejecting entire schedule file. Error: '{error}'. Please re-upload for reprocessing", itemCount, scheduleFilePath, dataEx.Message);
throw;
Expand Down
Loading

0 comments on commit 9fbcf33

Please sign in to comment.