Skip to content

Commit

Permalink
Better error handling, fixed model comparison
Browse files Browse the repository at this point in the history
  • Loading branch information
n00mkrad committed Aug 22, 2021
1 parent f60c1e8 commit 3b5e237
Show file tree
Hide file tree
Showing 10 changed files with 172 additions and 110 deletions.
10 changes: 4 additions & 6 deletions Code/Forms/ModelComparisonForm.cs
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,6 @@ async Task DoUpscale (int index, ModelData mdl, bool fullImage)
Program.mainForm.SetBusy(true);

Upscale.currentMode = Upscale.UpscaleMode.Composition;
//await ImageProcessing.PreProcessImages(Paths.previewPath, !bool.Parse(Config.Get("alpha")));
//if (cutoutMode)
//currentSourcePath += ".png";

string outImg = null;

Expand All @@ -150,18 +147,19 @@ async Task DoUpscale (int index, ModelData mdl, bool fullImage)
string inpath = Paths.previewPath;
if (fullImage) inpath = Paths.tempImgPath.GetParentDir();
await Upscale.Run(inpath, Paths.compositionOut, mdl, false, Config.GetBool("alpha"), PreviewUi.PreviewMode.None);
outImg = Directory.GetFiles(Paths.compositionOut, ".*.png*", SearchOption.AllDirectories)[0];
outImg = Directory.GetFiles(Paths.compositionOut, "*.png", SearchOption.AllDirectories)[0];
await PostProcessing.PostprocessingSingle(outImg, false);
await ProcessImage(PreviewUi.lastOutfile, mdl.model1Name);
IoUtils.TryCopy(PreviewUi.lastOutfile, Path.Combine(Paths.imgOutPath, $"{index}-{mdl.model1Name}.png"), true);
}
catch (Exception e)
{
if (Program.canceled) return;

if (e.Message.ToLower().Contains("index"))
Program.ShowMessage("The upscale process seems to have exited before completion!", "Error");

Logger.ErrorMessage("An error occured during upscaling:", e);
Program.mainForm.SetProgress(0f, "Cancelled.");
Program.Cancel($"An error occured: {e.Message}");
}

Program.mainForm.SetProgress(0, "Done.");
Expand Down
37 changes: 15 additions & 22 deletions Code/Implementations/EsrganNcnn.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,51 +41,44 @@ public static async Task Run(string inpath, string outpath, ModelData mdl)
$" -g {Config.GetInt("esrganNcnnGpu")} -m {NcnnUtils.currentNcnnModel.Wrap()} -s {scale} {tta} {ts}";
Logger.Log("[CMD] " + cmd);

Process ncnnProcess = OsUtils.NewProcess(!showWindow);
ncnnProcess.StartInfo.Arguments = cmd;
Process proc = OsUtils.NewProcess(!showWindow);
proc.StartInfo.Arguments = cmd;

if (!showWindow)
{
ncnnProcess.OutputDataReceived += NcnnOutputHandler;
ncnnProcess.ErrorDataReceived += NcnnOutputHandler;
proc.OutputDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, false); };
proc.ErrorDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, true); };
}

Program.currentEsrganProcess = ncnnProcess;
ncnnProcess.Start();
Program.lastImpProcess = proc;
proc.Start();

if (!showWindow)
{
ncnnProcess.BeginOutputReadLine();
ncnnProcess.BeginErrorReadLine();
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
}

while (!ncnnProcess.HasExited)
while (!proc.HasExited)
await Task.Delay(50);

if (Upscale.currentMode == Upscale.UpscaleMode.Batch)
{
await Task.Delay(1000);
Program.mainForm.SetProgress(100f, "[ESRGAN] Post-Processing...");
Program.mainForm.SetProgress(100f, "Post-Processing...");
PostProcessingQueue.Stop();
}
}

private static void NcnnOutputHandler(object sendingProcess, DataReceivedEventArgs output)
private static void OutputHandler(string line, bool error)
{
if (output == null || output.Data == null)
if (string.IsNullOrWhiteSpace(line) || line.Length < 6)
return;

string data = output.Data;
Logger.Log("[NCNN] " + data.Replace("\n", " ").Replace("\r", " "));
Logger.Log("[NCNN] " + line.Replace("\n", " ").Replace("\r", " "));

if (data.Contains("failed"))
{
Program.KillEsrgan();
Program.ShowMessage("Error occurred during upscaling: \n\n" + data + "\n\n", "Error");
}

if (data.Contains("vkAllocateMemory"))
Program.ShowMessage("ESRGAN-NCNN ran out of memory. Try reducing the tile size and avoid running programs in the background (especially games) that take up your VRAM.", "Error");
if(error)
GeneralOutputHandler.HandleImpErrorMsgs(line, Imps.esrganNcnn);
}
}
}
52 changes: 15 additions & 37 deletions Code/Implementations/EsrganPytorch.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,11 @@ public static async Task Run(string inpath, string outpath, ModelData mdl, bool

if (!showWindow)
{
esrganProcess.OutputDataReceived += OutputHandler;
esrganProcess.ErrorDataReceived += OutputHandler;
esrganProcess.OutputDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, false); };
esrganProcess.ErrorDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, true); };
}

Program.currentEsrganProcess = esrganProcess;
Program.lastImpProcess = esrganProcess;
esrganProcess.Start();

if (!showWindow)
Expand Down Expand Up @@ -134,7 +134,7 @@ public static string Interpolate(ModelData mdl)
bool showWindow = Config.GetInt("cmdDebugMode") > 0;
bool stayOpen = Config.GetInt("cmdDebugMode") == 2;

Process py = OsUtils.NewProcess(!showWindow);
Process proc = OsUtils.NewProcess(!showWindow);

string opt = stayOpen ? "/K" : "/C";
string alphaStr = (mdl.interp / 100f).ToString("0.00").Replace(",", ".");
Expand All @@ -145,12 +145,12 @@ public static string Interpolate(ModelData mdl)
string cmd = $"{opt} cd /D {Paths.GetAiDir(Imps.esrganPytorch).Wrap()} & ";
cmd += $"{EmbeddedPython.GetPyCmd()} interp.py {mdl.model1Path.Wrap()} {mdl.model2Path.Wrap()} {alphaStr} {outPath.Wrap()}";

py.StartInfo.Arguments = cmd;
Logger.Log("[ESRGAN Interp] CMD: " + py.StartInfo.Arguments);
py.Start();
py.WaitForExit();
string output = py.StandardOutput.ReadToEnd();
string err = py.StandardError.ReadToEnd();
proc.StartInfo.Arguments = cmd;
Logger.Log("[ESRGAN Interp] CMD: " + proc.StartInfo.Arguments);
proc.Start();
proc.WaitForExit();
string output = proc.StandardOutput.ReadToEnd();
string err = proc.StandardError.ReadToEnd();
if (!string.IsNullOrWhiteSpace(err)) output += "\n" + err;
Logger.Log("[ESRGAN Interp] Output: " + output);

Expand All @@ -160,37 +160,15 @@ public static string Interpolate(ModelData mdl)
return outPath;
}

private static void OutputHandler(object sendingProcess, DataReceivedEventArgs output)
private static void OutputHandler(string line, bool error)
{
if (output == null || output.Data == null)
if (string.IsNullOrWhiteSpace(line) || line.Length < 3)
return;

string data = output.Data;
Logger.Log("[Python] " + data);
Logger.Log("[Python] " + line.Replace("\n", " ").Replace("\r", " "));

if (data.ToLower().Contains("error"))
{
Program.KillEsrgan();
Program.ShowMessage("Error occurred: \n\n" + data + "\n\n", "Error");
}

if (data.ToLower().Contains("out of memory"))
Program.ShowMessage("ESRGAN ran out of memory. Try reducing the tile size and avoid running programs in the background (especially games) that take up your VRAM.", "Error");

if (data.Contains("Python was not found"))
Program.ShowMessage("Python was not found. Make sure you have a working Python 3 installation.", "Error");

if (data.Contains("ModuleNotFoundError"))
Program.ShowMessage("You are missing ESRGAN Python dependencies. Make sure Pytorch and cv2 (opencv-python) are installed.", "Error");

if (data.Contains("RRDBNet"))
Program.ShowMessage("Model appears to be incompatible!", "Error");

if (data.Contains("UnpicklingError"))
Program.ShowMessage("Failed to load model!", "Error");

if (PreviewUi.currentMode == PreviewUi.MdlMode.Interp && (data.Contains("must match the size of tensor b") || data.Contains("KeyError: 'model.")))
Program.ShowMessage("It seems like you tried to interpolate incompatible models!", "Error");
if (error)
GeneralOutputHandler.HandleImpErrorMsgs(line, Imps.esrganPytorch);
}

static string lastProgressString = "";
Expand Down
29 changes: 11 additions & 18 deletions Code/Implementations/RealEsrganNcnn.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ public static async Task Run(string inpath, string outpath, ModelData mdl)

if (!showWindow)
{
proc.OutputDataReceived += RealEsrganOutputHandler;
proc.ErrorDataReceived += RealEsrganOutputHandler;
proc.OutputDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, false); };
proc.ErrorDataReceived += (sender, outLine) => { OutputHandler(outLine.Data, true); };
}

Program.currentEsrganProcess = proc;
Program.lastImpProcess = proc;
proc.Start();

if (!showWindow)
Expand All @@ -72,36 +72,29 @@ public static async Task Run(string inpath, string outpath, ModelData mdl)
if (Upscale.currentMode == Upscale.UpscaleMode.Batch)
{
await Task.Delay(1000);
Program.mainForm.SetProgress(100f, "[ESRGAN] Post-Processing...");
Program.mainForm.SetProgress(100f, "Post-Processing...");
PostProcessingQueue.Stop();
}

}

private static void RealEsrganOutputHandler(object sendingProcess, DataReceivedEventArgs output)
private static void OutputHandler(string line, bool error)
{
if (output == null || output.Data == null)
if (string.IsNullOrWhiteSpace(line) || line.Length < 6)
return;

string data = output.Data;
Logger.Log("[NCNN] " + data.Replace("\n", " ").Replace("\r", " "));
Logger.Log("[NCNN] " + line.Replace("\n", " ").Replace("\r", " "));

bool showTileProgress = Upscale.currentMode == Upscale.UpscaleMode.Preview || Upscale.currentMode == Upscale.UpscaleMode.Single;

if (showTileProgress && data.Trim().EndsWith("%"))
if (showTileProgress && line.Trim().EndsWith("%"))
{
float percent = float.Parse(data.Replace("%", "").Replace(",", ".")) / 100f;
float percent = float.Parse(line.Replace("%", "").Replace(",", ".")) / 100f;
Program.mainForm.SetProgress(percent, $"Upscaling Tiles ({percent}%)");
}

if (data.Contains("failed"))
{
Program.KillEsrgan();
Program.ShowMessage("Error occurred during upscaling: \n\n" + data + "\n\n", "Error");
}

if (data.Contains("vkAllocateMemory"))
Program.ShowMessage("ESRGAN-NCNN ran out of memory. Try reducing the tile size and avoid running programs in the background (especially games) that take up your VRAM.", "Error");
if (error)
GeneralOutputHandler.HandleImpErrorMsgs(line, Imps.realEsrganNcnn);
}
}
}
76 changes: 76 additions & 0 deletions Code/Main/GeneralOutputHandler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
using Cupscale.UI;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Cupscale.Main
{
class GeneralOutputHandler
{
public static void HandleImpErrorMsgs(string log, Implementations.Implementation imp = null)
{
bool errored = false;

if (imp == Implementations.Imps.esrganNcnn || imp == Implementations.Imps.realEsrganNcnn)
{
if (!errored && log.Contains("vkAllocateMemory"))
{
Program.Cancel("NCNN ran out of memory.\nTry reducing the tile size and avoid " +
"running programs in the background (especially games) that take up your VRAM.");
errored = true;
}

}

if (imp == Implementations.Imps.esrganPytorch)
{
if (log.ToLower().Contains("out of memory"))
{
Program.ShowMessage("ESRGAN ran out of memory. Try reducing the tile size and avoid running programs in the background (especially games) that take up your VRAM.", "Error");
errored = true;
}

if (log.Contains("Python was not found"))
{
Program.Cancel("Python was not found. Make sure you have a working Python 3 installation, or use the embedded runtime.");
errored = true;
}

if (log.Contains("ModuleNotFoundError"))
{
Program.Cancel("You are missing Python dependencies. Make sure Pytorch and cv2 (opencv-python) are installed.");
errored = true;
}

if (log.Contains("RRDBNet"))
{
Program.Cancel("Model appears to be incompatible!");
errored = true;
}

if (log.Contains("UnpicklingError"))
{
Program.Cancel("Failed to load model!\nPossibly it's corrupted or in an unknown format.");
errored = true;
}

if (PreviewUi.currentMode == PreviewUi.MdlMode.Interp && (log.Contains("must match the size of tensor b") || log.Contains("KeyError: 'model.")))
{
Program.Cancel("It seems like you tried to interpolate incompatible models!");
errored = true;
}
}

if (!errored && log.Contains("failed"))
{
Program.Cancel($"Error occurred while running AI implementation: \n\n{log}\n\n");
errored = true;
}

if (errored)
Program.Cancel();
}
}
}
4 changes: 2 additions & 2 deletions Code/Main/MainForm.cs
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public async void BusyCheckLoop ()
await Task.Delay(100);
continue;
}
cancelBtn.Visible = (Program.busy && Program.currentEsrganProcess != null && !Program.currentEsrganProcess.HasExited);
cancelBtn.Visible = (Program.busy && Program.lastImpProcess != null && !Program.lastImpProcess.HasExited);
await Task.Delay(100);
}
}
Expand Down Expand Up @@ -794,7 +794,7 @@ private void MainForm_Resize(object sender, EventArgs e)

private void cancelBtn_Click(object sender, EventArgs e)
{
Program.KillEsrgan();
Program.Cancel();
}

private void paypalBtn_Click(object sender, EventArgs e)
Expand Down
15 changes: 10 additions & 5 deletions Code/Main/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ internal static class Program
public static List<MsgBox> openMessageBoxes = new List<MsgBox>(); // Temp forms that get closed when something gets cancelled

public static bool lastUpscaleIsVideo;
public static Process currentEsrganProcess;
public static Process lastImpProcess;
public static bool canceled = false;

public static bool busy;
Expand Down Expand Up @@ -70,12 +70,17 @@ private static void Main()
Application.Run(new MainForm());
}

public static void KillEsrgan (bool cleanup = true)
public static void Cancel (string reason = "", bool cleanup = true)
{
if (currentEsrganProcess == null || currentEsrganProcess.HasExited)
return;
canceled = true;
OsUtils.KillProcessTree(currentEsrganProcess.Id);
OsUtils.KillProcessTree(lastImpProcess);

mainForm.SetProgress(0, "Canceled.");
mainForm.SetBusy(false);

if (reason.Trim().Length > 0)
ShowMessage(reason);

if (cleanup)
{
IoUtils.ClearDir(Paths.imgInPath);
Expand Down
Loading

0 comments on commit 3b5e237

Please sign in to comment.