From 690afd5834457ec7bf4a597979af7938c556d077 Mon Sep 17 00:00:00 2001 From: Sugita Akira Date: Fri, 15 Dec 2023 23:54:09 -0800 Subject: [PATCH] worldline-r supports directs --- OpenUtau.Core/Classic/ClassicRenderer.cs | 1 - OpenUtau.Core/Classic/ExeWavtool.cs | 4 +- OpenUtau.Core/Classic/ResamplerItem.cs | 42 +++++++++++++- OpenUtau.Core/Classic/SharpWavtool.cs | 67 +++++----------------- OpenUtau.Core/Classic/WorldlineRenderer.cs | 28 +++++++++ OpenUtau.Core/Format/USTx.cs | 4 +- OpenUtau.Core/Render/RenderPhrase.cs | 7 ++- OpenUtau.Core/Render/Worldline.cs | 2 +- 8 files changed, 95 insertions(+), 60 deletions(-) diff --git a/OpenUtau.Core/Classic/ClassicRenderer.cs b/OpenUtau.Core/Classic/ClassicRenderer.cs index 2f1d19fda..56e2ec7e8 100644 --- a/OpenUtau.Core/Classic/ClassicRenderer.cs +++ b/OpenUtau.Core/Classic/ClassicRenderer.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using NAudio.Wave; diff --git a/OpenUtau.Core/Classic/ExeWavtool.cs b/OpenUtau.Core/Classic/ExeWavtool.cs index 1c5c597e7..08f76c55e 100644 --- a/OpenUtau.Core/Classic/ExeWavtool.cs +++ b/OpenUtau.Core/Classic/ExeWavtool.cs @@ -102,7 +102,7 @@ void WriteSetUp(StreamWriter writer, List resamplerItems, string void WriteItem(StreamWriter writer, ResamplerItem item, int index, int total) { writer.WriteLine($"@set resamp={item.resampler.FilePath}"); - writer.WriteLine($"@set params={item.volume} {item.modulation} !{item.tempo.ToString("G999")} {Base64.Base64EncodeInt12(item.pitches)}"); + writer.WriteLine($"@set params={item.volume} {item.modulation} !{item.tempo:G999} {Base64.Base64EncodeInt12(item.pitches)}"); writer.WriteLine($"@set flag=\"{item.GetFlagsString()}\""); writer.WriteLine($"@set env={GetEnvelope(item)}"); writer.WriteLine($"@set stp={item.skipOver}"); @@ -110,7 +110,7 @@ void WriteItem(StreamWriter writer, ResamplerItem item, int index, int total) { string relOutputFile = Path.GetRelativePath(PathManager.Inst.CachePath, item.outputFile); writer.WriteLine($"@set temp=\"%cachedir%\\{relOutputFile}\""); string toneName = MusicMath.GetToneName(item.tone); - string dur = $"{item.phone.duration.ToString("G999")}@{item.phone.adjustedTempo.ToString("G999")}{(item.durCorrection >= 0 ? "+" : "")}{item.durCorrection}"; + string dur = $"{item.phone.duration:G999}@{item.phone.adjustedTempo:G999}{(item.durCorrection >= 0 ? "+" : "")}{item.durCorrection}"; string relInputTemp = Path.GetRelativePath(PathManager.Inst.CachePath, item.inputTemp); writer.WriteLine($"@echo {MakeProgressBar(index + 1, total)}"); writer.WriteLine($"@call %helper% \"%oto%\\{relInputTemp}\" {toneName} {dur} {item.preutter} {item.offset} {item.durRequired} {item.consonant} {item.cutoff} {index}"); diff --git a/OpenUtau.Core/Classic/ResamplerItem.cs b/OpenUtau.Core/Classic/ResamplerItem.cs index e6c597221..9cfba16e8 100644 --- a/OpenUtau.Core/Classic/ResamplerItem.cs +++ b/OpenUtau.Core/Classic/ResamplerItem.cs @@ -2,12 +2,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Numerics; using System.Text; using K4os.Hash.xxHash; using NAudio.Wave; using OpenUtau.Core; using OpenUtau.Core.Render; using OpenUtau.Core.Ustx; +using static NetMQ.NetMQSelector; using static OpenUtau.Api.Phonemizer; namespace OpenUtau.Classic { @@ -80,7 +82,7 @@ public ResamplerItem(RenderPhrase phrase, RenderPhone phone) { var pitchIntervalMs = MusicMath.TempoTickToMs(tempo, 5); var pitchSampleStartMs = phone.positionMs - pitchLeadingMs; - for (int i=0; i EnvelopeMsToSamples() { + int skipOverSamples = (int)(skipOver * 44100 / 1000); + var envelope = phone.envelope.ToList(); + double shift = -envelope[0].X; + for (int i = 0; i < envelope.Count; ++i) { + var point = envelope[i]; + point.X = (float)((point.X + shift) * 44100 / 1000) + skipOverSamples; + point.Y /= 100; + envelope[i] = point; + } + return envelope; + } + + public void ApplyEnvelope(float[] samples) { + var envelope = EnvelopeMsToSamples(); + int nextPoint = 0; + for (int i = 0; i < samples.Length; ++i) { + while (nextPoint < envelope.Count && i > envelope[nextPoint].X) { + nextPoint++; + } + float gain; + if (nextPoint == 0) { + gain = envelope.First().Y; + } else if (nextPoint >= envelope.Count) { + gain = envelope.Last().Y; + } else { + var p0 = envelope[nextPoint - 1]; + var p1 = envelope[nextPoint]; + if (p0.X >= p1.X) { + gain = p0.Y; + } else { + gain = p0.Y + (p1.Y - p0.Y) * (i - p0.X) / (p1.X - p0.X); + } + } + samples[i] *= gain; + } + } } } diff --git a/OpenUtau.Core/Classic/SharpWavtool.cs b/OpenUtau.Core/Classic/SharpWavtool.cs index 8cd477265..84ac074f4 100644 --- a/OpenUtau.Core/Classic/SharpWavtool.cs +++ b/OpenUtau.Core/Classic/SharpWavtool.cs @@ -56,23 +56,23 @@ public float[] Concatenate(List resamplerItems, string tempPath, segment.posMs = item.phone.positionMs - item.phone.leadingMs - (phrase.positionMs - phrase.leadingMs); segment.posSamples = (int)Math.Round(segment.posMs * 44100 / 1000); segment.skipSamples = (int)Math.Round(item.skipOver * 44100 / 1000); - segment.envelope = EnvelopeMsToSamples(item.phone.envelope, segment.skipSamples); - - if (!phaseComp) { - continue; + segment.envelope = item.EnvelopeMsToSamples(); + + if (phaseComp) { + var headWindow = GetHeadWindow(segment.samples, segment.envelope, out segment.headWindowStart); + segment.headWindowF0 = GetF0AtSample(phrase, + segment.posSamples - segment.skipSamples + segment.headWindowStart + headWindow.Length / 2); + segment.headPhase = CalcPhase(headWindow, + segment.posSamples - segment.skipSamples + segment.headWindowStart, 44100, segment.headWindowF0); + + var tailWindow = GetTailWindow(segment.samples, segment.envelope, out segment.tailWindowStart); + segment.tailWindowF0 = GetF0AtSample(phrase, + segment.posSamples - segment.skipSamples + segment.tailWindowStart + tailWindow.Length / 2); + segment.tailPhase = CalcPhase(tailWindow, + segment.posSamples - segment.skipSamples + segment.tailWindowStart, 44100, segment.tailWindowF0); } - var headWindow = GetHeadWindow(segment.samples, segment.envelope, out segment.headWindowStart); - segment.headWindowF0 = GetF0AtSample(phrase, - segment.posSamples - segment.skipSamples + segment.headWindowStart + headWindow.Length / 2); - segment.headPhase = CalcPhase(headWindow, - segment.posSamples - segment.skipSamples + segment.headWindowStart, 44100, segment.headWindowF0); - - var tailWindow = GetTailWindow(segment.samples, segment.envelope, out segment.tailWindowStart); - segment.tailWindowF0 = GetF0AtSample(phrase, - segment.posSamples - segment.skipSamples + segment.tailWindowStart + tailWindow.Length / 2); - segment.tailPhase = CalcPhase(tailWindow, - segment.posSamples - segment.skipSamples + segment.tailWindowStart, 44100, segment.tailWindowF0); + item.ApplyEnvelope(segment.samples); } if (phaseComp) { @@ -100,7 +100,6 @@ public float[] Concatenate(List resamplerItems, string tempPath, var phraseSamples = new float[0]; foreach (var segment in segments) { Array.Resize(ref phraseSamples, segment.posSamples + segment.correction + segment.samples.Length - segment.skipSamples); - ApplyEnvelope(segment.samples, segment.envelope); for (int i = Math.Max(0, -segment.skipSamples); i < segment.samples.Length - segment.skipSamples; i++) { phraseSamples[segment.posSamples + segment.correction + i] += segment.samples[segment.skipSamples + i]; } @@ -108,42 +107,6 @@ public float[] Concatenate(List resamplerItems, string tempPath, return phraseSamples; } - private static void ApplyEnvelope(float[] data, IList envelope) { - int nextPoint = 0; - for (int i = 0; i < data.Length; ++i) { - while (nextPoint < envelope.Count && i > envelope[nextPoint].X) { - nextPoint++; - } - float gain; - if (nextPoint == 0) { - gain = envelope.First().Y; - } else if (nextPoint >= envelope.Count) { - gain = envelope.Last().Y; - } else { - var p0 = envelope[nextPoint - 1]; - var p1 = envelope[nextPoint]; - if (p0.X >= p1.X) { - gain = p0.Y; - } else { - gain = p0.Y + (p1.Y - p0.Y) * (i - p0.X) / (p1.X - p0.X); - } - } - data[i] *= gain; - } - } - - private static IList EnvelopeMsToSamples(IList envelope, int skipOverSamples) { - envelope = new List(envelope); - double shift = -envelope[0].X; - for (var i = 0; i < envelope.Count; i++) { - var point = envelope[i]; - point.X = (float)((point.X + shift) * 44100 / 1000) + skipOverSamples; - point.Y /= 100; - envelope[i] = point; - } - return envelope; - } - private float[] GetHeadWindow(float[] samples, IList envelope, out int windowStart) { var windowCenter = (envelope[0] + envelope[1]) * 0.5f; windowStart = Math.Max((int)windowCenter.X - 440, 0); diff --git a/OpenUtau.Core/Classic/WorldlineRenderer.cs b/OpenUtau.Core/Classic/WorldlineRenderer.cs index 9f7f5623e..4b6a17862 100644 --- a/OpenUtau.Core/Classic/WorldlineRenderer.cs +++ b/OpenUtau.Core/Classic/WorldlineRenderer.cs @@ -5,11 +5,13 @@ using System.Threading; using System.Threading.Tasks; using NAudio.Wave; +using NumSharp; using OpenUtau.Core; using OpenUtau.Core.Format; using OpenUtau.Core.Render; using OpenUtau.Core.SignalChain; using OpenUtau.Core.Ustx; +using static NetMQ.NetMQSelector; namespace OpenUtau.Classic { public class WorldlineRenderer : IRenderer { @@ -28,6 +30,7 @@ public class WorldlineRenderer : IRenderer { Ustx.BREC, Ustx.TENC, Ustx.VOIC, + Ustx.DIR, }; public USingerType SingerType => USingerType.Classic; @@ -83,6 +86,7 @@ public Task Render(RenderPhrase phrase, Progress progress, int tra var voicing = SampleCurve(phrase, phrase.voicing, 1.0, frames, x => 0.01 * x); phraseSynth.SetCurves(f0, gender, tension, breathiness, voicing); result.samples = phraseSynth.Synth(); + AddDirects(phrase, resamplerItems, result); var source = new WaveSource(0, 0, 0, 1); source.SetSamples(result.samples); WaveFileWriter.CreateWaveFile16(wavPath, new ExportAdapter(source).ToMono(1, 0)); @@ -114,6 +118,30 @@ double[] SampleCurve(RenderPhrase phrase, float[] curve, double defaultValue, in return result; } + private static void AddDirects(RenderPhrase phrase, List resamplerItems, RenderResult result) { + foreach (var item in resamplerItems) { + if (!item.phone.direct) { + continue; + } + double posMs = item.phone.positionMs - item.phone.leadingMs - (phrase.positionMs - phrase.leadingMs); + int startPhraseIndex = (int)(posMs / 1000 * 44100); + using (var waveStream = Wave.OpenFile(item.phone.oto.File)) { + if (waveStream == null) { + continue; + } + float[] samples = Wave.GetSamples(waveStream!.ToSampleProvider().ToMono(1, 0)); + int offset = (int)(item.phone.oto.Offset / 1000 * 44100); + int cutoff = (int)(item.phone.oto.Cutoff / 1000 * 44100); + int length = cutoff >= 0 ? (samples.Length - offset - cutoff) : -cutoff; + samples = samples.Skip(offset).Take(length).ToArray(); + item.ApplyEnvelope(samples); + for (int i = 0; i < Math.Min(samples.Length, result.samples.Length - startPhraseIndex); ++i) { + result.samples[startPhraseIndex + i] = samples[i]; + } + } + } + } + public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) { return null; } diff --git a/OpenUtau.Core/Format/USTx.cs b/OpenUtau.Core/Format/USTx.cs index aa7e12af3..e186c612c 100644 --- a/OpenUtau.Core/Format/USTx.cs +++ b/OpenUtau.Core/Format/USTx.cs @@ -26,12 +26,13 @@ public class Ustx { public const string LPF = "lpf"; public const string MOD = "mod"; public const string ALT = "alt"; + public const string DIR = "dir"; public const string SHFT = "shft"; public const string SHFC = "shfc"; public const string TENC = "tenc"; public const string VOIC = "voic"; - public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC, }; + public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC }; public static void AddDefaultExpressions(UProject project) { project.RegisterExpression(new UExpressionDescriptor("dynamics (curve)", DYN, -240, 120, 0) { type = UExpressionType.Curve }); @@ -49,6 +50,7 @@ public static void AddDefaultExpressions(UProject project) { project.RegisterExpression(new UExpressionDescriptor("lowpass", LPF, 0, 100, 0, "H")); project.RegisterExpression(new UExpressionDescriptor("modulation", MOD, 0, 100, 0)); project.RegisterExpression(new UExpressionDescriptor("alternate", ALT, 0, 16, 0)); + project.RegisterExpression(new UExpressionDescriptor("direct", DIR, false, new string[] { "off", "on" })); project.RegisterExpression(new UExpressionDescriptor("tone shift", SHFT, -36, 36, 0)); project.RegisterExpression(new UExpressionDescriptor("tone shift (curve)", SHFC, -1200, 1200, 0) { type = UExpressionType.Curve }); project.RegisterExpression(new UExpressionDescriptor("tension (curve)", TENC, -100, 100, 0) { type = UExpressionType.Curve }); diff --git a/OpenUtau.Core/Render/RenderPhrase.cs b/OpenUtau.Core/Render/RenderPhrase.cs index 1557239be..d73066f5f 100644 --- a/OpenUtau.Core/Render/RenderPhrase.cs +++ b/OpenUtau.Core/Render/RenderPhrase.cs @@ -63,6 +63,7 @@ public class RenderPhone { public readonly float volume; public readonly float velocity; public readonly float modulation; + public readonly bool direct; public readonly Vector2[] envelope; public readonly UOto oto; @@ -109,12 +110,13 @@ internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note flags = phoneme.GetResamplerFlags(project, track); string voiceColor = phoneme.GetVoiceColor(project, track); suffix = track.Singer.Subbanks.FirstOrDefault( - subbank => subbank.Color == voiceColor)?.Suffix; + subbank => subbank.Color == voiceColor)?.Suffix ?? string.Empty; volume = phoneme.GetExpression(project, track, Format.Ustx.VOL).Item1 * 0.01f; velocity = phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 * 0.01f; modulation = phoneme.GetExpression(project, track, Format.Ustx.MOD).Item1 * 0.01f; leadingMs = phoneme.preutter; envelope = phoneme.envelope.data.ToArray(); + direct = phoneme.GetExpression(project, track, Format.Ustx.DIR).Item1 == 1; oto = phoneme.oto; hash = Hash(); @@ -134,10 +136,11 @@ private ulong Hash() { writer.Write(flag.Item2.Value); } } - writer.Write(suffix ?? string.Empty); + writer.Write(suffix); writer.Write(volume); writer.Write(velocity); writer.Write(modulation); + writer.Write(direct); writer.Write(leadingMs); foreach (var point in envelope) { writer.Write(point.X); diff --git a/OpenUtau.Core/Render/Worldline.cs b/OpenUtau.Core/Render/Worldline.cs index c31a9a6fd..dd72f533f 100644 --- a/OpenUtau.Core/Render/Worldline.cs +++ b/OpenUtau.Core/Render/Worldline.cs @@ -206,7 +206,7 @@ public SynthRequestWrapper(ResamplerItem item) { required_length = item.durRequired, consonant = item.consonant, cut_off = item.cutoff, - volume = item.volume, + volume = item.phone.direct ? 0 : item.volume, modulation = item.modulation, tempo = item.tempo, pitch_bend_length = item.pitches.Length,