Skip to content

Commit

Permalink
worldline-r supports directs
Browse files Browse the repository at this point in the history
  • Loading branch information
@stakira
stakira committed Dec 16, 2023
1 parent 7582032 commit 690afd5
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 60 deletions.
1 change: 0 additions & 1 deletion OpenUtau.Core/Classic/ClassicRenderer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NAudio.Wave;
Expand Down
4 changes: 2 additions & 2 deletions OpenUtau.Core/Classic/ExeWavtool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,15 @@ void WriteSetUp(StreamWriter writer, List<ResamplerItem> resamplerItems, string

void WriteItem(StreamWriter writer, ResamplerItem item, int index, int total) {
writer.WriteLine($"@set resamp={item.resampler.FilePath}");
writer.WriteLine($"@set params={item.volume} {item.modulation} !{item.tempo.ToString("G999")} {Base64.Base64EncodeInt12(item.pitches)}");
writer.WriteLine($"@set params={item.volume} {item.modulation} !{item.tempo:G999} {Base64.Base64EncodeInt12(item.pitches)}");
writer.WriteLine($"@set flag=\"{item.GetFlagsString()}\"");
writer.WriteLine($"@set env={GetEnvelope(item)}");
writer.WriteLine($"@set stp={item.skipOver}");
writer.WriteLine($"@set vel={item.velocity}");
string relOutputFile = Path.GetRelativePath(PathManager.Inst.CachePath, item.outputFile);
writer.WriteLine($"@set temp=\"%cachedir%\\{relOutputFile}\"");
string toneName = MusicMath.GetToneName(item.tone);
string dur = $"{item.phone.duration.ToString("G999")}@{item.phone.adjustedTempo.ToString("G999")}{(item.durCorrection >= 0 ? "+" : "")}{item.durCorrection}";
string dur = $"{item.phone.duration:G999}@{item.phone.adjustedTempo:G999}{(item.durCorrection >= 0 ? "+" : "")}{item.durCorrection}";
string relInputTemp = Path.GetRelativePath(PathManager.Inst.CachePath, item.inputTemp);
writer.WriteLine($"@echo {MakeProgressBar(index + 1, total)}");
writer.WriteLine($"@call %helper% \"%oto%\\{relInputTemp}\" {toneName} {dur} {item.preutter} {item.offset} {item.durRequired} {item.consonant} {item.cutoff} {index}");
Expand Down
42 changes: 41 additions & 1 deletion OpenUtau.Core/Classic/ResamplerItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Text;
using K4os.Hash.xxHash;
using NAudio.Wave;
using OpenUtau.Core;
using OpenUtau.Core.Render;
using OpenUtau.Core.Ustx;
using static NetMQ.NetMQSelector;
using static OpenUtau.Api.Phonemizer;

namespace OpenUtau.Classic {
Expand Down Expand Up @@ -80,7 +82,7 @@ public ResamplerItem(RenderPhrase phrase, RenderPhone phone) {
var pitchIntervalMs = MusicMath.TempoTickToMs(tempo, 5);
var pitchSampleStartMs = phone.positionMs - pitchLeadingMs;

for (int i=0; i<pitches.Length; i++) {
for (int i = 0; i < pitches.Length; i++) {
var samplePosMs = pitchSampleStartMs + pitchIntervalMs * i;
var samplePosTick = (int)Math.Floor(phrase.timeAxis.MsPosToNonExactTickPos(samplePosMs));

Expand Down Expand Up @@ -145,5 +147,43 @@ ulong Hash() {
}
}
}

public List<Vector2> EnvelopeMsToSamples() {
int skipOverSamples = (int)(skipOver * 44100 / 1000);
var envelope = phone.envelope.ToList();
double shift = -envelope[0].X;
for (int i = 0; i < envelope.Count; ++i) {
var point = envelope[i];
point.X = (float)((point.X + shift) * 44100 / 1000) + skipOverSamples;
point.Y /= 100;
envelope[i] = point;
}
return envelope;
}

public void ApplyEnvelope(float[] samples) {
var envelope = EnvelopeMsToSamples();
int nextPoint = 0;
for (int i = 0; i < samples.Length; ++i) {
while (nextPoint < envelope.Count && i > envelope[nextPoint].X) {
nextPoint++;
}
float gain;
if (nextPoint == 0) {
gain = envelope.First().Y;
} else if (nextPoint >= envelope.Count) {
gain = envelope.Last().Y;
} else {
var p0 = envelope[nextPoint - 1];
var p1 = envelope[nextPoint];
if (p0.X >= p1.X) {
gain = p0.Y;
} else {
gain = p0.Y + (p1.Y - p0.Y) * (i - p0.X) / (p1.X - p0.X);
}
}
samples[i] *= gain;
}
}
}
}
67 changes: 15 additions & 52 deletions OpenUtau.Core/Classic/SharpWavtool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,23 +56,23 @@ public float[] Concatenate(List<ResamplerItem> resamplerItems, string tempPath,
segment.posMs = item.phone.positionMs - item.phone.leadingMs - (phrase.positionMs - phrase.leadingMs);
segment.posSamples = (int)Math.Round(segment.posMs * 44100 / 1000);
segment.skipSamples = (int)Math.Round(item.skipOver * 44100 / 1000);
segment.envelope = EnvelopeMsToSamples(item.phone.envelope, segment.skipSamples);

if (!phaseComp) {
continue;
segment.envelope = item.EnvelopeMsToSamples();

if (phaseComp) {
var headWindow = GetHeadWindow(segment.samples, segment.envelope, out segment.headWindowStart);
segment.headWindowF0 = GetF0AtSample(phrase,
segment.posSamples - segment.skipSamples + segment.headWindowStart + headWindow.Length / 2);
segment.headPhase = CalcPhase(headWindow,
segment.posSamples - segment.skipSamples + segment.headWindowStart, 44100, segment.headWindowF0);

var tailWindow = GetTailWindow(segment.samples, segment.envelope, out segment.tailWindowStart);
segment.tailWindowF0 = GetF0AtSample(phrase,
segment.posSamples - segment.skipSamples + segment.tailWindowStart + tailWindow.Length / 2);
segment.tailPhase = CalcPhase(tailWindow,
segment.posSamples - segment.skipSamples + segment.tailWindowStart, 44100, segment.tailWindowF0);
}

var headWindow = GetHeadWindow(segment.samples, segment.envelope, out segment.headWindowStart);
segment.headWindowF0 = GetF0AtSample(phrase,
segment.posSamples - segment.skipSamples + segment.headWindowStart + headWindow.Length / 2);
segment.headPhase = CalcPhase(headWindow,
segment.posSamples - segment.skipSamples + segment.headWindowStart, 44100, segment.headWindowF0);

var tailWindow = GetTailWindow(segment.samples, segment.envelope, out segment.tailWindowStart);
segment.tailWindowF0 = GetF0AtSample(phrase,
segment.posSamples - segment.skipSamples + segment.tailWindowStart + tailWindow.Length / 2);
segment.tailPhase = CalcPhase(tailWindow,
segment.posSamples - segment.skipSamples + segment.tailWindowStart, 44100, segment.tailWindowF0);
item.ApplyEnvelope(segment.samples);
}

if (phaseComp) {
Expand Down Expand Up @@ -100,50 +100,13 @@ public float[] Concatenate(List<ResamplerItem> resamplerItems, string tempPath,
var phraseSamples = new float[0];
foreach (var segment in segments) {
Array.Resize(ref phraseSamples, segment.posSamples + segment.correction + segment.samples.Length - segment.skipSamples);
ApplyEnvelope(segment.samples, segment.envelope);
for (int i = Math.Max(0, -segment.skipSamples); i < segment.samples.Length - segment.skipSamples; i++) {
phraseSamples[segment.posSamples + segment.correction + i] += segment.samples[segment.skipSamples + i];
}
}
return phraseSamples;
}

private static void ApplyEnvelope(float[] data, IList<Vector2> envelope) {
int nextPoint = 0;
for (int i = 0; i < data.Length; ++i) {
while (nextPoint < envelope.Count && i > envelope[nextPoint].X) {
nextPoint++;
}
float gain;
if (nextPoint == 0) {
gain = envelope.First().Y;
} else if (nextPoint >= envelope.Count) {
gain = envelope.Last().Y;
} else {
var p0 = envelope[nextPoint - 1];
var p1 = envelope[nextPoint];
if (p0.X >= p1.X) {
gain = p0.Y;
} else {
gain = p0.Y + (p1.Y - p0.Y) * (i - p0.X) / (p1.X - p0.X);
}
}
data[i] *= gain;
}
}

private static IList<Vector2> EnvelopeMsToSamples(IList<Vector2> envelope, int skipOverSamples) {
envelope = new List<Vector2>(envelope);
double shift = -envelope[0].X;
for (var i = 0; i < envelope.Count; i++) {
var point = envelope[i];
point.X = (float)((point.X + shift) * 44100 / 1000) + skipOverSamples;
point.Y /= 100;
envelope[i] = point;
}
return envelope;
}

private float[] GetHeadWindow(float[] samples, IList<Vector2> envelope, out int windowStart) {
var windowCenter = (envelope[0] + envelope[1]) * 0.5f;
windowStart = Math.Max((int)windowCenter.X - 440, 0);
Expand Down
28 changes: 28 additions & 0 deletions OpenUtau.Core/Classic/WorldlineRenderer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
using System.Threading;
using System.Threading.Tasks;
using NAudio.Wave;
using NumSharp;
using OpenUtau.Core;
using OpenUtau.Core.Format;
using OpenUtau.Core.Render;
using OpenUtau.Core.SignalChain;
using OpenUtau.Core.Ustx;
using static NetMQ.NetMQSelector;

namespace OpenUtau.Classic {
public class WorldlineRenderer : IRenderer {
Expand All @@ -28,6 +30,7 @@ public class WorldlineRenderer : IRenderer {
Ustx.BREC,
Ustx.TENC,
Ustx.VOIC,
Ustx.DIR,
};

public USingerType SingerType => USingerType.Classic;
Expand Down Expand Up @@ -83,6 +86,7 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
var voicing = SampleCurve(phrase, phrase.voicing, 1.0, frames, x => 0.01 * x);
phraseSynth.SetCurves(f0, gender, tension, breathiness, voicing);
result.samples = phraseSynth.Synth();
AddDirects(phrase, resamplerItems, result);
var source = new WaveSource(0, 0, 0, 1);
source.SetSamples(result.samples);
WaveFileWriter.CreateWaveFile16(wavPath, new ExportAdapter(source).ToMono(1, 0));
Expand Down Expand Up @@ -114,6 +118,30 @@ double[] SampleCurve(RenderPhrase phrase, float[] curve, double defaultValue, in
return result;
}

private static void AddDirects(RenderPhrase phrase, List<ResamplerItem> resamplerItems, RenderResult result) {
foreach (var item in resamplerItems) {
if (!item.phone.direct) {
continue;
}
double posMs = item.phone.positionMs - item.phone.leadingMs - (phrase.positionMs - phrase.leadingMs);
int startPhraseIndex = (int)(posMs / 1000 * 44100);
using (var waveStream = Wave.OpenFile(item.phone.oto.File)) {
if (waveStream == null) {
continue;
}
float[] samples = Wave.GetSamples(waveStream!.ToSampleProvider().ToMono(1, 0));
int offset = (int)(item.phone.oto.Offset / 1000 * 44100);
int cutoff = (int)(item.phone.oto.Cutoff / 1000 * 44100);
int length = cutoff >= 0 ? (samples.Length - offset - cutoff) : -cutoff;
samples = samples.Skip(offset).Take(length).ToArray();
item.ApplyEnvelope(samples);
for (int i = 0; i < Math.Min(samples.Length, result.samples.Length - startPhraseIndex); ++i) {
result.samples[startPhraseIndex + i] = samples[i];
}
}
}
}

public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) {
return null;
}
Expand Down
4 changes: 3 additions & 1 deletion OpenUtau.Core/Format/USTx.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ public class Ustx {
public const string LPF = "lpf";
public const string MOD = "mod";
public const string ALT = "alt";
public const string DIR = "dir";
public const string SHFT = "shft";
public const string SHFC = "shfc";
public const string TENC = "tenc";
public const string VOIC = "voic";

public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC, };
public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC };

public static void AddDefaultExpressions(UProject project) {
project.RegisterExpression(new UExpressionDescriptor("dynamics (curve)", DYN, -240, 120, 0) { type = UExpressionType.Curve });
Expand All @@ -49,6 +50,7 @@ public static void AddDefaultExpressions(UProject project) {
project.RegisterExpression(new UExpressionDescriptor("lowpass", LPF, 0, 100, 0, "H"));
project.RegisterExpression(new UExpressionDescriptor("modulation", MOD, 0, 100, 0));
project.RegisterExpression(new UExpressionDescriptor("alternate", ALT, 0, 16, 0));
project.RegisterExpression(new UExpressionDescriptor("direct", DIR, false, new string[] { "off", "on" }));
project.RegisterExpression(new UExpressionDescriptor("tone shift", SHFT, -36, 36, 0));
project.RegisterExpression(new UExpressionDescriptor("tone shift (curve)", SHFC, -1200, 1200, 0) { type = UExpressionType.Curve });
project.RegisterExpression(new UExpressionDescriptor("tension (curve)", TENC, -100, 100, 0) { type = UExpressionType.Curve });
Expand Down
7 changes: 5 additions & 2 deletions OpenUtau.Core/Render/RenderPhrase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public class RenderPhone {
public readonly float volume;
public readonly float velocity;
public readonly float modulation;
public readonly bool direct;
public readonly Vector2[] envelope;

public readonly UOto oto;
Expand Down Expand Up @@ -109,12 +110,13 @@ internal RenderPhone(UProject project, UTrack track, UVoicePart part, UNote note
flags = phoneme.GetResamplerFlags(project, track);
string voiceColor = phoneme.GetVoiceColor(project, track);
suffix = track.Singer.Subbanks.FirstOrDefault(
subbank => subbank.Color == voiceColor)?.Suffix;
subbank => subbank.Color == voiceColor)?.Suffix ?? string.Empty;
volume = phoneme.GetExpression(project, track, Format.Ustx.VOL).Item1 * 0.01f;
velocity = phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 * 0.01f;
modulation = phoneme.GetExpression(project, track, Format.Ustx.MOD).Item1 * 0.01f;
leadingMs = phoneme.preutter;
envelope = phoneme.envelope.data.ToArray();
direct = phoneme.GetExpression(project, track, Format.Ustx.DIR).Item1 == 1;

oto = phoneme.oto;
hash = Hash();
Expand All @@ -134,10 +136,11 @@ private ulong Hash() {
writer.Write(flag.Item2.Value);
}
}
writer.Write(suffix ?? string.Empty);
writer.Write(suffix);
writer.Write(volume);
writer.Write(velocity);
writer.Write(modulation);
writer.Write(direct);
writer.Write(leadingMs);
foreach (var point in envelope) {
writer.Write(point.X);
Expand Down
2 changes: 1 addition & 1 deletion OpenUtau.Core/Render/Worldline.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ public SynthRequestWrapper(ResamplerItem item) {
required_length = item.durRequired,
consonant = item.consonant,
cut_off = item.cutoff,
volume = item.volume,
volume = item.phone.direct ? 0 : item.volume,
modulation = item.modulation,
tempo = item.tempo,
pitch_bend_length = item.pitches.Length,
Expand Down

0 comments on commit 690afd5

Please sign in to comment.