Skip to content

Commit

Permalink
Merge pull request #896 from The-UTAU-Black-Supermarket/master
Browse files Browse the repository at this point in the history 
Better testing for phoneme attributes in multi-phoneme notes
  • Loading branch information
@stakira
stakira authored Nov 14, 2023
2 parents 282a583 + e7939c9 commit 64709da
Show file tree
Hide file tree
Showing 16 changed files with 10,294 additions and 47 deletions.
6 changes: 2 additions & 4 deletions OpenUtau.Plugin.Builtin/JapaneseCVVCPhonemizer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using OpenUtau.Api;
Expand Down Expand Up @@ -131,11 +131,9 @@ private bool checkOtoUntilHit(string[] input, Note note, out UOto oto) {

// checking VCs
// when VC does not exist, it will not be inserted
// TODO: fix duplicate voice color fallback bug (for now, this is better than nothing)
private bool checkOtoUntilHitVc(string[] input, Note note, out UOto oto) {
oto = default;
var attr = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == 0) ?? default;
var attr1 = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == 1) ?? default;
var attr = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == 1) ?? default;

var otos = new List<UOto>();
foreach (string test in input) {
Expand Down
3 changes: 1 addition & 2 deletions OpenUtau.Plugin.Builtin/JapanesePresampPhonemizer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,10 +334,9 @@ private bool checkOtoUntilHit(List<string> input, Note note, out UOto oto) {

// checking VCs
// when VC does not exist, it will not be inserted
// TODO: fix duplicate voice color fallback bug (for now, this is better than nothing)
private bool checkOtoUntilHitVc(List<string> input, Note note, out UOto oto) {
oto = default;
var attr = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == 0) ?? default;
var attr = note.phonemeAttributes?.FirstOrDefault(attr => attr.index == 1) ?? default;

var otos = new List<UOto>();
foreach (string test in input) {
Expand Down
67 changes: 54 additions & 13 deletions OpenUtau.Plugin.Builtin/SyllableBasedPhonemizer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System.IO;
using Serilog;
using System.Threading.Tasks;
using static OpenUtau.Api.Phonemizer;

namespace OpenUtau.Plugin.Builtin {
/// <summary>
Expand Down Expand Up @@ -60,9 +61,17 @@ protected struct Syllable {
/// </summary>
public int tone;
/// <summary>
/// Other phoneme attributes for VC and CC
/// </summary>
public PhonemeAttributes[] attr;
/// <summary>
/// tone for base "vowel" phoneme
/// </summary>
public int vowelTone;
/// <summary>
/// Other phoneme attributes for base "vowel" phoneme
/// </summary>
public PhonemeAttributes[] vowelAttr;

/// <summary>
/// 0 if no consonants are taken from previous word;
Expand Down Expand Up @@ -119,6 +128,10 @@ protected struct Ending {
/// the tone from last syllable, for all ending phonemes
/// </summary>
public int tone;
/// <summary>
/// Other phoneme attributes from last syllable
/// </summary>
public PhonemeAttributes[] attr;

// helpers
public bool IsEndingV => cc.Length == 0;
Expand Down Expand Up @@ -148,16 +161,44 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN

var phonemes = new List<Phoneme>();
foreach (var syllable in syllables) {
phonemes.AddRange(MakePhonemes(ProcessSyllable(syllable), mainNote.phonemeAttributes, phonemes.Count, syllable.duration, syllable.position,
syllable.tone, syllable.vowelTone, false));
phonemes.AddRange(MakePhonemes(ProcessSyllable(syllable), syllable.duration, syllable.position, false));
}
if (!nextNeighbour.HasValue) {
var tryEnding = MakeEnding(notes);
if (tryEnding.HasValue) {
var ending = tryEnding.Value;
phonemes.AddRange(MakePhonemes(ProcessEnding(ending), mainNote.phonemeAttributes, phonemes.Count, ending.duration, ending.position,
ending.tone, ending.tone, true));
phonemes.AddRange(MakePhonemes(ProcessEnding(ending), ending.duration, ending.position, true));
}
}

// Assign pitch/color suffixes
int noteIndex = 0;
for (int i = 0; i < phonemes.Count; i++) {
var attr = notes[0].phonemeAttributes?.FirstOrDefault(attr => attr.index == i) ?? default;
string alt = attr.alternate?.ToString() ?? string.Empty;
string color = attr.voiceColor;
int toneShift = attr.toneShift;
var phoneme = phonemes[i];
while (noteIndex < notes.Length - 1 && notes[noteIndex].position - notes[0].position < phoneme.position) {
noteIndex++;
}
var noteStartPosition = notes[noteIndex].position - notes[0].position;
int tone = (prevNeighbours != null && prevNeighbours.Length > 0 && phoneme.position < noteStartPosition) ?
prevNeighbours.Last().tone : (noteIndex > 0 && phoneme.position < noteStartPosition) ?
notes[noteIndex-1].tone : notes[noteIndex].tone;

var validatedAlias = phoneme.phoneme;
if (validatedAlias != null) {
validatedAlias = ValidateAliasIfNeeded(validatedAlias, tone + toneShift);
validatedAlias = MapPhoneme(validatedAlias, tone + toneShift, color, alt, singer);

phoneme.phoneme = validatedAlias;
} else {
phoneme.phoneme = null;
phoneme.position = 0;
}

phonemes[i] = phoneme;
}

return new Result() {
Expand Down Expand Up @@ -313,9 +354,11 @@ protected virtual Syllable[] MakeSyllables(Note[] inputNotes, Ending? prevEnding
cc = beginningCc.ToArray(),
v = symbols[firstVowelId],
tone = prevEndingValue.tone,
attr = prevEndingValue.attr,
duration = prevEndingValue.duration,
position = 0,
vowelTone = notes[0].tone,
vowelAttr = notes[0].phonemeAttributes,
prevWordConsonantsCount = prevEndingValue.cc.Count()
};
} else {
Expand All @@ -325,9 +368,11 @@ protected virtual Syllable[] MakeSyllables(Note[] inputNotes, Ending? prevEnding
cc = symbols.Take(firstVowelId).ToArray(),
v = symbols[firstVowelId],
tone = notes[0].tone,
attr = notes[0].phonemeAttributes,
duration = -1,
position = 0,
vowelTone = notes[0].tone
vowelTone = notes[0].tone,
vowelAttr = notes[0].phonemeAttributes
};
}

Expand All @@ -346,9 +391,11 @@ protected virtual Syllable[] MakeSyllables(Note[] inputNotes, Ending? prevEnding
cc = ccs.ToArray(),
v = symbols[lastSymbolI],
tone = notes[noteI - 1].tone,
attr = notes[noteI - 1].phonemeAttributes,
duration = notes[noteI - 1].duration,
position = position,
vowelTone = notes[noteI].tone,
vowelAttr = notes[noteI].phonemeAttributes,
canAliasBeExtended = true // for all not-first notes is allowed
};
ccs = new List<string>();
Expand Down Expand Up @@ -378,6 +425,7 @@ protected virtual Syllable[] MakeSyllables(Note[] inputNotes, Ending? prevEnding
prevV = symbols[vowelIds.Last()],
cc = symbols.Skip(vowelIds.Last() + 1).ToArray(),
tone = notes.Last().tone,
attr = notes.Last().phonemeAttributes,
duration = notes.Skip(vowelIds.Length - 1).Sum(n => n.duration),
position = notes.Sum(n => n.duration)
};
Expand Down Expand Up @@ -695,20 +743,14 @@ private List<int> ExtractVowels(string[] symbols) {
return vowelIds;
}

private Phoneme[] MakePhonemes(List<string> phonemeSymbols, PhonemeAttributes[] phonemeAttributes, int phonemesOffset,
int containerLength, int position, int tone, int lastTone, bool isEnding) {
private Phoneme[] MakePhonemes(List<string> phonemeSymbols, int containerLength, int position, bool isEnding) {

var phonemes = new Phoneme[phonemeSymbols.Count];
for (var i = 0; i < phonemeSymbols.Count; i++) {
var phonemeI = phonemeSymbols.Count - i - 1;
var attr = phonemeAttributes?.FirstOrDefault(attr => attr.index == phonemesOffset + phonemeI) ?? default;
var currentTone = phonemeI == phonemeSymbols.Count - 1 ? lastTone : tone;

var validatedAlias = phonemeSymbols[phonemeI];
if (validatedAlias != null) {
validatedAlias = ValidateAliasIfNeeded(validatedAlias, currentTone + attr.toneShift);
validatedAlias = MapPhoneme(validatedAlias, currentTone + attr.toneShift, attr.voiceColor, attr.alternate?.ToString() ?? string.Empty, singer);

phonemes[phonemeI].phoneme = validatedAlias;
var transitionLengthTick = MsToTick(GetTransitionBasicLengthMs(phonemes[phonemeI].phoneme));
if (i == 0) {
Expand All @@ -723,7 +765,6 @@ private Phoneme[] MakePhonemes(List<string> phonemeSymbols, PhonemeAttributes[]
} else {
phonemes[phonemeI].phoneme = null;
phonemes[phonemeI].position = 0;
phonemesOffset -= 1;
}
}

Expand Down
11 changes: 11 additions & 0 deletions OpenUtau.Test/Files/en_delta0/character.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
subbanks:
- color: ""
prefix: ""
suffix: _H
tone_ranges:
- G4-B7
- color: ""
prefix: ""
suffix: _F3
tone_ranges:
- C1-F#4
Loading

0 comments on commit 64709da

Please sign in to comment.