Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixed bug in KoreanPhonemizerUtil, KO CBNN Phonemizer #1121

Merged
merged 7 commits into from
May 10, 2024
Merged

Fixed bug in KoreanPhonemizerUtil, KO CBNN Phonemizer #1121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c17dfcf
refactored Romanize function
EX3exp Apr 29, 2024
59b48e5
Fixed bug that generates " " instead of "ㅇ" in first consonants
EX3exp Apr 30, 2024
d38e0b7
Deleted deprecated function in CV,CBNN
EX3exp Apr 30, 2024
70eeff4
Fixed bug in CBNN that not generates VC with semivowel
EX3exp Apr 30, 2024
1b507f3
fixed typo
EX3exp Apr 30, 2024
a4b623d
fixed typo
EX3exp Apr 30, 2024
bca3ce9
Fixed to use alternative VCs if needed
EX3exp Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 2 additions & 66 deletions OpenUtau.Core/Enunu/EnunuKoreanPhonemizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ public enum BatchimType{
}

Dictionary<Note[], Phoneme[]> partResult = new Dictionary<Note[], Phoneme[]>();

public override void SetUp(Note[][] notes, UProject project, UTrack track) {
partResult.Clear();
if (notes.Length == 0 || singer == null || !singer.Found) {
Expand Down Expand Up @@ -484,7 +484,7 @@ static Phoneme[] ParseLabel(string path) {
}

protected override EnunuNote[] NoteGroupsToEnunu(Note[][] notes) {
KoreanPhonemizerUtil.RomanizeNotes(notes, FirstConsonants, MiddleVowels, LastConsonants, semivowelSep);
KoreanPhonemizerUtil.RomanizeNotes(notes, true, FirstConsonants, MiddleVowels, LastConsonants, semivowelSep);
var result = new List<EnunuNote>();
int position = 0;
int index = 0;
Expand Down Expand Up @@ -513,69 +513,6 @@ protected override EnunuNote[] NoteGroupsToEnunu(Note[][] notes) {
return result.ToArray();
}

// public void AdjustPos(Phoneme[] phonemes, Note[] prevNote){
// //TODO
// Phoneme? prevPhone = null;
// Phoneme? nextPhone = null;
// Phoneme currPhone;

// int length = phonemes.Last().position;
// int prevLength;
// if (prevNote == null){
// prevLength = length;
// }
// else{
// prevLength = MsToTick(prevNote.Sum(n => n.duration));
// }

// for (int i=0; i < phonemes.Length; i++) {
// currPhone = phonemes[i];
// if (i < phonemes.Length - 1){
// nextPhone = phonemes[i+1];
// }
// else{
// nextPhone = null;
// }

// if (i == 0){
// // TODO 받침 + 자음 오면 받침길이 + 자음길이 / 2의 위치에 자음이 오도록 하기
// if (isPlainVowel(phonemes[i].phoneme)) {
// phonemes[i].position = 0;
// }
// else if (nextPhone != null && ! isPlainVowel(((Phoneme)nextPhone).phoneme) && ! isSemivowel(((Phoneme)nextPhone).phoneme) && isPlainVowel(((Phoneme)nextPhone).phoneme) && isSemivowel(currPhone.phoneme)) {
// phonemes[i + 1].position = length / 10;
// }
// else if (nextPhone != null && isSemivowel(((Phoneme)nextPhone).phoneme)){
// if (i + 2 < phonemes.Length){
// phonemes[i + 2].position = length / 10;
// }

// }
// }
// prevPhone = currPhone;
// }
// }

// private bool isPlainVowel(string phoneme){
// if (phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅏ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅣ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅜ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅔ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅗ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅡ") || phoneme == koreanENUNUSetting.GetPlainVowelPhoneme("ㅓ")){
// return true;
// }
// return false;
// }

// private bool isBatchim(string phoneme){
// if (phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㄱ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㄴ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㄷ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㄹ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㅁ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㅂ") || phoneme == koreanENUNUSetting.GetFinalConsonantPhoneme("ㅇ")){
// return true;
// }
// return false;
// }

// private bool isSemivowel(string phoneme) {
// if (phoneme == koreanENUNUSetting.GetSemiVowelPhoneme("w") || phoneme == koreanENUNUSetting.GetSemiVowelPhoneme("y")){
// return true;
// }
// return false;
// }
public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, Note[] prevs) {
if (partResult.TryGetValue(notes, out var phonemes)) {
var phonemes_ = phonemes.Select(p => {
Expand All @@ -584,7 +521,6 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN
return p;
}).ToArray();

//AdjustPos(phonemes_, prevs);
return new Result {
phonemes = phonemes_,
};
Expand Down
111 changes: 53 additions & 58 deletions OpenUtau.Core/KoreanPhonemizerUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,20 +177,20 @@ public static Hashtable Separate(string character) {
}

/// <summary>
/// merges separated hangeul into complete hangeul. (Example: {[0]: "ㄱ", [1]: "ㅏ", [2]: " "} => "가"})
/// merges separated hangeul into complete hangeul. (Example: {[offset + 0]: "ㄱ", [offset + 1]: "ㅏ", [offset + 2]: " "} => "가"})
/// <para>자모로 쪼개진 한글을 합쳐진 한글로 반환합니다.</para>
/// </summary>
/// <param name="separated">separated Hangeul. </param>
/// <returns>Returns complete Hangeul Character.</returns>
public static string Merge(Hashtable separatedHangeul){
public static string Merge(Hashtable separatedHangeul, int offset = 0){

int firstConsonantIndex; // (ex) 2
int middleVowelIndex; // (ex) 2
int lastConsonantIndex; // (ex) 21

char firstConsonant = ((string)separatedHangeul[0])[0]; // (ex) "ㄴ"
char middleVowel = ((string)separatedHangeul[1])[0]; // (ex) "ㅑ"
char lastConsonant = ((string)separatedHangeul[2])[0]; // (ex) "ㅇ"
char firstConsonant = ((string)separatedHangeul[offset + 0])[0]; // (ex) "ㄴ"
char middleVowel = ((string)separatedHangeul[offset + 1])[0]; // (ex) "ㅑ"
char lastConsonant = ((string)separatedHangeul[offset + 2])[0]; // (ex) "ㅇ"

if (firstConsonant == ' ') {firstConsonant = 'ㅇ';}

Expand Down Expand Up @@ -371,7 +371,7 @@ private static Hashtable Variate(Hashtable firstCharSeparated, Hashtable nextCha
nextFirstConsonant = "ㅇ";
}

if ((!firstLastConsonant.Equals("")) && nextFirstConsonant.Equals("ㅇ") && (!firstLastConsonant.Equals("ㅇ"))) {
if ((!firstLastConsonant.Equals(" ")) && nextFirstConsonant.Equals("ㅇ") && (!firstLastConsonant.Equals("ㅇ"))) {
// 연음 2
nextFirstConsonant = firstLastConsonant;
firstLastConsonant = " ";
Expand Down Expand Up @@ -919,10 +919,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, thisNoteSeparated[4]);
result.Add(8, thisNoteSeparated[5]);

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
}
else if ((lyrics[0] != null) && (lyrics[2] == null)) {
Expand All @@ -941,10 +938,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, "null");
result.Add(8, "null");

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
else if (whereYeonEum == 0) {
// 앞 노트에서 단어가 끝났다고 가정
Expand All @@ -959,10 +953,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, "null");
result.Add(8, "null");

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
else {
Hashtable result = Variate(lyrics[0], lyrics[1], 0); // 첫 글자
Expand All @@ -976,10 +967,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, "null");
result.Add(8, "null");

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
}
else if ((lyrics[0] != null) && (lyrics[2] != null)) {
Expand All @@ -998,10 +986,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, thisNoteSeparated[4]);
result.Add(8, thisNoteSeparated[5]);

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
else if (whereYeonEum == 0) {
// 앞 노트에서 단어가 끝났다고 가정 / 릎. [위] 놓
Expand All @@ -1016,10 +1001,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, thisNoteSeparated[4]);
result.Add(8, thisNoteSeparated[5]);

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
else {
Hashtable result = Variate(lyrics[0], lyrics[1], 0);
Expand All @@ -1033,10 +1015,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, thisNoteSeparated[4]);
result.Add(8, thisNoteSeparated[5]);

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]});
return Merge(result, 3);
}
}
else {
Expand All @@ -1060,11 +1039,7 @@ public static String Variate(String? prevNeighbour, String note, String? nextNei
result.Add(7, "null");
result.Add(8, "null");

return Merge(new Hashtable{
[0] = (string)result[3],
[1] = (string)result[4],
[2] = (string)result[5]
});
return Merge(result, 3);
}
}

Expand All @@ -1081,21 +1056,41 @@ public static Note[] ChangeLyric(Note[] group, string lyric) {
};
return group;
}
public static void RomanizeNotes(Note[][] groups, Dictionary<string, string[]> firstConsonants, Dictionary<string, string[]> vowels, Dictionary<string, string[]> lastConsonants, string semivowelSeparator=" ") {
// for ENUNU Phonemizer


public static void ModifyLyrics(Hashtable lyricSeparated,string lyric, Dictionary<string, string[]> firstConsonants, Dictionary<string, string[]> vowels, Dictionary<string, string[]> lastConsonants, string semivowelSeparator){
lyric += firstConsonants[(string)lyricSeparated[3]][0];
if (vowels[(string)lyricSeparated[4]][1] != "") {
// this vowel contains semivowel
lyric += semivowelSeparator + vowels[(string)lyricSeparated[4]][1] + vowels[(string)lyricSeparated[4]][2];
}
else{
lyric += " " + vowels[(string)lyricSeparated[4]][2];
}

lyric += lastConsonants[(string)lyricSeparated[5]][0];
}

public static void RomanizeNotes(Note[][] groups, bool _modifyLyrics = false, Dictionary<string, string[]> firstConsonants = null, Dictionary<string, string[]> vowels = null, Dictionary<string, string[]> lastConsonants = null, string semivowelSeparator = " ") {
// for ENUNU & DIFFS Phonemizer

int noteIdx = 0;
string lyric;
bool modifyLyrics = (!_modifyLyrics || firstConsonants == null || vowels == null || lastConsonants == null) ? false : true;

Note[] currentNote;
Note[]? prevNote = null;
Note[]? nextNote;

Note? prevNote_;
Note? nextNote_;


List<string> ResultLyrics = new List<string>();

foreach (Note[] group in groups){
currentNote = groups[noteIdx];
string originalLyric; // uses this when no variation needed
originalLyric = currentNote[0].lyric;

if (groups.Length > noteIdx + 1 && IsHangeul(groups[noteIdx + 1][0].lyric)) {
nextNote = groups[noteIdx + 1];
}
Expand All @@ -1120,7 +1115,7 @@ public static void RomanizeNotes(Note[][] groups, Dictionary<string, string[]> f
}
else{nextNote_ = null;}

string lyric = "";
lyric = originalLyric;

if (! IsHangeul(currentNote[0].lyric)){
ResultLyrics.Add(currentNote[0].lyric);
Expand All @@ -1129,27 +1124,27 @@ public static void RomanizeNotes(Note[][] groups, Dictionary<string, string[]> f
continue;
}

Hashtable lyricSeparated = Variate(prevNote_, currentNote[0], nextNote_);
lyric += firstConsonants[(string)lyricSeparated[3]][0];
if (vowels[(string)lyricSeparated[4]][1] != "") {
// this vowel contains semivowel
lyric += semivowelSeparator + vowels[(string)lyricSeparated[4]][1] + vowels[(string)lyricSeparated[4]][2];
}
else{
lyric += " " + vowels[(string)lyricSeparated[4]][2];
}

lyric += lastConsonants[(string)lyricSeparated[5]][0];

Hashtable lyricSeparated = Variate(prevNote_, currentNote[0], nextNote_);

ResultLyrics.Add(lyric.Trim());
if (modifyLyrics) {
ModifyLyrics(lyricSeparated, lyric, firstConsonants, vowels, lastConsonants, semivowelSeparator);
}
else {
lyric = Merge(lyricSeparated, 3);
}

ResultLyrics.Add(lyric.Trim());

prevNote = currentNote;
prevNote = currentNote;

noteIdx++;
noteIdx++;

}
Enumerable.Zip(groups, ResultLyrics.ToArray(), ChangeLyric).Last();
}


/// <summary>
/// abstract class for Ini Management
/// To use, child phonemizer should implement this class(BaseIniManager) with its own setting values!
Expand Down
25 changes: 15 additions & 10 deletions OpenUtau.Plugin.Builtin/KoreanCBNNPhonemizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,11 +141,11 @@ private Result ConvertForCBNN(Note[] notes, string[] prevLyric, string[] thisLyr
}
string frontCV;
string batchim;
string VC = $"{thisMidVowelTail} {FIRST_CONSONANTS[nextLyric[0]]}";
string VC = $"{thisMidVowelTail} {FIRST_CONSONANTS[nextLyric[0]]}{MIDDLE_VOWELS[nextLyric[1]][1]}";
string VV = $"{MIDDLE_VOWELS[prevLyric[1]][2]} {thisMidVowelTail}";
string VSv = $"{thisMidVowelTail} {MIDDLE_VOWELS[nextLyric[1]][1]}";
isItNeedsVSv = thisLyric[2] == " " && nextLyric[0] == "ㅇ" && !PLAIN_VOWELS.Contains(nextLyric[1]) && FindInOto(VSv, note, true) != null;
isItNeedsVC = thisLyric[2] == " " && nextLyric[0] != "ㅇ" && nextLyric[0] != "null" && FindInOto(VC, note, true) != null;
isItNeedsVC = thisLyric[2] == " " && nextLyric[0] != "ㅇ" && nextLyric[0] != "null";

frontCV = $"- {CV}";
if (FindInOto(frontCV, note, true) == null) {
Expand All @@ -155,6 +155,15 @@ private Result ConvertForCBNN(Note[] notes, string[] prevLyric, string[] thisLyr
}
}

if (FindInOto(VC, note, true) == null) {
if (VC.EndsWith("w") || VC.EndsWith("y")) {
VC = VC.Substring(0, VC.Length - 1);
}
if (FindInOto(VC, note, true) == null) {
isItNeedsVC = false;
}
}

if (isItNeedsVV) {CV = VV;}


Expand Down Expand Up @@ -214,26 +223,22 @@ private Result ConvertForCBNN(Note[] notes, string[] prevLyric, string[] thisLyr
}


private string HandleEmptyFirstConsonant(string lyric) {
return lyric == " " ? "ㅇ" : lyric;
}

public override Result ConvertPhonemes(Note[] notes, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, Note[] prevNeighbours) {
Note note = notes[0];

Hashtable lyrics = KoreanPhonemizerUtil.Variate(prevNeighbour, note, nextNeighbour);
string[] prevLyric = new string[]{ // "ㄴ", "ㅑ", "ㅇ"
HandleEmptyFirstConsonant((string)lyrics[0]),
(string)lyrics[0],
(string)lyrics[1],
(string)lyrics[2]
};
string[] thisLyric = new string[]{ // "ㄴ", "ㅑ", "ㅇ"
HandleEmptyFirstConsonant((string)lyrics[3]),
(string)lyrics[3],
(string)lyrics[4],
(string)lyrics[5]
};
string[] nextLyric = new string[]{ // "ㄴ", "ㅑ", "ㅇ"
HandleEmptyFirstConsonant((string)lyrics[6]),
(string)lyrics[6],
(string)lyrics[7],
(string)lyrics[8]
};
Expand All @@ -257,7 +262,7 @@ public override Result GenerateEndSound(Note[] notes, Note? prev, Note? next, No
Hashtable lyrics = KoreanPhonemizerUtil.Separate(prevNeighbour_.lyric);

string[] prevLyric = new string[]{ // "ㄴ", "ㅑ", "ㅇ"
HandleEmptyFirstConsonant((string)lyrics[0]),
(string)lyrics[0],
(string)lyrics[1],
(string)lyrics[2]
};
Expand Down
Loading
Loading