Merge branch 'feature/charset' into develop

ztl8702 · Dec 17, 2015 · d4015e5 · d4015e5
2 parents 8dde91a + 1b9ea97
commit d4015e5
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 11 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -39,7 +39,7 @@ set(Boost_USE_MULTITHREADED ON)
 if(MSVC)
   set(Boost_USE_STATIC_RUNTIME ON)
 endif(MSVC)
-find_package(Boost 1.46.0 REQUIRED COMPONENTS filesystem regex signals system)
+find_package(Boost 1.46.0 REQUIRED COMPONENTS filesystem regex signals system locale)
 if(Boost_FOUND)
   include_directories(${Boost_INCLUDE_DIRS})
   link_directories(${Boost_LIBRARY_DIRS})

diff --git a/include/rime/gear/charset_filter.h b/include/rime/gear/charset_filter.h
@@ -15,14 +15,15 @@ namespace rime {
 
 class CharsetFilterTranslation : public Translation {
  public:
-  CharsetFilterTranslation(an<Translation> translation);
+  CharsetFilterTranslation(an<Translation> translation, const string& charset = "");
   virtual bool Next();
   virtual an<Candidate> Peek();
 
  protected:
   bool LocateNextCandidate();
 
   an<Translation> translation_;
+  string charset_;
 };
 
 struct DictEntry;
@@ -39,7 +40,7 @@ class CharsetFilter : public Filter, TagMatching {
   }
 
   // return true to accept, false to reject the tested item
-  static bool FilterText(const string& text);
+  static bool FilterText(const string& text, const string& charset = "");
   static bool FilterDictEntry(an<DictEntry> entry);
 };
 

diff --git a/src/gear/charset_filter.cc b/src/gear/charset_filter.cc
@@ -12,6 +12,7 @@
 #include <rime/engine.h>
 #include <rime/dict/vocabulary.h>
 #include <rime/gear/charset_filter.h>
+#include <boost/locale/encoding.hpp>
 
 namespace rime {
 
@@ -45,8 +46,8 @@ bool contains_extended_cjk(const string& text)
 // CharsetFilterTranslation
 
 CharsetFilterTranslation::CharsetFilterTranslation(
-    an<Translation> translation)
-    : translation_(translation) {
+    an<Translation> translation, const string& charset)
+    : translation_(translation), charset_(charset) {
   LocateNextCandidate();
 }
 
@@ -67,7 +68,7 @@ an<Candidate> CharsetFilterTranslation::Peek() {
 bool CharsetFilterTranslation::LocateNextCandidate() {
   while (!translation_->exhausted()) {
     auto cand = translation_->Peek();
-    if (cand && CharsetFilter::FilterText(cand->text()))
+    if (cand && CharsetFilter::FilterText(cand->text(), charset_))
       return true;
     translation_->Next();
   }
@@ -77,8 +78,17 @@ bool CharsetFilterTranslation::LocateNextCandidate() {
 
 // CharsetFilter
 
-bool CharsetFilter::FilterText(const string& text) {
-  return !contains_extended_cjk(text);
+bool CharsetFilter::FilterText(const string& text, const string& charset) {
+  if (charset.empty()) return !contains_extended_cjk(text);
+  try {
+    boost::locale::conv::from_utf(text, charset, boost::locale::conv::method_type::stop);
+  }
+  catch(boost::locale::conv::conversion_error const&  ex) {
+    return false;
+  }
+  catch(...) {
+  }
+  return true;
 }
 
 bool CharsetFilter::FilterDictEntry(an<DictEntry> entry) {
@@ -91,10 +101,13 @@ CharsetFilter::CharsetFilter(const Ticket& ticket)
 
 an<Translation> CharsetFilter::Apply(
     an<Translation> translation, CandidateList* candidates) {
-  if (engine_->context()->get_option("extended_charset")) {
-    return translation;
+  if (name_space_.empty() && !engine_->context()->get_option("extended_charset")) {
+    return New<CharsetFilterTranslation>(translation);
+  }
+  if (!name_space_.empty() && engine_->context()->get_option(name_space_)) {
+    return New<CharsetFilterTranslation>(translation, name_space_);
   }
-  return New<CharsetFilterTranslation>(translation);
+  return translation;
 }
 
 }  // namespace rime