From 335a327f47a5ec766f00c45187ded0bc08af7f3d Mon Sep 17 00:00:00 2001
From: "Navaneeth.K.N" <navaneethkn@gmail.com>
Date: Sat, 28 Jul 2012 22:46:25 +0530
Subject: [PATCH] Fixing rendering.

This also introduces ability to create tags in the mapping file.
---
 CMakeLists.txt                      |   2 +-
 renderer/ml_unicode.c               |  73 ++++++++++++
 {rendering => renderer}/renderers.h |  18 ++-
 rendering.c                         | 132 ++++++++++++++++------
 rendering.h                         |   6 +-
 rendering/ml_unicode.c              |  73 ------------
 schemes/ml-unicode                  |  23 ++--
 schemes/varnam-compile              |  24 ++--
 tests/03-vst-compilation.c          |  67 ++++++-----
 varnam-api.h                        |   9 ++
 varnam-symbol-table.c               | 169 +++-------------------------
 varnam-symbol-table.h               |  33 +-----
 varnam-tl.c                         |  29 ++---
 varnam-token.c                      |   2 +-
 varnam-types.h                      |  17 ++-
 varnam.c                            |  49 ++++++--
 16 files changed, 341 insertions(+), 385 deletions(-)
 create mode 100644 renderer/ml_unicode.c
 rename {rendering => renderer}/renderers.h (71%)
 delete mode 100644 rendering/ml_unicode.c

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 184eb83..1594e39 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,7 +38,7 @@ list (APPEND VARNAM_SOURCES
   vword.c
   learn.c
   rendering.c
-  rendering/ml_unicode.c
+  renderer/ml_unicode.c
   varnam.c
   )
 
diff --git a/renderer/ml_unicode.c b/renderer/ml_unicode.c
new file mode 100644
index 0000000..ba605db
--- /dev/null
+++ b/renderer/ml_unicode.c
@@ -0,0 +1,73 @@
+/* ml_unicode.c
+
+Copyright (C) 2010 Navaneeth.K.N
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#include <string.h>
+
+#include "renderers.h"
+#include "../varnam-util.h"
+#include "../varnam-types.h"
+#include "../varnam-result-codes.h"
+#include "../varnam-symbol-table.h"
+
+#define RENDER_VALUE2_TAG "render_value2"
+#define CHIL_TAG "chill"
+
+int
+ml_unicode_renderer(varnam *handle,
+                    vtoken *previous,
+                    vtoken *current,
+                    strbuf *output)
+{
+    int rc;
+    vtoken *virama;
+
+    rc = vst_get_virama (handle, &virama);
+    if (rc) return rc;
+
+    /* Not sure about this */
+    /* if(strcmp(match->pattern, "r") == 0 || strcmp(match->pattern, "R") == 0) { */
+    /*     if(handle->internal->last_token_available && !strbuf_endswith(output, virama->value1)) { */
+    /*         strbuf_add(output, "ര്‍"); */
+    /*         return VARNAM_SUCCESS; */
+    /*     } */
+    /* } */
+
+    if (strcmp(current->tag, RENDER_VALUE2_TAG) == 0 && previous != NULL)
+    {
+        strbuf_add(output, current->value2);
+        return VARNAM_SUCCESS;
+    }
+
+    return VARNAM_PARTIAL_RENDERING;
+}
+
+int
+ml_unicode_rtl_renderer(varnam *handle,
+                        vtoken *previous,
+                        vtoken *current,
+                        strbuf *output)
+{
+    if (strcmp(current->tag, CHIL_TAG) == 0) {
+        strbuf_add (output, current->pattern);
+        strbuf_add (output, "_");
+        return VARNAM_SUCCESS;
+    }
+
+    return VARNAM_PARTIAL_RENDERING;
+}
diff --git a/rendering/renderers.h b/renderer/renderers.h
similarity index 71%
rename from rendering/renderers.h
rename to renderer/renderers.h
index e520100..feed612 100644
--- a/rendering/renderers.h
+++ b/renderer/renderers.h
@@ -17,12 +17,22 @@ License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
 
-#include "../varnam-types.h"
-
 #ifndef RENDERING_RENDERERS_H_INCLUDED_105305
 #define RENDERING_RENDERERS_H_INCLUDED_105305
 
-int ml_unicode_renderer(varnam*, struct token*, struct strbuf*);
-int ml_unicode_rtl_renderer(varnam*, struct token*, struct strbuf*);
+#include "../varnam-types.h"
+#include "../varnam-util.h"
+
+int
+ml_unicode_renderer(varnam *handle,
+                    vtoken *previous,
+                    vtoken *current,
+                    strbuf *output);
+
+int
+ml_unicode_rtl_renderer(varnam *handle,
+                        vtoken *previous,
+                        vtoken *current,
+                        strbuf *output);
 
 #endif
diff --git a/rendering.c b/rendering.c
index d27f8d5..3124f31 100644
--- a/rendering.c
+++ b/rendering.c
@@ -17,6 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA */
 
 #include <assert.h>
+#include <string.h>
 
 #include "varnam-symbol-table.h"
 #include "varnam-result-codes.h"
@@ -24,31 +25,33 @@
 #include "varnam-types.h"
 #include "rendering.h"
 #include "vword.h"
+#include "varnam-api.h"
+#include "varnam-array.h"
 
-struct varnam_token_rendering*
-get_additional_rendering_rule(varnam *handle)
+static vtoken_renderer*
+get_renderer(varnam *handle)
 {
-    /* struct varnam_token_rendering *tr; */
-    /* int i; */
-
-    if(handle->internal->renderers == NULL) return NULL;
-
-    /* Will be fixed when this module is touched */
-    /* if(handle->internal->scheme_identifier[0] == '\0') { */
-    /*     fill_general_values(handle, handle->internal->scheme_identifier, "scheme_identifier"); */
-    /* } */
-
-    /* Will be fixed when this module is touched */
-    /* for(i = 0; i < 1; i++) */
-    /* { */
-    /*     tr = &(handle->internal->renderers[i]); */
-    /*     if(strcmp(tr->scheme_identifier, handle->internal->scheme_identifier) == 0) */
-    /*         return tr; */
-    /* } */
+    vtoken_renderer *r;
+    int i;
+
+    const char *scheme_id = varnam_get_scheme_identifier (handle);
+    if (scheme_id == NULL) {
+        varnam_log (handle, "Scheme id is not set. Custom rendering will not be processed");
+        return NULL;
+    }
+
+    for (i = 0; i < varray_length (v_->renderers); i++)
+    {
+        r = varray_get (v_->renderers, i);
+        assert (r);
+        if (strcmp(r->scheme_id, scheme_id) == 0)
+            return r;
+    }
+
     return NULL;
 }
 
-/* Resolves the tokens. 
+/* Resolves the tokens.
  * tokens will be a single dimensional array where each item is vtoken instances
  */
 int
@@ -56,9 +59,9 @@ resolve_tokens(varnam *handle,
                varray *tokens,
                vword **word)
 {
-    vtoken *virama, *token, *previous;
+    vtoken *virama, *token = NULL, *previous = NULL;
     strbuf *string;
-    struct varnam_token_rendering *rule;
+    vtoken_renderer *r;
     int rc, i;
 
     assert(handle);
@@ -67,28 +70,30 @@ resolve_tokens(varnam *handle,
     if (rc)
         return rc;
 
-    /* will be fixed after implementing register_renderer() */
-    /* rule = get_additional_rendering_rule(handle); */
-    /* if(rule != NULL) { */
-    /*     rc = rule->render(handle, match, string); */
-    /*     if(rc == VARNAM_SUCCESS) { */
-    /*         return; */
-    /*     } */
-    /* } */
-
     string = get_pooled_string (handle);
     for(i = 0; i < varray_length(tokens); i++)
     {
         token = varray_get (tokens, i);
-        if (token->type == VARNAM_TOKEN_VIRAMA) 
+
+        r = get_renderer (handle);
+        if (r != NULL)
         {
-            /* we are resolving a virama. If the output ends with a virama already, add a 
+            rc = r->rtl (handle, previous, token, string);
+            if (rc == VARNAM_ERROR)
+                return rc;
+            if (rc == VARNAM_SUCCESS)
+                continue;
+        }
+
+        if (token->type == VARNAM_TOKEN_VIRAMA)
+        {
+            /* we are resolving a virama. If the output ends with a virama already, add a
                ZWNJ to it, so that following character will not be combined.
                if output not ends with virama, add a virama and ZWNJ */
             if(strbuf_endswith (string, virama->value1)) {
                 strbuf_add (string, ZWNJ());
             }
-            else {            
+            else {
                 strbuf_add (string, virama->value1);
                 strbuf_add (string, ZWNJ());
             }
@@ -119,3 +124,62 @@ resolve_tokens(varnam *handle,
     *word = get_pooled_word (handle, strbuf_to_s (string), 1);
     return VARNAM_SUCCESS;
 }
+
+/*
+ * Resolve tokens for reverse transliteration. tokens will be multidimensional array
+ */
+int
+resolve_rtl_tokens(varnam *handle,
+                  varray *all_tokens,
+                  char **output)
+{
+    int rc, i, j;
+    vtoken_renderer *r;
+    strbuf *rtl;
+    vtoken *token = NULL, *previous = NULL;
+    varray *tokens;
+
+    assert (handle);
+    assert (all_tokens);
+
+    rtl = get_pooled_string (handle);
+    assert (rtl);
+    for (i = 0; i < varray_length (all_tokens); i++)
+    {
+        tokens = varray_get (all_tokens, i);
+        assert (tokens);
+        for (j = 0; j < varray_length (tokens); j++)
+        {
+            token = varray_get (tokens, j);
+            assert (token);
+
+            r = get_renderer (handle);
+            if (r != NULL)
+            {
+                rc = r->rtl (handle, previous, token, rtl);
+                if (rc == VARNAM_ERROR)
+                    return rc;
+                if (rc == VARNAM_SUCCESS) {
+                    previous = token;
+                    break;
+                }
+            }
+
+            strbuf_add (rtl, token->pattern);
+
+            /* vowel is standing in it's full form in between a word. need to prefix _
+               to avoid unnecessary conjunctions */
+            if (token->type == VARNAM_TOKEN_VOWEL && previous != NULL)
+            {
+                if (strcmp(token->value1, previous->value1) == 0)
+                    strbuf_add (rtl, "_");
+            }
+
+            previous = token;
+            break; /* We only care about first element in each array */
+        }
+    }
+
+    *output = rtl->buffer;
+    return VARNAM_SUCCESS;
+}
diff --git a/rendering.h b/rendering.h
index 2b1daf0..79c0eae 100644
--- a/rendering.h
+++ b/rendering.h
@@ -27,7 +27,9 @@ resolve_tokens(varnam *handle,
                varray *tokens,
                vword **word);
 
-struct varnam_token_rendering*
-get_additional_rendering_rule(varnam *handle);
+int
+resolve_rtl_tokens(varnam *handle,
+                  varray *tokens,
+                  char **output);
 
 #endif
diff --git a/rendering/ml_unicode.c b/rendering/ml_unicode.c
deleted file mode 100644
index 963eaf7..0000000
--- a/rendering/ml_unicode.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* ml_unicode.c
-
-Copyright (C) 2010 Navaneeth.K.N
-
-This library is free software; you can redistribute it and/or
-modify it under the terms of the GNU Lesser General Public
-License as published by the Free Software Foundation; either
-version 2.1 of the License, or (at your option) any later version.
-
-This library is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-*/
-
-#include <string.h>
-
-#include "renderers.h"
-#include "../varnam-util.h"
-#include "../varnam-types.h"
-#include "../varnam-result-codes.h"
-
-#define CHIL_TAG "chil"
-
-int 
-ml_unicode_renderer(varnam *handle, 
-                    struct token *match, 
-                    struct strbuf *output)
-{
-    struct token *virama;
-    vst_get_virama (handle, &virama);
-
-    if(strcmp(match->pattern, "r") == 0 || strcmp(match->pattern, "R") == 0) {
-        if(handle->internal->last_token_available && !strbuf_endswith(output, virama->value1)) {
-            strbuf_add(output, "ര്‍");
-            return VARNAM_SUCCESS;
-        }
-    }
-
-    if(strcmp(match->tag, "nj") == 0) {
-        if(handle->internal->last_token_available) {
-            strbuf_add(output, match->value2);
-            return VARNAM_SUCCESS;
-        }
-    }
-
-    if(strcmp(match->tag, "ng") == 0) {
-        if(handle->internal->last_token_available) {
-            strbuf_add(output, match->value2);
-            return VARNAM_SUCCESS;
-        }
-    }
-   
-    return VARNAM_PARTIAL_RENDERING;
-}
-
-int 
-ml_unicode_rtl_renderer(varnam *handle, 
-                        struct token *match, 
-                        struct strbuf *output)
-{
-    if (strcmp(match->tag, CHIL_TAG) == 0) {
-        strbuf_add (output, match->pattern);
-        strbuf_add (output, "_");
-        return VARNAM_SUCCESS;
-    }
-   
-    return VARNAM_PARTIAL_RENDERING;
-}
diff --git a/schemes/ml-unicode b/schemes/ml-unicode
index aa4a0a4..571a090 100644
--- a/schemes/ml-unicode
+++ b/schemes/ml-unicode
@@ -27,14 +27,12 @@ consonants ["ka", "ca", "qa"] => "ക",
            ["kha", ["gha"]] => "ഖ",
            "ga"  => "ഗ",
            ["gha", ["kha"]] => "ഘ",
-           "nga" => "ങ",
            ["NGa", ["nga"]] =>  "ങ്ങ",
            "cha" => "ച",
            ["CHa", ["cha"]] => "ഛ",
            [["cha"]] => "ച്ഛ",
            "ja"  => "ജ",
-           ["jha", "JHa"] => "ഝ",
-           "nja" => "ഞ",
+           ["jha", "JHa"] => "ഝ",           
            ["NJa", ["nja"]] => "ഞ്ഞ",
            ["ta", ["tta"]] => "റ്റ",
            ["Ta", ["ta", "da"]] => "ട",
@@ -59,6 +57,11 @@ consonants ["ka", "ca", "qa"] => "ക",
            "zha" => "ഴ",
            "xa" => "ക്സ",
            ["ksha", "Xa"] => "ക്ഷ"
+           
+tag "render_value2" do
+   consonants "nja" => ["ഞ", "ഞ്ഞ"],
+              "nga" => ["ങ", "ങ്ങ"]
+end           
 
 consonants "nka" => "ങ്ക",
            ["ncha", ["nja"]] => "ഞ്ച",
@@ -118,12 +121,14 @@ symbols "/" => "ഽ",
 
 # Chillaksharams are specified as consonant because it is a special form of consonant
 # value1 = atomic chil. value2 = old style
-consonants "m" => "ം",
-           "n" => ["ൻ", "ന്‍"],
-           ["N", ["n"]] => ["ൺ", "ണ്‍"],
-           "l" => ["ൽ", "ല്‍"],
-           ["L", ["l"]] => ["ൾ", "ള്‍"],
-           ["R", ["r"]] => ["ർ", "ര്‍"]
+tag "chill" do
+   consonants "m" => "ം",
+              "n" => ["ൻ", "ന്‍"],
+              ["N", ["n"]] => ["ൺ", "ണ്‍"],
+              "l" => ["ൽ", "ല്‍"],
+              ["L", ["l"]] => ["ൾ", "ള്‍"],
+              ["R", ["r"]] => ["ർ", "ര്‍"]
+end
 
 infer_dead_consonants false
 ignore_duplicates true
diff --git a/schemes/varnam-compile b/schemes/varnam-compile
index 9072e80..c0bb44c 100755
--- a/schemes/varnam-compile
+++ b/schemes/varnam-compile
@@ -51,13 +51,12 @@ module VarnamLibrary
   ffi_lib $options[:library]
 
   VARNAM_SYMBOL_MAX      = 30
-  VARNAM_TOKEN_TAG_MAX   = 15
 
   class Token < FFI::Struct
     layout :id, :int,
     :type, :int,
     :match_type, :int,
-    :tag, [:char, VARNAM_TOKEN_TAG_MAX],
+    :tag, [:char, VARNAM_SYMBOL_MAX],
     :pattern, [:char, VARNAM_SYMBOL_MAX],
     :value1, [:char, VARNAM_SYMBOL_MAX],
     :value2, [:char, VARNAM_SYMBOL_MAX],
@@ -65,7 +64,7 @@ module VarnamLibrary
   end
 
   attach_function :varnam_init, [:string, :pointer, :pointer], :int
-  attach_function :varnam_create_token, [:pointer, :string, :string, :string, :int, :int, :int], :int
+  attach_function :varnam_create_token, [:pointer, :string, :string, :string, :string, :int, :int, :int], :int
   attach_function :varnam_generate_cv_combinations, [:pointer], :int
   attach_function :varnam_set_scheme_details, [:pointer, :string, :string, :string, :string, :string], :int
   attach_function :varnam_get_last_error, [:pointer], :string
@@ -100,7 +99,7 @@ if (initialized != 0)
   exit(1)
 end
 
-VarnamToken = Struct.new(:type, :pattern, :value1, :value2, :match_type)
+VarnamToken = Struct.new(:type, :pattern, :value1, :value2, :tag, :match_type)
 
 module Varnam
   VARNAM_TOKEN_VOWEL           = 1
@@ -131,6 +130,7 @@ module Varnam
       @current_expression = ""
       @error_messages = []
       @warning_messages = []
+      @current_tag = nil
     end
 
     def errored
@@ -149,7 +149,7 @@ module Varnam
       @warnings
     end
 
-    attr_accessor :tokens, :current_expression, :error_messages, :warning_messages
+    attr_accessor :tokens, :current_expression, :error_messages, :warning_messages, :current_tag
   end
 end
 
@@ -257,7 +257,9 @@ def _persist_key_values(pattern, values, token_type, match_type)
     value2 = ""
   end
 
-  created = VarnamLibrary.varnam_create_token($varnam_handle.get_pointer(0), pattern, value1, value2, token_type, match_type, 1)
+  tag = _context.current_tag
+  tag = "" if tag.nil?
+  created = VarnamLibrary.varnam_create_token($varnam_handle.get_pointer(0), pattern, value1, value2, tag, token_type, match_type, 1)
   if created != 0
     error_message = VarnamLibrary.varnam_get_last_error($varnam_handle.get_pointer(0))
     error error_message
@@ -265,7 +267,7 @@ def _persist_key_values(pattern, values, token_type, match_type)
   end
 
   _context.tokens[token_type] = [] if _context.tokens[token_type].nil?
-  _context.tokens[token_type].push(VarnamToken.new(token_type, pattern, value1, value2, match_type))
+  _context.tokens[token_type].push(VarnamToken.new(token_type, pattern, value1, value2, tag, match_type))
 
   inform "Persisting #{match} - #{pattern} => #{value1},#{value2}"
 end
@@ -358,6 +360,12 @@ def consonants(hash)
   _create_token(hash, Varnam::VARNAM_TOKEN_CONSONANT)
 end
 
+def tag(name, &block)
+   _context.current_tag = name
+   block.call
+   _context.current_tag = nil
+end
+
 def consonant_vowel_combinations(hash)
   _ensure_sanity(hash)
   _create_token(hash, Varnam::VARNAM_TOKEN_CONSONANT_VOWEL)
@@ -448,7 +456,7 @@ def get_dead_consonants(criteria = {})
     item = VarnamLibrary::Token.new(tok)
     varnam_token = VarnamToken.new(item[:type],
                                    ffito_string(item[:pattern]), ffito_string(item[:value1]),
-                                   ffito_string(item[:value2]), item[:match_type])
+                                   ffito_string(item[:value2]), ffito_string(item[:tag]), item[:match_type])
     _context.tokens[token_type].push(varnam_token)
     i += 1
   end
diff --git a/tests/03-vst-compilation.c b/tests/03-vst-compilation.c
index 623522e..c22c48f 100644
--- a/tests/03-vst-compilation.c
+++ b/tests/03-vst-compilation.c
@@ -50,7 +50,7 @@ int create_without_buffering()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "tag", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -74,34 +74,34 @@ int generate_cv_combinations()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "~", "്", NULL, VARNAM_TOKEN_VIRAMA, VARNAM_MATCH_EXACT, 1);
+    rc = varnam_create_token(handle, "~", "്", NULL,"", VARNAM_TOKEN_VIRAMA, VARNAM_MATCH_EXACT, 1);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "a", "അ", NULL, VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "a", "അ", NULL,"", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "aa", "ആ", "ാ", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "aa", "ആ", "ാ","", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "A", "ആ", "ാ", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "A", "ആ", "ാ", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "a", "ആ", "ാ", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "a", "ആ", "ാ", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "ka", "ക", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "ka", "ക", NULL, "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "kha", "ഖ", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "kha", "ഖ", NULL, "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "gha", "ഘ", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "gha", "ഘ", NULL, "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "gha", "ഖ", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "gha", "ഖ", NULL, "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
     return_on_error (rc);
 
-    rc = varnam_create_token(handle, "kha", "ഘ", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "kha", "ഘ", NULL, "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
     return_on_error (rc);
 
     rc = varnam_generate_cv_combinations(handle);
@@ -127,21 +127,21 @@ int get_all_tokens()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2","", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern1", "value11", "value21", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern1", "value11", "value21", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern2", "value12", "value22", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern2", "value12", "value22", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -150,14 +150,14 @@ int get_all_tokens()
 
     varnam_config(handle, VARNAM_CONFIG_USE_DEAD_CONSONANTS, 0);
 
-    rc = varnam_create_token(handle, "p", "v", "v", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "p", "v", "v", "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "p", "v12", "v", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "p", "v12", "v", "", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_POSSIBILITY, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -226,7 +226,7 @@ int ignore_duplicates()
 
     varnam_enable_logging (handle, VARNAM_LOG_DEFAULT, &log_fun);
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -234,7 +234,7 @@ int ignore_duplicates()
     }
 
     /* Creating again. This should fail */
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc == VARNAM_SUCCESS)
     {
         printf("Duplicate check is not working. %s", varnam_get_last_error(handle));
@@ -250,7 +250,7 @@ int ignore_duplicates()
     }
 
     /* Creating again. This should ignore the duplicates */
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("Duplicate ignore is not working. %s", varnam_get_last_error(handle));
@@ -277,21 +277,21 @@ int auto_create_dead_consonants()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "~", "്", NULL, VARNAM_TOKEN_VIRAMA, VARNAM_MATCH_EXACT, 1);
+    rc = varnam_create_token(handle, "~", "്", NULL, "tag", VARNAM_TOKEN_VIRAMA, VARNAM_MATCH_EXACT, 1);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "ka", "ക", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 1);
+    rc = varnam_create_token(handle, "ka", "ക", NULL, "tag", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 1);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "p", "പ്", NULL, VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 1);
+    rc = varnam_create_token(handle, "p", "പ്", NULL, "tag", VARNAM_TOKEN_CONSONANT, VARNAM_MATCH_EXACT, 1);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -300,7 +300,12 @@ int auto_create_dead_consonants()
 
     varnam_flush_buffer(handle);
 
-    varnam_get_all_tokens (handle, VARNAM_TOKEN_DEAD_CONSONANT, &tokens);
+    rc = varnam_get_all_tokens (handle, VARNAM_TOKEN_DEAD_CONSONANT, &tokens);
+    if (rc != VARNAM_SUCCESS)
+    {
+        printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
+        return 1;
+    }
 
     for (i = 0; i < varray_length (tokens); i++) {
         token = varray_get (tokens, i);
@@ -328,7 +333,7 @@ int create_with_buffering()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 1);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 1);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -360,14 +365,14 @@ int create_exact_match_duplicates()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern ", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, "pattern ", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_ERROR)
     {
         printf("VARNAM_ERROR expected. Never got. Looks like duplicate exact match token are allowed %s", varnam_get_last_error(handle));
@@ -392,7 +397,7 @@ int create_possibility_match_duplicates()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("VARNAM_SUCCESS expected. Never got. %s", varnam_get_last_error(handle));
@@ -400,14 +405,14 @@ int create_possibility_match_duplicates()
     }
 
     /* This should be allowed as it is a different value */
-    rc = varnam_create_token(handle, "pattern ", "value11", "value22", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "pattern ", "value11", "value22", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
     if (rc != VARNAM_SUCCESS)
     {
         printf("Creating possible matches for same pattern with different value failing. %s", varnam_get_last_error(handle));
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern ", "value1", "value2", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
+    rc = varnam_create_token(handle, "pattern ", "value1", "value2", "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_POSSIBILITY, 0);
     if (rc != VARNAM_ERROR)
     {
         printf("Creating possible matches for same pattern and same value again is allowed. %s", varnam_get_last_error(handle));
@@ -432,7 +437,7 @@ int only_valid_matchtypes()
         return 1;
     }
 
-    rc = varnam_create_token(handle, "pattern", "value1", "value2", VARNAM_TOKEN_VOWEL, 10, 0);
+    rc = varnam_create_token(handle, "pattern", "value1", "value2", "", VARNAM_TOKEN_VOWEL, 10, 0);
     if (rc != VARNAM_ERROR)
     {
         printf("VARNAM_ERROR expected. Never got. %s", varnam_get_last_error(handle));
@@ -468,7 +473,7 @@ int maxlength_check()
         return 1;
     }
 
-    rc = varnam_create_token(handle, pattern, value1, value2, VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
+    rc = varnam_create_token(handle, pattern, value1, value2, "", VARNAM_TOKEN_VOWEL, VARNAM_MATCH_EXACT, 0);
     if (rc != VARNAM_ARGS_ERROR)
     {
         printf("VARNAM_ARGS_ERROR expected. Never got. %s", varnam_get_last_error(handle));
diff --git a/varnam-api.h b/varnam-api.h
index e7712b4..ac6b0ae 100644
--- a/varnam-api.h
+++ b/varnam-api.h
@@ -46,6 +46,14 @@
 VARNAM_EXPORT extern int
 varnam_init(const char *scheme_file, varnam **handle, char **msg);
 
+VARNAM_EXPORT extern int
+varnam_register_renderer(
+    varnam *handle,
+    const char *scheme_id,
+    int (*tl)(varnam *handle, vtoken *previous, vtoken *current,  strbuf *output),
+    int (*rtl)(varnam *handle, vtoken *previous, vtoken *current,  strbuf *output)
+);
+
 /**
  * Configure varnam library.
  *
@@ -116,6 +124,7 @@ VARNAM_EXPORT extern int varnam_create_token(
     const char *pattern,
     const char *value1,
     const char *value2,
+    const char *tag,
     int token_type,
     int match_type,
     int buffered
diff --git a/varnam-symbol-table.c b/varnam-symbol-table.c
index 390fa64..e2dfa4b 100644
--- a/varnam-symbol-table.c
+++ b/varnam-symbol-table.c
@@ -26,143 +26,6 @@
 #include "varnam-api.h"
 #include "varnam-token.h"
 
-struct token*
-find_rtl_token(varnam *handle, const char *lookup)
-{
-    struct varnam_internal *internal;
-    struct token *tok = NULL;
-    char sql[500];
-    const char *pattern, *value1, *value2, *tag;
-    sqlite3_stmt *stmt; sqlite3 *db;
-    int rc, type, has_children;
-
-    assert( handle ); assert( lookup );
-
-    internal = handle->internal;
-    db = internal->db;
-
-    snprintf( sql, 500, "select type, pattern, value1, value2, children, tag from symbols where value1 = ?1 or value2 = ?1 limit 1;");
-    rc = sqlite3_prepare_v2( db, sql, 500, &stmt, NULL );
-    if( rc == SQLITE_OK )
-    {
-        sqlite3_bind_text (stmt, 1, lookup, (int) strlen(lookup), NULL);
-        rc = sqlite3_step (stmt);
-        if( rc == SQLITE_ROW )
-        {
-            type = sqlite3_column_int( stmt, 0 );
-            pattern = (const char*) sqlite3_column_text( stmt, 1 );
-            value1 = (const char*) sqlite3_column_text( stmt, 2 );
-            value2 = (const char*) sqlite3_column_text( stmt, 3 );
-            has_children = sqlite3_column_int( stmt, 4 );
-            tag = (const char*) sqlite3_column_text( stmt, 5 );
-
-            if(internal->current_rtl_token == NULL) {
-                internal->current_rtl_token = (struct token *) xmalloc(sizeof (struct token));
-                assert( internal->current_rtl_token );
-            }
-
-            tok = internal->current_rtl_token;
-            tok->type = type;
-            strncpy( tok->pattern, pattern, VARNAM_SYMBOL_MAX);
-            strncpy( tok->value1, value1, VARNAM_SYMBOL_MAX);
-            strncpy( tok->value2, value2, VARNAM_SYMBOL_MAX);
-            strncpy( tok->tag, tag, VARNAM_TOKEN_TAG_MAX);
-            tok->children = has_children;
-        }
-    }
-
-    sqlite3_finalize( stmt );
-    return tok;
-}
-
-int
-can_find_token(varnam *handle, struct token *last, const char *lookup)
-{
-    char sql[500];
-    sqlite3_stmt *stmt;
-    sqlite3 *db;
-    int rc; int result = 0;
-
-    assert( lookup );
-    assert( handle );
-
-    db = handle->internal->db;
-
-    snprintf( sql, 500, "select count(id) as cnt from symbols where pattern like '%s%%';", lookup );
-    rc = sqlite3_prepare_v2( db, sql, 500, &stmt, NULL );
-    if( rc == SQLITE_OK ) {
-        rc = sqlite3_step( stmt );
-        if( rc == SQLITE_ROW ) {
-            if( sqlite3_column_int( stmt, 0 ) > 0 ) {
-                result = 1;
-            }
-        }
-    }
-
-    sqlite3_finalize( stmt );
-    return result;
-}
-
-int
-can_find_rtl_token(varnam *handle, struct token *last, const char *lookup)
-{
-    char sql[500];
-    sqlite3_stmt *stmt;
-    sqlite3 *db;
-    int rc; int result = 0;
-
-    assert( lookup );
-    assert( handle );
-
-    db = handle->internal->db;
-
-    snprintf( sql, 500, "select count(pattern) as cnt from symbols where value1 like '%s%%' or value2 like '%s%%';", lookup, lookup );
-    rc = sqlite3_prepare_v2( db, sql, 500, &stmt, NULL );
-    if( rc == SQLITE_OK ) {
-        rc = sqlite3_step( stmt );
-        if( rc == SQLITE_ROW ) {
-            if( sqlite3_column_int( stmt, 0 ) > 0 ) {
-                result = 1;
-            }
-        }
-    }
-
-    sqlite3_finalize( stmt );
-    return result;
-}
-
-/* void */
-/* fill_general_values(varnam *handle, char *output, const char *name) */
-/* { */
-/*     char sql[500]; */
-/*     const char *result; */
-/*     sqlite3_stmt *stmt; */
-/*     sqlite3 *db; */
-/*     int rc; */
-
-/*     assert( name ); */
-/*     assert( handle ); */
-
-/*     db = handle->internal->db; */
-
-/*     snprintf(sql, 500, "select value from general where key = ?1;"); */
-
-/*     rc = sqlite3_prepare_v2( db, sql, 500, &stmt, NULL ); */
-/*     if( rc == SQLITE_OK ) { */
-/*         sqlite3_bind_text(stmt, 1, name, (int) strlen(name), NULL); */
-/*         rc = sqlite3_step( stmt ); */
-/*         if( rc == SQLITE_ROW ) */
-/*         { */
-/*             result = (const char*) sqlite3_column_text(stmt, 0); */
-/*             if(result) { */
-/*                 strncpy(output, result, VARNAM_SYMBOL_MAX); */
-/*             } */
-/*         } */
-/*     } */
-
-/*     sqlite3_finalize( stmt ); */
-/* } */
-
 int
 ensure_schema_exists(varnam *handle, char **msg)
 {
@@ -279,6 +142,7 @@ vst_persist_token(
     const char *pattern,
     const char *value1,
     const char *value2,
+    const char *tag,
     int token_type,
     int match_type)
 {
@@ -317,8 +181,8 @@ vst_persist_token(
     sqlite3_bind_int (stmt, 1, token_type);
     sqlite3_bind_text(stmt, 2, pattern,    -1, NULL);
     sqlite3_bind_text(stmt, 3, value1,     -1, NULL);
-    sqlite3_bind_text(stmt, 4, value2 == NULL ? "" : value2,     -1, NULL);
-    sqlite3_bind_text(stmt, 5, "",         -1, NULL);
+    sqlite3_bind_text(stmt, 4, value2 == NULL ? "" : value2, -1, NULL);
+    sqlite3_bind_text(stmt, 5, tag == NULL ? "" : tag, -1, NULL);
     sqlite3_bind_int (stmt, 6, match_type);
 
     rc = sqlite3_step( stmt );
@@ -385,7 +249,7 @@ vst_get_virama(varnam* handle, struct token **output)
 
     db = handle->internal->db;
 
-    rc = sqlite3_prepare_v2( db, "select id, type, match_type, pattern, value1, value2 from symbols where type = ?1 and match_type = ?2 limit 1;", -1, &stmt, NULL );
+    rc = sqlite3_prepare_v2( db, "select id, type, match_type, pattern, value1, value2, tag from symbols where type = ?1 and match_type = ?2 limit 1;", -1, &stmt, NULL );
     if(rc != SQLITE_OK)
     {
         set_last_error (handle, "Failed to get virama : %s", sqlite3_errmsg(db));
@@ -399,12 +263,14 @@ vst_get_virama(varnam* handle, struct token **output)
     rc = sqlite3_step( stmt );
     if( rc == SQLITE_ROW )
     {
-        *output = Token( (int) sqlite3_column_int(stmt, 0),
-                         (int) sqlite3_column_int(stmt, 1),
-                         (int) sqlite3_column_int(stmt, 2),
-                         (const char*) sqlite3_column_text(stmt, 3),
-                         (const char*) sqlite3_column_text(stmt, 4),
-                         (const char*) sqlite3_column_text(stmt, 5), NULL);
+        *output = get_pooled_token(handle,
+                                   (int) sqlite3_column_int(stmt, 0),
+                                   (int) sqlite3_column_int(stmt, 1),
+                                   (int) sqlite3_column_int(stmt, 2),
+                                   (const char*) sqlite3_column_text(stmt, 3),
+                                   (const char*) sqlite3_column_text(stmt, 4),
+                                   (const char*) sqlite3_column_text(stmt, 5),
+                                   (const char*) sqlite3_column_text(stmt, 6));
     }
     else if ( rc == SQLITE_DONE )
     {
@@ -574,7 +440,7 @@ vst_generate_cv_combinations(varnam* handle)
             rc = varnam_create_token(handle,
                                      newpattern,
                                      newvalue1,
-                                     newvalue2, VARNAM_TOKEN_CONSONANT_VOWEL, new_match_type, 1);
+                                     newvalue2, "", VARNAM_TOKEN_CONSONANT_VOWEL, new_match_type, 1);
 
             if (rc != VARNAM_SUCCESS)
             {
@@ -673,7 +539,7 @@ prepare_tokenization_stmt (varnam *handle, int tokenize_using, int match_type, s
     case VARNAM_TOKENIZER_PATTERN:
         if (v_->tokenize_using_pattern == NULL)
         {
-            rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2 from symbols where pattern = ?1 and match_type = 1;",
+            rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2, tag from symbols where pattern = ?1 and match_type = 1;",
                                      -1, &v_->tokenize_using_pattern, NULL );
             if (rc != SQLITE_OK) {
                 set_last_error (handle, "Failed to tokenize : %s", sqlite3_errmsg(v_->db));
@@ -687,7 +553,7 @@ prepare_tokenization_stmt (varnam *handle, int tokenize_using, int match_type, s
         {
             if (v_->tokenize_using_value == NULL)
             {
-                rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2 from symbols where value1 = ?1 or value2 = ?1;",
+                rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2, tag from symbols where value1 = ?1 or value2 = ?1;",
                                          -1, &v_->tokenize_using_value, NULL );
                 if (rc != SQLITE_OK) {
                     set_last_error (handle, "Failed to tokenize : %s", sqlite3_errmsg(v_->db));
@@ -700,7 +566,7 @@ prepare_tokenization_stmt (varnam *handle, int tokenize_using, int match_type, s
         {
             if (v_->tokenize_using_value_and_match_type == NULL)
             {
-                rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2 from symbols where (value1 = ?1 or value2 = ?1) and match_type = ?2;",
+                rc = sqlite3_prepare_v2( v_->db, "select id, type, match_type, pattern, value1, value2, tag from symbols where (value1 = ?1 or value2 = ?1) and match_type = ?2;",
                                          -1, &v_->tokenize_using_value_and_match_type, NULL );
                 if (rc != SQLITE_OK) {
                     set_last_error (handle, "Failed to tokenize : %s", sqlite3_errmsg(v_->db));
@@ -743,7 +609,8 @@ read_all_tokens_and_add_to_array (varnam *handle, const char *lookup, int tokeni
                                     sqlite3_column_int( stmt, 2 ),
                                     (const char*) sqlite3_column_text( stmt, 3 ),
                                     (const char*) sqlite3_column_text( stmt, 4 ),
-                                    (const char*) sqlite3_column_text( stmt, 5 ), "");
+                                    (const char*) sqlite3_column_text( stmt, 5 ),
+                                    (const char*) sqlite3_column_text( stmt, 6 ));
             assert (tok);
 
             if (!cleared) {
diff --git a/varnam-symbol-table.h b/varnam-symbol-table.h
index 1e19cce..e431ecd 100644
--- a/varnam-symbol-table.h
+++ b/varnam-symbol-table.h
@@ -23,38 +23,6 @@
 #include "varnam-types.h"
 #include "varnam-array.h"
 
-/**
-* This function will try to get a token for the lookup text provided.
-* search will be done directly on the symbol table.
-* A valid instance of token will returned upon success.
-* NULL value indicates a failure to get the token
-**/
-struct token*
-find_token(varnam *handle, const char *lookup);
-
-/**
-* This function will try to get a token for the lookup text provided.
-* search will be done directly on the symbol table on value1 and value2 fields.
-* A valid instance of token will returned upon success.
-* NULL value indicates a failure to get the token
-**/
-struct token*
-find_rtl_token(varnam *handle, const char *lookup);
-
-/**
-* Does a search in the symbol table and retrns a boolean value indicating
-* the possibility of finding a token for the lookup text.
-**/
-int
-can_find_token(varnam *handle, struct token *last, const char *lookup);
-
-/**
-* Does a search in the symbol table and retrns a boolean value indicating
-* the possibility of finding a rtl token for the lookup text.
-**/
-int
-can_find_rtl_token(varnam *handle, struct token *last, const char *lookup);
-
 /**
  * checks the schema availability. this function will create it when necessary
  **/
@@ -76,6 +44,7 @@ vst_persist_token(
     const char *pattern,
     const char *value1,
     const char *value2,
+    const char *tag,
     int token_type,
     int match_type);
 
diff --git a/varnam-tl.c b/varnam-tl.c
index cffc6df..f7baaa7 100644
--- a/varnam-tl.c
+++ b/varnam-tl.c
@@ -89,33 +89,22 @@ varnam_reverse_transliterate(varnam *handle,
                              const char *input,
                              char **output)
 {
-    int rc, i, j;
-    varray *result, *tokens;
-    strbuf *rtl;
-    vtoken *token;
+    int rc;
+    varray *result;
 
     if(handle == NULL || input == NULL)
         return VARNAM_ARGS_ERROR;
 
+    reset_pool (handle);
+
     result = get_pooled_array (handle);
     rc = vst_tokenize (handle, input, VARNAM_TOKENIZER_VALUE, VARNAM_MATCH_EXACT, result);
-    if (rc) return rc;
+    if (rc)
+        return rc;
 
-    rtl = get_pooled_string (handle);
-    assert (rtl);
-    for (i = 0; i < varray_length (result); i++)
-    {
-        tokens = varray_get (result, i);
-        assert (tokens);
-        for (j = 0; j < varray_length (tokens); j++)
-        {
-            token = varray_get (tokens, j);
-            assert (token);
-            strbuf_add (rtl, token->pattern);
-            break; /* We only care about first element in each array */
-        }
-    }
+    rc = resolve_rtl_tokens (handle, result, output);
+    if (rc)
+        return rc;
 
-    *output = rtl->buffer;
     return VARNAM_SUCCESS;
 }
diff --git a/varnam-token.c b/varnam-token.c
index 2c678a5..d65a771 100644
--- a/varnam-token.c
+++ b/varnam-token.c
@@ -45,7 +45,7 @@ initialize_token (vtoken *tok,
         tok->value2[0] = '\0';
 
     if (tag != NULL)
-        strncpy( tok->tag, tag, VARNAM_TOKEN_TAG_MAX);
+        strncpy( tok->tag, tag, VARNAM_SYMBOL_MAX);
     else
         tok->tag[0] = '\0';
 }
diff --git a/varnam-types.h b/varnam-types.h
index 2d778be..b29d729 100644
--- a/varnam-types.h
+++ b/varnam-types.h
@@ -22,7 +22,6 @@
 #include "foreign/sqlite3.h"
 
 #define VARNAM_SYMBOL_MAX           30
-#define VARNAM_TOKEN_TAG_MAX        15
 #define VARNAM_LIB_TEMP_BUFFER_SIZE 100
 #define VARNAM_WORD_MAX             25
 
@@ -77,8 +76,8 @@ struct varnam_internal
 {
     sqlite3 *db, *known_words;
     char *message;
-    struct varnam_token_rendering *renderers;
-
+    struct varray_t *renderers;
+    
     struct token *virama;
 
     struct strbuf *output;
@@ -136,7 +135,7 @@ typedef struct varnam {
 
 typedef struct token {
     int id, type, match_type;
-    char tag[VARNAM_TOKEN_TAG_MAX];
+    char tag[VARNAM_SYMBOL_MAX];
     char pattern[VARNAM_SYMBOL_MAX];
     char value1[VARNAM_SYMBOL_MAX];
     char value2[VARNAM_SYMBOL_MAX];
@@ -154,11 +153,11 @@ struct varnam_rule {
     struct varnam_rule *next;
 };
 
-struct varnam_token_rendering {
-    const char *scheme_identifier;
-    int (*render)(varnam *handle, struct token *match,  struct strbuf *output);
-    int (*render_rtl)(varnam *handle, struct token *match,  struct strbuf *output);
-};
+typedef struct varnam_token_rendering {
+    const char *scheme_id;
+    int (*tl)(varnam *handle, vtoken *previous, vtoken *current,  struct strbuf *output);
+    int (*rtl)(varnam *handle, vtoken *previous, vtoken *current,  struct strbuf *output);
+} vtoken_renderer;
 
 typedef struct varnam_info_t {
     const char *scheme_file;
diff --git a/varnam.c b/varnam.c
index 26e3884..d8b86f5 100644
--- a/varnam.c
+++ b/varnam.c
@@ -28,11 +28,7 @@
 #include "varnam-symbol-table.h"
 #include "varnam-words-table.h"
 #include "varnam-token.h"
-#include "rendering/renderers.h"
-
-static struct varnam_token_rendering renderers[] = {
-    { "ml-unicode", ml_unicode_renderer, ml_unicode_rtl_renderer }
-};
+#include "renderer/renderers.h"
 
 static struct varnam_internal*
 initialize_internal()
@@ -41,6 +37,7 @@ initialize_internal()
     vi = (struct varnam_internal *) xmalloc(sizeof (struct varnam_internal));
     if(vi) {
         vi->virama = NULL;
+        vi->renderers = NULL;
         vi->last_token_available = 0;
         vi->last_rtl_token_available = 0;
         vi->last_token = NULL;
@@ -128,17 +125,47 @@ varnam_init(const char *scheme_file, varnam **handle, char **msg)
         return VARNAM_MEMORY_ERROR;
 
     strncpy(c->scheme_file, scheme_file, filename_length + 1);
-    vi->renderers = renderers;
     c->internal = vi;
 
     rc = ensure_schema_exists(c, msg);
     if (rc != VARNAM_SUCCESS)
         return rc;
 
+    rc = varnam_register_renderer (c, "ml-unicode", &ml_unicode_renderer, &ml_unicode_rtl_renderer);
+    if (rc != VARNAM_SUCCESS)
+        return rc;
+
     *handle = c;
     return VARNAM_SUCCESS;
 }
 
+int
+varnam_register_renderer(
+    varnam *handle,
+    const char *scheme_id,
+    int (*tl)(varnam *handle, vtoken *previous, vtoken *current,  strbuf *output),
+    int (*rtl)(varnam *handle, vtoken *previous, vtoken *current,  strbuf *output))
+{
+    vtoken_renderer *r;
+    
+    if (handle == NULL)
+        return VARNAM_ARGS_ERROR;
+
+    if (v_->renderers == NULL)
+        v_->renderers = varray_init();
+
+    r = xmalloc (sizeof (vtoken_renderer));
+    if (r == NULL)
+        return VARNAM_ERROR;
+
+    r->scheme_id = scheme_id;
+    r->tl = tl;
+    r->rtl = rtl;
+
+    varray_push (v_->renderers, r);
+    return VARNAM_SUCCESS;
+}
+
 int
 varnam_set_scheme_details(
     varnam *handle,
@@ -303,6 +330,7 @@ varnam_create_token(
     const char *pattern,
     const char *value1,
     const char *value2,
+    const char *tag,
     int token_type,
     int match_type,
     int buffered)
@@ -319,9 +347,10 @@ varnam_create_token(
 
     if (strlen(pattern) > VARNAM_SYMBOL_MAX ||
         strlen(value1) > VARNAM_SYMBOL_MAX  ||
-        (value2 != NULL && strlen(value2) > VARNAM_SYMBOL_MAX))
+        (value2 != NULL && strlen(value2) > VARNAM_SYMBOL_MAX) ||
+        (tag != NULL && strlen(tag) > VARNAM_SYMBOL_MAX))
     {
-        set_last_error (handle, "Length of pattern, value1 or value2 should be less than VARNAM_SYMBOL_MAX");
+        set_last_error (handle, "Length of pattern, tag, value1 or value2 should be less than VARNAM_SYMBOL_MAX");
         return VARNAM_ARGS_ERROR;
     }
 
@@ -366,7 +395,7 @@ varnam_create_token(
             else
                 v2[0] = '\0';
 
-            rc = vst_persist_token (handle, p, v1, v2, VARNAM_TOKEN_DEAD_CONSONANT, match_type);
+            rc = vst_persist_token (handle, p, v1, v2, tag, VARNAM_TOKEN_DEAD_CONSONANT, match_type);
             if (rc != VARNAM_SUCCESS)
             {
                 if (buffered) vst_discard_changes(handle);
@@ -379,7 +408,7 @@ varnam_create_token(
     if (token_type == VARNAM_TOKEN_NON_JOINER)
         value1 = value2 = ZWNJ();
 
-    rc = vst_persist_token (handle, pattern, value1, value2, token_type, match_type);
+    rc = vst_persist_token (handle, pattern, value1, value2, tag, token_type, match_type);
     if (rc != VARNAM_SUCCESS)
     {
         if (buffered) vst_discard_changes(handle);