Bundle language support within .app as resources (#3)

* Use Logger
* Improve varnam initing
* Include VST, VLF in bundle resources
* Get list of languages from varnam
* Move to schemeID instead of scriptName
* Remove dependency on LipikaEngine from IME
* Use Logger from Lipika-engine
* Close varnam if out of focus, better consistent DB
* Add Kannada support
* Don't gzip .vlf
* Import VLF on postinstall

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "GoVarnam/govarnam"]
 	path = GoVarnam/govarnam
 	url = [email protected]:varnamproject/govarnam.git
+[submodule "GoVarnam/schemes"]
+	path = GoVarnam/schemes
+	url = https://github.com/varnamproject/schemes.git

GoVarnam/.gitignore

@@ -1 +1,4 @@
 *.dylib
+assets/
+*.vst
+*.vlf

GoVarnam/Varnam.swift

@@ -8,7 +8,7 @@
 
 import Foundation
 
-struct VarnamException: Error {
+public struct VarnamException: Error {
     let message: String
 
     init(_ message: String) {
@@ -20,10 +20,57 @@ struct VarnamException: Error {
     }
 }
 
+public struct SchemeDetails {
+    var Identifier: String
+    var LangCode: String
+    var DisplayName: String
+    var Author: String
+    var CompiledDate: String
+    var IsStable: Bool
+}
+
+extension String {
+    func toCStr() -> UnsafeMutablePointer<CChar>? {
+        return UnsafeMutablePointer(mutating: (self as NSString).utf8String)
+    }
+}
+
 public class Varnam {
     private var varnamHandle: Int32 = 0;
 
+    static let assetsFolderPath = Bundle.main.resourceURL!.appendingPathComponent("assets").path
+    static func importAllVLFInAssets() {
+        // TODO import only necessary ones
+        let fm = FileManager.default
+        for scheme in getAllSchemeDetails() {
+            do {
+                let varnam = try! Varnam(scheme.Identifier)
+                let items = try fm.contentsOfDirectory(atPath: assetsFolderPath)
+
+                for item in items {
+                    if item.hasSuffix(".vlf") && item.hasPrefix(scheme.Identifier) {
+                        let path = assetsFolderPath + "/" + item
+                        varnam.importFromFile(path)
+                    }
+                }
+            } catch {
+                Logger.log.error("Couldn't import")
+            }
+        }
+    }
+
+    // This will only run once
+    struct VarnamInit {
+        static let once = VarnamInit()
+        init() {
+            print(assetsFolderPath)
+            varnam_set_vst_lookup_dir(assetsFolderPath.toCStr())
+        }
+    }
+
     internal init(_ schemeID: String = "ml") throws {
+        _ = VarnamInit.once
+
         schemeID.withCString {
             let rc = varnam_init_from_id(UnsafeMutablePointer(mutating: $0), &varnamHandle)
             try! checkError(rc)
@@ -39,14 +86,17 @@ public class Varnam {
             throw VarnamException(getLastError())
         }
     }
+
+    public func close() {
+        varnam_close(varnamHandle)
+    }
 
     public func transliterate(_ input: String) -> [String] {
         var arr: UnsafeMutablePointer<varray>? = varray_init()
-        let cInput = (input as NSString).utf8String
         varnam_transliterate(
             varnamHandle,
             1,
-            UnsafeMutablePointer(mutating: cInput),
+            input.toCStr(),
             &arr
         )
 
@@ -59,4 +109,30 @@ public class Varnam {
         }
         return results
     }
+
+    public func importFromFile(_ path: String) {
+        varnam_import(varnamHandle, path.toCStr())
+    }
+
+    public static func getAllSchemeDetails() -> [SchemeDetails] {
+        _ = VarnamInit.once
+
+        var schemes = [SchemeDetails]()
+
+        let arr = varnam_get_all_scheme_details()
+        for i in (0..<varray_length(arr)) {
+            let sdPointer = varray_get(arr, i).assumingMemoryBound(to: SchemeDetails_t.self
+            )
+            let sd = sdPointer.pointee
+            schemes.append(SchemeDetails(
+                Identifier: String(cString: sd.Identifier),
+                LangCode: String(cString: sd.LangCode),
+                DisplayName: String(cString: sd.DisplayName),
+                Author: String(cString: sd.Author),
+                CompiledDate: String(cString: sd.CompiledDate),
+                IsStable: (sd.IsStable != 0)
+            ))
+        }
+        return schemes
+    }
 }

GoVarnam/update_assets.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import gzip
+import json
+from os.path import basename
+from pathlib import Path
+import shutil
+from urllib import request
+
+# Copies .vst, .vlf from schemes folder to assets
+# You need to build inside schemes first before running this script
+# Use build_all_packs.sh script to do that
+
+def copyScheme(schemeID):
+    programDir = str(Path(__file__).parent.absolute())
+    source = programDir + '/schemes/schemes/' + schemeID
+    target = programDir + '/assets'
+
+    packsInfo = []
+
+    for path in Path(source + '/').rglob('*'):
+        if basename(path) == schemeID + '.vst':
+            shutil.copy2(path, target)
+            continue
+
+        for packPath in Path(path).rglob('*'):
+            if basename(packPath) == 'pack.json':
+                packsInfo.append(json.load(open(packPath, 'r')))
+                continue
+
+            if ".vlf" not in basename(packPath):
+                continue
+
+            with open(
+                packPath, 'rb'
+            ) as f_in, open(
+                target + '/' + basename(packPath),
+                'wb'
+            ) as f_out:
+                f_out.writelines(f_in)
+
+    with open(target + '/packs.json', 'w') as f:
+        json.dump(packsInfo, f, ensure_ascii=False)
+
+# For now just Malayalam, Kannada for govarnam-macOS
+for schemeID in ["ml", "kn"]:
+    copyScheme(schemeID)

Input Source/AppDelegate.swift

@@ -8,7 +8,6 @@
  */
 
 import InputMethodKit
-import LipikaEngine_OSX
 
 @NSApplicationMain
 class AppDelegate: NSObject, NSApplicationDelegate {
@@ -31,7 +30,7 @@ class AppDelegate: NSObject, NSApplicationDelegate {
                 item.keyEquivalentModifierMask = NSEvent.ModifierFlags(rawValue: flags)
                 item.keyEquivalent = item.keyEquivalentModifierMask.contains(.shift) ? key : key.lowercased()
             }
-            if entry.identifier == config.scriptName {
+            if entry.identifier == config.schemeID {
                 item.state = .on
             }
             item.representedObject = entry.identifier
@@ -42,6 +41,13 @@ class AppDelegate: NSObject, NSApplicationDelegate {
     }}
 
     func applicationDidFinishLaunching(_ aNotification: Notification) {
+        for arg in CommandLine.arguments {
+            if arg == "-import" {
+                importVLF()
+                exit(0)
+            }
+        }
+
         guard let connectionName = Bundle.main.infoDictionary?["InputMethodConnectionName"] as? String else {
             fatalError("Unable to get Connection Name from Info dictionary!")
         }
@@ -64,4 +70,8 @@ class AppDelegate: NSObject, NSApplicationDelegate {
         Logger.log.debug("Comitting all editing before terminating")
         server.commitComposition(self)
     }
+
+    func importVLF() {
+        Varnam.importAllVLFInAssets()
+    }
 }

Input Source/ClientManager.swift

@@ -31,22 +31,22 @@ class ClientManager: CustomStringConvertible {
 
     init?(client: IMKTextInput) {
         guard let bundleId = client.bundleIdentifier(), let clientId = client.uniqueClientIdentifierString() else {
-            Log.warning("bundleIdentifier: \(client.bundleIdentifier() ?? "nil") or uniqueClientIdentifierString: \(client.uniqueClientIdentifierString() ?? "nil") - failing ClientManager.init()")
+            Logger.log.warning("bundleIdentifier: \(client.bundleIdentifier() ?? "nil") or uniqueClientIdentifierString: \(client.uniqueClientIdentifierString() ?? "nil") - failing ClientManager.init()")
             return nil
         }
-        Log.debug("Initializing client: \(bundleId) with Id: \(clientId)")
+        Logger.log.debug("Initializing client: \(bundleId) with Id: \(clientId)")
         self.client = client
         if !client.supportsUnicode() {
-            Log.warning("Client: \(bundleId) does not support Unicode!")
+            Logger.log.warning("Client: \(bundleId) does not support Unicode!")
         }
         if !client.supportsProperty(TSMDocumentPropertyTag(kTSMDocumentSupportDocumentAccessPropertyTag)) {
-            Log.warning("Client: \(bundleId) does not support Document Access!")
+            Logger.log.warning("Client: \(bundleId) does not support Document Access!")
         }
         _description = "\(bundleId) with Id: \(clientId)"
     }
 
     func setGlobalCursorLocation(_ location: Int) {
-        Log.debug("Setting global cursor location to: \(location)")
+        Logger.log.debug("Setting global cursor location to: \(location)")
         client.setMarkedText("|", selectionRange: NSMakeRange(0, 0), replacementRange: NSMakeRange(location, 0))
         client.setMarkedText("", selectionRange: NSMakeRange(0, 0), replacementRange: NSMakeRange(location, 0))
     }
@@ -56,11 +56,10 @@ class ClientManager: CustomStringConvertible {
     }
 
     func updateCandidates(_ sugs: [String]) {
-        Log.debug(sugs)
         // Remove duplicates
         // For some weird reason, when there are duplicates,
         // candidate window makes them hidden
-        candidates = NSOrderedSet(array: sugs).array as! [String]
+        candidates = sugs.uniqued()
         updateLookupTable()
     }
 
@@ -92,13 +91,13 @@ class ClientManager: CustomStringConvertible {
     }
 
     func finalize(_ output: String) {
-        Log.debug("Finalizing with: \(output)")
+        Logger.log.debug("Finalizing with: \(output)")
         client.insertText(output, replacementRange: notFoundRange)
         candidatesWindow.hide()
     }
 
     func clear() {
-        Log.debug("Clearing MarkedText and Candidate window")
+        Logger.log.debug("Clearing MarkedText and Candidate window")
         client.setMarkedText("", selectionRange: NSMakeRange(0, 0), replacementRange: notFoundRange)
         candidates = []
         candidatesWindow.hide()

Input Source/Common.swift

@@ -0,0 +1,71 @@
+/*
+ * LipikaEngine is a multi-codepoint, user-configurable, phonetic, Transliteration Engine.
+ * Copyright (C) 2017 Ranganath Atreya
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+import Foundation
+
+func synchronize<T>(_ lockObject: AnyObject, _ closure: () -> T) -> T {
+    objc_sync_enter(lockObject)
+    defer { objc_sync_exit(lockObject) }
+    return closure()
+}
+
+func synchronize<T>(_ lockObject: AnyObject, _ closure: () throws -> T) throws -> T {
+    objc_sync_enter(lockObject)
+    defer { objc_sync_exit(lockObject) }
+    return try closure()
+}
+
+let keyBase = Bundle.main.bundleIdentifier ?? "LipikaEngine"
+
+func getThreadLocalData(key: String) -> Any? {
+    let fullKey: NSString = "\(keyBase).\(key)" as NSString
+    return Thread.current.threadDictionary.object(forKey: fullKey)
+}
+
+func setThreadLocalData(key: String, value: Any) {
+    let fullKey: NSString = "\(keyBase).\(key)" as NSString
+    Thread.current.threadDictionary.setObject(value, forKey: fullKey)
+}
+
+func removeThreadLocalData(key: String) {
+    let fullKey: NSString = "\(keyBase).\(key)" as NSString
+    Thread.current.threadDictionary.removeObject(forKey: fullKey)
+}
+
+func filesInDirectory(directory: URL, withExtension ext: String) throws -> [String] {
+    let files = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: [], options: [])
+    return files.filter({$0.pathExtension == ext}).compactMap { $0.deletingPathExtension().lastPathComponent }
+}
+
+extension String {
+    func unicodeScalars() -> [UnicodeScalar] {
+        return Array(self.unicodeScalars)
+    }
+
+    func unicodeScalarReversed() -> String {
+        var result = ""
+        result.unicodeScalars.append(contentsOf: self.unicodeScalars.reversed())
+        return result
+    }
+
+    static func + (lhs: String, rhs: [UnicodeScalar]) -> String {
+        var stringRHS = ""
+        stringRHS.unicodeScalars.append(contentsOf: rhs)
+        return lhs + stringRHS
+    }
+}
+
+// Copyright mxcl, CC-BY-SA 4.0
+// https://stackoverflow.com/a/46354989/1372424
+public extension Array where Element: Hashable {
+    func uniqued() -> [Element] {
+        var seen = Set<Element>()
+        return filter{ seen.insert($0).inserted }
+    }
+}

Input Source/Config.swift

@@ -0,0 +1,44 @@
+/*
+ * LipikaEngine is a multi-codepoint, user-configurable, phonetic, Transliteration Engine.
+ * Copyright (C) 2018 Ranganath Atreya
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+import Foundation
+
+/// This class provides default config values that the client can override, typically using `UserDefaults` and pass an instance into `LiteratorFactory`.
+open class Config {
+    /**
+     Empty public init to enable clients to call super.init()
+     */
+    public init() {}
+
+    /**
+     This character is used to break input aggregation. Typically this is the forward-slash character (`\`).
+
+     __Example__: if `a` maps to `1` and `b` maps to `2` and `ab` maps to `3` then inputting `ab` will output `3` but inputting `a\b` will output `12`
+     */
+    open var stopCharacter: UnicodeScalar { return "\\" }
+
+    /**
+     All input characters enclosed by this character will be echoed to the output as-is and not converted.
+
+     __Example__: if `a` maps to `1` and `b` maps to `2` and `ab` maps to `3` then inputting `ab` will output `3` but inputting `` `ab` `` will output `ab`
+     */
+    open var escapeCharacter: UnicodeScalar { return "`" }
+
+    /**
+     The URL path to the top-level directory where the schemes files are present. Usually this would return something like `Bundle.main.bundleURL.appendingPathComponent("Mapping")`
+     */
+    open var mappingDirectory: URL { return Bundle(for: Config.self).bundleURL.appendingPathComponent("Resources", isDirectory: true).appendingPathComponent("Mapping", isDirectory: true) }
+
+    /**
+     The level at which to NSLog log messages generated by LipikaEngine.
+
+     - Important: This configuration only holds within the same thread in which `LiteratorFactory` was initialized.
+     */
+    open var logLevel: Logger.Level { return .warning }
+}