Better suggestion with diacritics (#1223)

* Update cdict

* scripts/subst_of_compose.py: Compute substitutions

from compose mappings. They are used when building dictionaries.

* Add substitutions compose data

* Better suggestion with diacritics

This improves the suggestions for words that contain diacritics and
uppercase letters.

This works by stripping diacritics both when building the dictionaries
(using word aliases added in cdict: https://github.com/Julow/cdict/pull/3)
and during lookup. Cdict then takes care of resolving the correct word.

The substitutions are generated using mappings from `fn`, `shift` and
all the `accent_*` modifiers into srcs/compose/substitutions.json
This can be updated easily when more mappings are added.

1 files changed, 21 insertions, 9 deletions

diff --git a/srcs/juloo.keyboard2/suggestions/Suggestions.java b/srcs/juloo.keyboard2/suggestions/Suggestions.java
index 638ac64..41a7941 100644
--- a/srcs/juloo.keyboard2/suggestions/Suggestions.java
+++ b/srcs/juloo.keyboard2/suggestions/Suggestions.java
@@ -5,6 +5,8 @@ import java.util.List;
 import juloo.cdict.Cdict;
 import juloo.keyboard2.dict.Dictionaries;
 import juloo.keyboard2.Config;
+import juloo.keyboard2.ComposeKey;
+import juloo.keyboard2.ComposeKeyData;
 
 /** Keep track of the word being typed and provide suggestions for
     [CandidatesView]. */
@@ -49,18 +51,12 @@ public final class Suggestions
 
   int query_suggestions(Cdict dict, String word, String[] dst, int max_count)
   {
+    boolean first_char_upper = Character.isUpperCase(word.charAt(0));
+    word = apply_substitutions(word);
     Cdict.Result r = dict.find(word);
     int i = 0;
     if (r.found)
-      dst[i++] = word;
-    boolean first_char_upper = Character.isUpperCase(word.charAt(0));
-    // Do the dictionary query in lower case and re-apply the upper case after
-    if (first_char_upper)
-    {
-      r = dict.find(word.toLowerCase());
-      if (r.found)
-        dst[i++] = word;
-    }
+      dst[i++] = dict.word(r.index);
     int[] suffixes = dict.suffixes(r, max_count);
     // Disable distance search for small words
     int[] dist = (word.length() < 3 || i + 1 >= max_count) ? NO_RESULTS :
@@ -84,6 +80,22 @@ public final class Suggestions
         rs[i] = rs[i].substring(0, 1).toUpperCase() + rs[i].substring(1);
   }
 
+  /** Apply the same substitutions that were used when building the
+      dictionaries to find word aliases. This catches missing diacritics for
+      example. */
+  String apply_substitutions(String w)
+  {
+    StringBuilder b = new StringBuilder(w);
+    int len = w.length();
+    for (int i = 0; i < len; i++)
+    {
+      char r =
+        ComposeKey.transform_char(ComposeKeyData.substitutions, b.charAt(i));
+      if (r != 0) b.setCharAt(i, r);
+    }
+    return b.toString();
+  }
+
   void set_suggestions(List<String> ws)
   {
     _callback.set_suggestions(ws);