Improve Autosuggestions

CONTRIBUTING.md

@@ -23,6 +23,7 @@ If you have questions or would like to communicate with the team, please [join u
 - [Bug reports](#bug-reports)
 - [Feature requests](#feature-requests)
 - [Pull requests](#pull-requests)
+- [Adding keyboards](#adding-keyboards)
 - [Data edits](#data-edits)
 - [Localization](#localization)
 - [Documentation](#documentation)
@@ -356,6 +357,16 @@ Thank you in advance for your contributions!
 
 <sub><a href="#top">Back to top.</a></sub>
 
+## Adding keyboards
+
+Scribe has interest in adding keyboards for any language! Please let the community know if you'd like a Scribe keyboard in your second or native language.
+
+As of now the Scribe-Android keyboard application is leveraging AOSP based autosuggestions and autocompletions. This means that adding a new keyboard that has AOSP based dictionaries available is dramatically easier than adding other languages. Please see the following for a list of available dictionaries:
+
+- [Codeberg:Helium314/aosp-dictionaries](https://codeberg.org/Helium314/aosp-dictionaries)
+
+<sub><a href="#top">Back to top.</a></sub>
+
 ## Data edits
 
 > [!NOTE]\

app/src/keyboards/java/be/scri/helpers/AutocompletionHandler.kt

@@ -19,40 +19,50 @@ class AutocompletionHandler(
     private var autocompleteRunnable: Runnable? = null
 
     companion object {
-        private const val AUTOCOMPLETE_DELAY_MS = 50L
+        private const val AUTOCOMPLETE_DELAY_MS = 150L
+        private const val MAX_COMPLETIONS = 2
+
+        /**
+         * Filters dictionary/engine [completions] down to the ones worth showing
+         * alongside the word the user already typed: no duplicate of [typedWord],
+         * capped at [MAX_COMPLETIONS] (the word itself takes the remaining slot).
+         */
+        internal fun buildCompletions(
+            typedWord: String,
+            completions: List<String>,
+        ): List<String> =
+            completions
+                .filterNot { it.equals(typedWord, ignoreCase = true) }
+                .take(MAX_COMPLETIONS)
     }
 
     /**
      * Processes the current word for autocompletion.
      *
-     * This function is called whenever the user types.
-     * It cancels any pending autocomplete request and schedules a new one
-     * after a short delay to prevent excessive lookups.
+     * This function is called whenever the user types. The word being typed is
+     * shown immediately (it's already known, no lookup needed), while the
+     * dictionary/engine completions are debounced to avoid excessive lookups.
      *
      * @param currentWord The word currently being typed by the user.
      */
     fun processAutocomplete(currentWord: String?) {
         autocompleteRunnable?.let { handler.removeCallbacks(it) }
 
+        if (ime.currentState != ScribeState.IDLE || currentWord.isNullOrEmpty()) {
+            ime.clearAutocomplete()
+            return
+        }
+
+        ime.updateTypedWordSuggestion(currentWord)
+
         autocompleteRunnable =
             Runnable {
-                if (ime.currentState != ScribeState.IDLE) {
-                    ime.clearAutocomplete()
-                    return@Runnable
-                }
-
-                if (currentWord.isNullOrEmpty()) {
-                    ime.clearAutocomplete()
-                    return@Runnable
-                }
+                if (ime.currentState != ScribeState.IDLE) return@Runnable
 
-                val completions = ime.getAutocompletions(currentWord, limit = 5)
+                val previousWord = ime.getPreviousWordBeforeCursor()
+                val completions = ime.getAutocompletions(currentWord, previousWord, limit = MAX_COMPLETIONS + 1)
 
-                if (completions.isNotEmpty()) {
-                    ime.updateAutocompleteSuggestions(completions)
-                } else {
-                    ime.clearAutocomplete()
-                }
+                ime.updateAutocompleteCompletions(buildCompletions(currentWord, completions))
             }
 
         handler.postDelayed(autocompleteRunnable!!, AUTOCOMPLETE_DELAY_MS)

app/src/keyboards/java/be/scri/services/GeneralKeyboardIME.kt

@@ -1208,16 +1208,17 @@ abstract class GeneralKeyboardIME(
      */
     fun getAutocompletions(
         prefix: String,
+        previousWord: String? = null,
         limit: Int = 3,
     ): List<String> {
         if (this::nativeSuggestionEngine.isInitialized) {
-            val nativeCompletions = nativeSuggestionEngine.getAutocompletions(language, prefix, limit)
+            val nativeCompletions = nativeSuggestionEngine.getAutocompletions(language, prefix, previousWord, limit)
             if (nativeCompletions.isNotEmpty()) {
-                return nativeCompletions
+                return nativeCompletions.map { it.substringBefore("-") }
             }
         }
         return try {
-            dbManagers.autocompletionManager.getAutocompletions(prefix, limit)
+            dbManagers.autocompletionManager.getAutocompletions(prefix, limit).map { it.substringBefore("-") }
         } catch (e: SQLiteException) {
             Log.e("GeneralKeyboardIME", "Database error in autocompletion", e)
             emptyList()
@@ -1252,6 +1253,18 @@ abstract class GeneralKeyboardIME(
      */
     fun getLastWordBeforeCursor(): String? = getText()?.trim()?.split("\\s+".toRegex())?.lastOrNull()
 
+    /**
+     * Extracts the word immediately before the one currently being composed, i.e. the last
+     * completed word preceding the in-progress word at the cursor. Used to give the autocomplete
+     * engine sentence context so it can bias completions instead of scoring the prefix in isolation.
+     *
+     * @return The previous completed word as a [String], or null if there isn't one.
+     */
+    fun getPreviousWordBeforeCursor(): String? {
+        val words = getText()?.trim()?.split("\\s+".toRegex()) ?: return null
+        return words.getOrNull(words.size - 2)
+    }
+
     /**
      * Retrieves the text immediately preceding the cursor.
      *
@@ -1499,10 +1512,10 @@ abstract class GeneralKeyboardIME(
         if (this::nativeSuggestionEngine.isInitialized) {
             val nativeSuggestions = nativeSuggestionEngine.getNextWordSuggestions(language, lastWord)
             if (nativeSuggestions.isNotEmpty()) {
-                return nativeSuggestions
+                return nativeSuggestions.map { it.substringBefore("-") }
             }
         }
-        return wordSuggestions[lastWord.lowercase()]
+        return wordSuggestions[lastWord.lowercase()]?.map { it.substringBefore("-") }
     }
 
     /**
@@ -1928,29 +1941,56 @@ abstract class GeneralKeyboardIME(
     // MARK: Autocomplete
 
     /**
-     * Updates autocomplete UI with a new list of suggestions.
-     * Clears it if not idle or no completions.
+     * Pins the word currently being typed into the first (leftmost) suggestion
+     * slot, quoted like most mobile keyboards do to mark it as "what you typed"
+     * rather than a dictionary suggestion. Called immediately on every keystroke
+     * — unlike the completions, it needs no lookup, so it should never lag.
      */
-    fun updateAutocompleteSuggestions(completions: List<String>?) {
-        if (currentState != ScribeState.IDLE) {
-            uiManager.disableAutoSuggest(language)
-            return
-        }
-        if (completions.isNullOrEmpty()) {
+    fun updateTypedWordSuggestion(word: String?) {
+        if (currentState != ScribeState.IDLE || word.isNullOrEmpty()) {
             uiManager.disableAutoSuggest(language)
             return
         }
 
+        setTypedWordButton(uiManager.binding.translateBtn, word)
+        setAutocompleteButton(uiManager.binding.conjugateBtn, "")
+        uiManager.pluralBtn?.let { setAutocompleteButton(it, "") }
+
+        uiManager.binding.separator1.visibility = View.VISIBLE
+        uiManager.binding.separator2.visibility = View.VISIBLE
+    }
+
+    /**
+     * Fills the remaining suggestion slots with dictionary/engine completions.
+     * Clears them (leaving the typed word alone) if not idle.
+     */
+    fun updateAutocompleteCompletions(completions: List<String>) {
+        if (currentState != ScribeState.IDLE) return
+
         val completion1 = completions.getOrNull(0) ?: ""
         val completion2 = completions.getOrNull(1) ?: ""
-        val completion3 = completions.getOrNull(2) ?: ""
 
         setAutocompleteButton(uiManager.binding.conjugateBtn, completion1)
-        setAutocompleteButton(uiManager.binding.translateBtn, completion2)
-        setAutocompleteButton(uiManager.pluralBtn!!, completion3)
+        uiManager.pluralBtn?.let { setAutocompleteButton(it, completion2) }
+    }
 
-        uiManager.binding.separator1.visibility = View.VISIBLE
-        uiManager.binding.separator2.visibility = View.VISIBLE
+    /**
+     * Sets up the "what you typed" button: displayed quoted, but tapping it
+     * doesn't re-insert the word (it's already in the text field) — it just
+     * confirms the word with a space, the same as pressing the space bar
+     * would, and moves on to next-word suggestions based on it.
+     */
+    private fun setTypedWordButton(
+        button: Button,
+        word: String,
+    ) {
+        setSuggestionButton(button, "\"$word\"")
+        button.setOnClickListener {
+            currentInputConnection?.commitText(" ", 1)
+            suggestionHandler.processLinguisticSuggestions(word)
+            suggestionHandler.processWordSuggestions(word)
+            moveToIdleState()
+        }
     }
 
     /**

app/src/main/java/be/scri/helpers/NativeSuggestionEngine.kt

@@ -116,25 +116,38 @@ class NativeSuggestionEngine(private val context: Context) {
     fun getAutocompletions(
         language: String,
         prefix: String,
+        previousWord: String? = null,
         limit: Int = 3
     ): List<String> {
         val dict = getDictionary(language) ?: return emptyList()
         if (prefix.isBlank()) return emptyList()
 
         return try {
             val composedData = ComposedData.createForWord(prefix)
+            val ngramContext =
+                if (previousWord.isNullOrBlank()) {
+                    NgramContext.EMPTY_PREV_WORDS_INFO
+                } else {
+                    NgramContext(NgramContext.WordInfo(previousWord))
+                }
             val suggestions = dict.getSuggestions(
                 composedData,
-                NgramContext.EMPTY_PREV_WORDS_INFO,
+                ngramContext,
                 dummyProximityInfo.nativeProximityInfo, // proximityInfoHandle
-                SettingsValuesForSuggestion(false, false),
+                SettingsValuesForSuggestion(true, false), // blockPotentiallyOffensive, spaceAwareGesture
                 1, // sessionId
                 1.0f, // weightForLocale
                 null // inOutWeightOfLangModelVsSpatialModel
             )
 
+            val isCapitalized = StringUtils.isWordCapitalized(prefix)
             suggestions?.map { it.mWord }
-                ?.filter { it.isNotBlank() && it.lowercase(Locale.ROOT) != prefix.lowercase(Locale.ROOT) }
+                ?.filter {
+                    it.isNotBlank() &&
+                        it.startsWith(prefix, ignoreCase = true) &&
+                        it.lowercase(Locale.ROOT) != prefix.lowercase(Locale.ROOT)
+                }
+                ?.map { if (isCapitalized) it.replaceFirstChar { c -> c.uppercaseChar() } else it }
                 ?.take(limit)
                 ?: emptyList()
         } catch (e: Exception) {
@@ -162,14 +175,14 @@ class NativeSuggestionEngine(private val context: Context) {
                 composedData,
                 ngramContext,
                 dummyProximityInfo.nativeProximityInfo, // proximityInfoHandle
-                SettingsValuesForSuggestion(false, false),
+                SettingsValuesForSuggestion(true, false), // blockPotentiallyOffensive, spaceAwareGesture
                 1, // sessionId
                 1.0f, // weightForLocale
                 null // inOutWeightOfLangModelVsSpatialModel
             )
 
             suggestions?.map { it.mWord }
-                ?.filter { it.isNotBlank() }
+                ?.filter { it.isNotBlank() && it.lowercase(Locale.ROOT) != lastWord.lowercase(Locale.ROOT) }
                 ?.take(limit)
                 ?: emptyList()
         } catch (e: Exception) {

app/src/main/java/be/scri/latin/common/ComposedData.kt

@@ -8,8 +8,6 @@
  */
 package be.scri.latin.common
 
-import kotlin.random.Random
-
 /** An immutable class that encapsulates a snapshot of word composition data. */
 class ComposedData(
     @JvmField val mInputPointers: InputPointers,
@@ -48,10 +46,12 @@ class ComposedData(
     companion object {
         fun createForWord(word: String): ComposedData {
             val codePoints = StringUtils.toCodePointArray(word)
-            val coordinates = CoordinateUtils.newCoordinateArray(codePoints.size)
-            for (i in codePoints.indices) {
-                CoordinateUtils.setXYInArray(coordinates, i, Random.nextBits(2), Random.nextBits(2))
-            }
+            val coordinates =
+                CoordinateUtils.newCoordinateArray(
+                    codePoints.size,
+                    Constants.NOT_A_COORDINATE,
+                    Constants.NOT_A_COORDINATE,
+                )
             val pointers = InputPointers(codePoints.size).apply {
                 for (i in codePoints.indices) {
                     addPointer(CoordinateUtils.xFromArray(coordinates, i), CoordinateUtils.yFromArray(coordinates, i), 0, 0)