Using heuristics autofill if fill assist does not have rules for that type

This commit is contained in:
Andre Rosado committed 2026-06-24 16:45:34 +01:00
1 parent 66b7f36567
commit 3a2fe1e011
3 files changed
+403 -56

No files matched your search

@@ -55,6 +55,22 @@ private val URL_BARS: Map<String, String> = mapOf(
"com.brave.browser_nightly" to "url_bar",
)
/**
* A list of categories from Fill Assist that are used for [AutofillView.Login]
*/
private val LOGIN_FILL_ASSIST_CATEGORIES: List<String> = listOf(
"account-login",
"account-creation",
"account-update",
)
/**
* A list of categories from Fill Assist that are used for [AutofillView.Card]
*/
private val CARD_FILL_ASSIST_CATEGORIES: List<String> = listOf(
"payment-card",
)
/**
* The default [AutofillParser] implementation for the app. This is a tool for parsing autofill data
* from the OS into domain models.
@@ -107,62 +123,27 @@ class AutofillParserImpl(
val urlBarWebsite = traversalDataList
.flatMap { it.urlBarWebsites }
.firstOrNull()
val autofillViews = traversalDataList.toAutofillViews(urlBarWebsite = urlBarWebsite)
// Take only the autofill views from the node that currently has focus.
// Then remove all the fields that cannot be filled with data.
// We fallback to taking all the fillable views if nothing has focus.
val autofillViewsList = traversalDataList.map { it.autofillViews }
val autofillViews = (autofillViewsList
.filter { views -> views.any { it.data.isFocused } }
.flatten()
.filter { it !is AutofillView.Unused }
.takeUnless { it.isEmpty() }
?: autofillViewsList
.flatten()
.filter { it !is AutofillView.Unused })
.map { it.updateWebsiteIfNecessary(website = urlBarWebsite) }
// Find the focused view, or fallback to the first fillable item on the screen (so
// we at least have something to hook into)
val focusedView = autofillViews
.firstOrNull { it.data.isFocused }
val focusedView = autofillViews.firstOrNull { it.data.isFocused }
?: autofillViews.firstOrNull()
?: return AutofillRequest.Unfillable
if (focusedView == null) {
// The view is unfillable if there are no focused views.
val packageName =
traversalDataList.buildPackageNameOrNull(assistStructure = assistStructure)
val uri = focusedView.buildUriOrNull(packageName = packageName)
if ((settingsRepository.blockedAutofillUris + BLOCK_LISTED_URIS).contains(uri)) {
return AutofillRequest.Unfillable
}
val packageName = traversalDataList.buildPackageNameOrNull(
val effectiveViews = resolveEffectiveViews(
assistStructure = assistStructure,
autofillViews = autofillViews,
uri = uri,
focusedView = focusedView,
urlBarWebsite = urlBarWebsite,
)
val uri = focusedView.buildUriOrNull(
packageName = packageName,
)
val blockListedURIs = settingsRepository.blockedAutofillUris + BLOCK_LISTED_URIS
if (blockListedURIs.contains(uri)) {
// The view is unfillable if the URI is block listed.
return AutofillRequest.Unfillable
}
// Apply fill-assist targeting rules when the feature flag is enabled. Rules take priority
// over heuristics; unmatched view nodes are excluded entirely (no heuristic fallback).
val fillAssistHostRules = uri
?.takeUnless { it.startsWith("androidapp://") }?.toUri()?.host
?.takeIf { featureFlagManager.getFeatureFlag(FlagKey.FillAssistTargetingRules) }
?.let { host ->
fillAssistManager.getFillAssistRules()?.hostRules?.get(host.removePrefix("www."))
}
val effectiveViews = fillAssistHostRules
?.let { rules ->
assistStructure.buildFillAssistViews(
hostRules = rules,
urlBarWebsite = urlBarWebsite,
)
}
?: autofillViews
val effectiveFocusedView = effectiveViews
.firstOrNull { it.data.isFocused }
@@ -196,7 +177,6 @@ class AutofillParserImpl(
// Get inline information if available
val isInlineAutofillEnabled = settingsRepository.isInlineAutofillEnabled
Timber.d("Autofill request isInlineEnabled=$isInlineAutofillEnabled -- ${fillRequest?.id}")
val maxInlineSuggestionsCount = fillRequest.getMaxInlineSuggestionsCount(
autofillAppInfo = autofillAppInfo,
isInlineAutofillEnabled = isInlineAutofillEnabled,
@@ -215,6 +195,46 @@ class AutofillParserImpl(
uri = uri,
)
}
/**
* Returns the effective [AutofillView] list for filling. Applies fill-assist targeting rules
* when the feature flag is enabled and the host rules cover the current partition type;
* otherwise returns the heuristic [autofillViews].
*/
private fun resolveEffectiveViews(
assistStructure: AssistStructure,
autofillViews: List<AutofillView>,
uri: String?,
focusedView: AutofillView,
urlBarWebsite: String?,
): List<AutofillView> {
val hostRules = uri
?.takeUnless { it.startsWith("androidapp://") }
?.toUri()
?.host
?.takeIf { featureFlagManager.getFeatureFlag(FlagKey.FillAssistTargetingRules) }
?.let { host ->
fillAssistManager.getFillAssistRules()?.hostRules?.get(host.removePrefix("www."))
}
?: return autofillViews
val coversCurrentPartition = hostRules.any { rule ->
when (focusedView) {
is AutofillView.Card -> rule.category in CARD_FILL_ASSIST_CATEGORIES
is AutofillView.Login -> rule.category in LOGIN_FILL_ASSIST_CATEGORIES
is AutofillView.Unused -> false
}
}
return if (coversCurrentPartition) {
assistStructure.buildFillAssistViews(
hostRules = hostRules,
urlBarWebsite = urlBarWebsite,
)
} else {
autofillViews
}
}
}
/**
@@ -231,6 +251,27 @@ private fun AssistStructure.traverse(): List<ViewNodeTraversalData> =
?.updateForMissingUsernameFields()
}
/**
* Assembles the [AutofillView] list from this [ViewNodeTraversalData] list.
* Take only the autofill views from the node that currently has focus.
* Then remove all the fields that cannot be filled with data.
* We fall back to taking all the fillable views if nothing has focus.
*/
private fun List<ViewNodeTraversalData>.toAutofillViews(
urlBarWebsite: String?,
): List<AutofillView> {
val viewsLists = map { it.autofillViews }
return (viewsLists
.filter { views -> views.any { it.data.isFocused } }
.flatten()
.filter { it !is AutofillView.Unused }
.takeUnless { it.isEmpty() }
?: viewsLists
.flatten()
.filter { it !is AutofillView.Unused })
.map { it.updateWebsiteIfNecessary(website = urlBarWebsite) }
}
/**
* This helper function updates the [ViewNodeTraversalData] if necessary for missing password
* fields that were marked invalid because they contained a specific `hint` or `idEntry`. If the
@@ -1,9 +1,12 @@
package com.x8bit.bitwarden.data.autofill.parser
import android.app.assist.AssistStructure
import android.net.Uri
import android.net.Uri.parse
import android.service.autofill.FillContext
import android.service.autofill.FillRequest
import android.view.View
import android.view.ViewStructure.HtmlInfo
import android.view.autofill.AutofillId
import android.widget.inline.InlinePresentationSpec
import com.bitwarden.core.data.manager.model.FlagKey
@@ -12,12 +15,15 @@ import com.x8bit.bitwarden.data.autofill.model.AutofillAppInfo
import com.x8bit.bitwarden.data.autofill.model.AutofillPartition
import com.x8bit.bitwarden.data.autofill.model.AutofillRequest
import com.x8bit.bitwarden.data.autofill.model.AutofillView
import com.x8bit.bitwarden.data.autofill.model.FillAssistRules
import com.x8bit.bitwarden.data.autofill.model.ViewNodeTraversalData
import com.x8bit.bitwarden.data.autofill.util.buildPackageNameOrNull
import com.x8bit.bitwarden.data.autofill.util.buildUriOrNull
import com.x8bit.bitwarden.data.autofill.util.getInlinePresentationSpecs
import com.x8bit.bitwarden.data.autofill.util.getMaxInlineSuggestionsCount
import com.x8bit.bitwarden.data.autofill.util.matchesSelectorClause
import com.x8bit.bitwarden.data.autofill.util.toAutofillView
import com.x8bit.bitwarden.data.autofill.util.toAutofillViewData
import com.x8bit.bitwarden.data.autofill.util.website
import com.x8bit.bitwarden.data.platform.manager.FeatureFlagManager
import com.x8bit.bitwarden.data.platform.repository.SettingsRepository
@@ -26,6 +32,7 @@ import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.unmockkStatic
import io.mockk.verify
import kotlinx.coroutines.flow.MutableStateFlow
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.BeforeEach
@@ -43,6 +50,7 @@ class AutofillParserTests {
every { this@mockk.autofillHints } returns arrayOf(cardAutofillHint)
every { this@mockk.autofillId } returns cardAutofillId
every { this@mockk.childCount } returns 0
every { this@mockk.htmlInfo } returns mockk(relaxed = true)
every { this@mockk.idPackage } returns ID_PACKAGE
every { this@mockk.idEntry } returns null
}
@@ -52,6 +60,7 @@ class AutofillParserTests {
every { this@mockk.autofillHints } returns arrayOf(loginAutofillHint)
every { this@mockk.autofillId } returns loginAutofillId
every { this@mockk.childCount } returns 0
every { this@mockk.htmlInfo } returns mockk(relaxed = true)
every { this@mockk.idPackage } returns ID_PACKAGE
every { this@mockk.idEntry } returns null
}
@@ -74,16 +83,39 @@ class AutofillParserTests {
every { blockedAutofillUris } returns emptyList()
}
private val fillAssistManager: FillAssistManager = mockk()
private val mutableFillAssistFlagFlow = MutableStateFlow(false)
private val featureFlagManager: FeatureFlagManager = mockk {
every { getFeatureFlag(FlagKey.FillAssistTargetingRules) } returns false
every {
getFeatureFlag(FlagKey.FillAssistTargetingRules)
} answers {
mutableFillAssistFlagFlow.value
}
every { getFeatureFlagFlow(FlagKey.ManageDevices) } returns mutableFillAssistFlagFlow
}
private var mockIsInlineAutofillEnabled = true
private var mockIsFillAssistEnabled = false
@BeforeEach
fun setup() {
mockIsFillAssistEnabled = false
// toAutofillView, website, and toAutofillViewData all compile into the same
// ViewNodeExtensionsKt class, so one mockkStatic call covers all three.
mockkStatic(AssistStructure.ViewNode::toAutofillView)
mockkStatic(AssistStructure.ViewNode::website)
// Default stub for toAutofillViewData (same mocked class — no separate mockkStatic needed).
every {
any<AssistStructure.ViewNode>().toAutofillViewData(autofillId = any(), website = any())
} answers {
AutofillView.Data(
autofillId = firstArg(),
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = false,
textValue = null,
hasPasswordTerms = false,
website = secondArg(),
)
}
mockkStatic(
FillRequest::getMaxInlineSuggestionsCount,
FillRequest::getInlinePresentationSpecs,
@@ -137,12 +169,20 @@ class AutofillParserTests {
fillAssistManager = fillAssistManager,
featureFlagManager = featureFlagManager,
)
mockkStatic(Uri::parse)
every { parse(any()) } returns mockk {
every { host } returns FILL_ASSIST_URI
}
mockkStatic(HtmlInfo::matchesSelectorClause)
every { any<HtmlInfo>().matchesSelectorClause(any()) } returns false
}
@AfterEach
fun teardown() {
unmockkStatic(AssistStructure.ViewNode::toAutofillView)
unmockkStatic(AssistStructure.ViewNode::website)
unmockkStatic(Uri::parse)
unmockkStatic(HtmlInfo::matchesSelectorClause)
unmockkStatic(
FillRequest::getMaxInlineSuggestionsCount,
FillRequest::getInlinePresentationSpecs,
@@ -554,10 +594,9 @@ class AutofillParserTests {
val rootViewNode: AssistStructure.ViewNode = mockk {
every { this@mockk.autofillHints } returns emptyArray()
every { this@mockk.autofillId } returns rootAutofillId
every { this@mockk.childCount } returns 0
every { this@mockk.childCount } returns 2
every { this@mockk.idPackage } returns ID_PACKAGE
every { this@mockk.website } returns WEBSITE
every { this@mockk.childCount } returns 2
every { this@mockk.getChildAt(0) } returns hiddenUserNameViewNode
every { this@mockk.getChildAt(1) } returns passwordViewNode
}
@@ -956,6 +995,273 @@ class AutofillParserTests {
}
}
@Suppress("MaxLineLength")
@Test
fun `parse should fall back to heuristics when fill-assist rules exist but only cover login and a card view is focused`() {
// Setup: fill-assist enabled with login-only rules, but a card view is focused.
mutableFillAssistFlagFlow.value = true
mockIsFillAssistEnabled = true
every {
any<AutofillView>().buildUriOrNull(PACKAGE_NAME)
} returns FILL_ASSIST_URI
every { fillAssistManager.getFillAssistRules() } returns FillAssistRules(
hostRules = mapOf(
FILL_ASSIST_URI to listOf(
FillAssistRules.HostRule(
category = "account-login",
fields = mapOf(
"username" to listOf(
FillAssistRules.SelectorClause(
tag = "input",
id = "user",
name = null,
type = null,
role = null,
),
),
),
),
),
),
)
every { assistStructure.windowNodeCount } returns 1
every { assistStructure.getWindowNodeAt(0) } returns cardWindowNode
val cardAutofillView = AutofillView.Card.ExpirationYear(
data = AutofillView.Data(
autofillId = cardAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = FILL_ASSIST_URI,
),
yearValue = null,
)
every { cardViewNode.toAutofillView(parentWebsite = any()) } returns cardAutofillView
// Test
val actual = parser.parse(autofillAppInfo = autofillAppInfo, fillRequest = fillRequest)
// Verify: heuristic card view used
val expected = AutofillRequest.Fillable(
ignoreAutofillIds = emptyList(),
inlinePresentationSpecs = inlinePresentationSpecs,
maxInlineSuggestionsCount = MAX_INLINE_SUGGESTION_COUNT,
packageName = PACKAGE_NAME,
partition = AutofillPartition.Card(views = listOf(cardAutofillView)),
uri = FILL_ASSIST_URI,
)
assertEquals(expected, actual)
}
@Suppress("MaxLineLength")
@Test
fun `parse should fall back to heuristics when fill-assist rules exist but only cover payment-card and a login view is focused`() {
// Setup: fill-assist enabled with card-only rules, but a login view is focused.
mutableFillAssistFlagFlow.value = true
mockIsFillAssistEnabled = true
every { any<AutofillView>().buildUriOrNull(PACKAGE_NAME) } returns FILL_ASSIST_URI
every { fillAssistManager.getFillAssistRules() } returns FillAssistRules(
hostRules = mapOf(
FILL_ASSIST_URI to listOf(
FillAssistRules.HostRule(
category = "payment-card",
fields = mapOf(
"cardNumber" to listOf(
FillAssistRules.SelectorClause(
tag = "input",
id = "card-number",
name = null,
type = null,
role = null,
),
),
),
),
),
),
)
every { assistStructure.windowNodeCount } returns 1
every { assistStructure.getWindowNodeAt(0) } returns loginWindowNode
val loginAutofillView = AutofillView.Login.Username(
data = AutofillView.Data(
autofillId = loginAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = FILL_ASSIST_URI,
),
)
every { loginViewNode.toAutofillView(parentWebsite = any()) } returns loginAutofillView
// Test
val actual = parser.parse(autofillAppInfo = autofillAppInfo, fillRequest = fillRequest)
// Verify: heuristic login view used
val expected = AutofillRequest.Fillable(
ignoreAutofillIds = emptyList(),
inlinePresentationSpecs = inlinePresentationSpecs,
maxInlineSuggestionsCount = MAX_INLINE_SUGGESTION_COUNT,
packageName = PACKAGE_NAME,
partition = AutofillPartition.Login(views = listOf(loginAutofillView)),
uri = FILL_ASSIST_URI,
)
assertEquals(expected, actual)
}
@Suppress("MaxLineLength")
@Test
fun `parse should use fill-assist views when rules cover login and a login view is focused`() {
// Setup: fill-assist with login rules, login view focused.
// The heuristic and fill-assist paths produce views with DIFFERENT autofillIds so the
// assertion proves which path was actually taken. If heuristics are used the partition
// contains loginAutofillId; if fill-assist is used it contains fillAssistAutofillId.
mutableFillAssistFlagFlow.value = true
mockIsFillAssistEnabled = true
every { any<AutofillView>().buildUriOrNull(PACKAGE_NAME) } returns FILL_ASSIST_URI
every { fillAssistManager.getFillAssistRules() } returns FillAssistRules(
hostRules = mapOf(
FILL_ASSIST_URI to listOf(
FillAssistRules.HostRule(
category = "account-login",
fields = mapOf(
"username" to listOf(
FillAssistRules.SelectorClause(
tag = "input",
id = "user",
name = null,
type = null,
role = null,
),
),
),
),
),
),
)
val heuristicLoginView = AutofillView.Login.Username(
data = AutofillView.Data(
autofillId = loginAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = FILL_ASSIST_URI,
),
)
val fillAssistAutofillId: AutofillId = mockk()
val fillAssistLoginData = AutofillView.Data(
autofillId = fillAssistAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = WEBSITE,
)
every { any<HtmlInfo>().matchesSelectorClause(any()) } returns true
every {
loginViewNode.toAutofillViewData(autofillId = loginAutofillId, website = WEBSITE)
} returns fillAssistLoginData
every { assistStructure.windowNodeCount } returns 1
every { assistStructure.getWindowNodeAt(0) } returns loginWindowNode
every { loginViewNode.toAutofillView(parentWebsite = any()) } returns heuristicLoginView
// Test
val actual = parser.parse(autofillAppInfo = autofillAppInfo, fillRequest = fillRequest)
// Verify: fill-assist views used — partition contains fillAssistAutofillId.
// Heuristics would have produced loginAutofillId
val expected = AutofillRequest.Fillable(
ignoreAutofillIds = emptyList(),
inlinePresentationSpecs = inlinePresentationSpecs,
maxInlineSuggestionsCount = MAX_INLINE_SUGGESTION_COUNT,
packageName = PACKAGE_NAME,
partition = AutofillPartition.Login(
views = listOf(AutofillView.Login.Username(data = fillAssistLoginData)),
),
uri = FILL_ASSIST_URI,
)
assertEquals(expected, actual)
}
@Suppress("MaxLineLength")
@Test
fun `parse should use fill-assist when rules cover payment-card and a card view is focused`() {
mutableFillAssistFlagFlow.value = true
mockIsFillAssistEnabled = true
every { any<AutofillView>().buildUriOrNull(PACKAGE_NAME) } returns FILL_ASSIST_URI
every { fillAssistManager.getFillAssistRules() } returns FillAssistRules(
hostRules = mapOf(
FILL_ASSIST_URI to listOf(
FillAssistRules.HostRule(
category = "payment-card",
fields = mapOf(
"cardNumber" to listOf(
FillAssistRules.SelectorClause(
tag = "input",
id = "card-number",
name = null,
type = null,
role = null,
),
),
),
),
),
),
)
val heuristicCardView = AutofillView.Card.ExpirationYear(
data = AutofillView.Data(
autofillId = cardAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = FILL_ASSIST_URI,
),
yearValue = null,
)
val fillAssistAutofillId: AutofillId = mockk()
val fillAssistCardData = AutofillView.Data(
autofillId = fillAssistAutofillId,
autofillOptions = emptyList(),
autofillType = AUTOFILL_TYPE,
isFocused = true,
textValue = null,
hasPasswordTerms = false,
website = WEBSITE,
)
every { any<HtmlInfo>().matchesSelectorClause(any()) } returns true
every {
cardViewNode.toAutofillViewData(autofillId = cardAutofillId, website = WEBSITE)
} returns fillAssistCardData
every { assistStructure.windowNodeCount } returns 1
every { assistStructure.getWindowNodeAt(0) } returns cardWindowNode
every { cardViewNode.toAutofillView(parentWebsite = any()) } returns heuristicCardView
// Test
val actual = parser.parse(autofillAppInfo = autofillAppInfo, fillRequest = fillRequest)
// Verify: fill-assist views used — partition contains fillAssistAutofillId.
val expected = AutofillRequest.Fillable(
ignoreAutofillIds = emptyList(),
inlinePresentationSpecs = inlinePresentationSpecs,
maxInlineSuggestionsCount = MAX_INLINE_SUGGESTION_COUNT,
packageName = PACKAGE_NAME,
partition = AutofillPartition.Card(
views = listOf(AutofillView.Card.Number(data = fillAssistCardData)),
),
uri = FILL_ASSIST_URI,
)
assertEquals(expected, actual)
}
/**
* Setup [assistStructure] to return window nodes with each [AutofillView] type (card and login)
* so we can test how different window node configurations produce different partitions.
@@ -967,6 +1273,8 @@ class AutofillParserTests {
}
}
private const val FILL_ASSIST_URI: String = "https://example.com"
private val BLOCK_LISTED_URIS: List<String> = listOf(
"androidapp://android",
"androidapp://com.android.settings",
@@ -23,14 +23,12 @@ class FillAssistViewNodeExtensionsTest {
@BeforeEach
fun setup() {
mockkStatic(AssistStructure.ViewNode::website)
mockkStatic(AssistStructure.ViewNode::toAutofillViewData)
mockkStatic(HtmlInfo::matchesSelectorClause)
}
@AfterEach
fun teardown() {
unmockkStatic(AssistStructure.ViewNode::website)
unmockkStatic(AssistStructure.ViewNode::toAutofillViewData)
unmockkStatic(HtmlInfo::matchesSelectorClause)
}