feat(slack): add option to include bot messages during indexing (#8309) to release v2.12 (#9399)

nmgarza5 · web-flow · commit 1263e5540452 · 2026-03-16T18:04:40.000-07:00
diff --git a/backend/onyx/connectors/slack/connector.py b/backend/onyx/connectors/slack/connector.py
@@ -308,6 +308,18 @@ def default_msg_filter(message: MessageType) -> SlackMessageFilterReason | None:
     return None
 
 
+def _bot_inclusive_msg_filter(
+    message: MessageType,
+) -> SlackMessageFilterReason | None:
+    """Like default_msg_filter but allows bot/app messages through.
+    Only filters out disallowed subtypes (channel_join, channel_leave, etc.).
+    """
+    if message.get("subtype", "") in _DISALLOWED_MSG_SUBTYPES:
+        return SlackMessageFilterReason.DISALLOWED
+
+    return None
+
+
 def filter_channels(
     all_channels: list[ChannelType],
     channels_to_connect: list[str] | None,
@@ -654,12 +666,18 @@ def __init__(
         # if specified, will treat the specified channel strings as
         # regexes, and will only index channels that fully match the regexes
         channel_regex_enabled: bool = False,
+        # if True, messages from bots/apps will be indexed instead of filtered out
+        include_bot_messages: bool = False,
         batch_size: int = INDEX_BATCH_SIZE,
         num_threads: int = SLACK_NUM_THREADS,
         use_redis: bool = True,
     ) -> None:
         self.channels = channels
         self.channel_regex_enabled = channel_regex_enabled
+        self.include_bot_messages = include_bot_messages
+        self.msg_filter_func = (
+            _bot_inclusive_msg_filter if include_bot_messages else default_msg_filter
+        )
         self.batch_size = batch_size
         self.num_threads = num_threads
         self.client: WebClient | None = None
@@ -839,6 +857,7 @@ def retrieve_all_slim_docs_perm_sync(
             client=self.client,
             channels=self.channels,
             channel_name_regex_enabled=self.channel_regex_enabled,
+            msg_filter_func=self.msg_filter_func,
             callback=callback,
             workspace_url=self._workspace_url,
         )
@@ -984,6 +1003,7 @@ def _load_from_checkpoint(
                             user_cache=self.user_cache,
                             seen_thread_ts=seen_thread_ts,
                             channel_access=checkpoint.current_channel_access,
+                            msg_filter_func=self.msg_filter_func,
                         )
                     )
 
diff --git a/backend/tests/unit/onyx/connectors/slack/test_message_filtering.py b/backend/tests/unit/onyx/connectors/slack/test_message_filtering.py
@@ -0,0 +1,151 @@
+import pytest
+
+from onyx.connectors.slack.connector import _bot_inclusive_msg_filter
+from onyx.connectors.slack.connector import default_msg_filter
+from onyx.connectors.slack.connector import SlackConnector
+from onyx.connectors.slack.connector import SlackMessageFilterReason
+from onyx.connectors.slack.models import MessageType
+
+
+# -- default_msg_filter tests --
+
+
+@pytest.mark.parametrize(
+    "message,expected_reason",
+    [
+        # Regular user message: not filtered
+        (
+            {"text": "hello", "user": "U123", "ts": "1.0"},
+            None,
+        ),
+        # Bot message with bot_id: filtered as BOT
+        (
+            {"text": "automated update", "bot_id": "B123", "ts": "1.0"},
+            SlackMessageFilterReason.BOT,
+        ),
+        # App message with app_id: filtered as BOT
+        (
+            {"text": "app notification", "app_id": "A123", "ts": "1.0"},
+            SlackMessageFilterReason.BOT,
+        ),
+        # Bot message with both bot_id and app_id: filtered as BOT
+        (
+            {"text": "bot+app", "bot_id": "B1", "app_id": "A1", "ts": "1.0"},
+            SlackMessageFilterReason.BOT,
+        ),
+        # DanswerBot Testing is explicitly allowed through
+        (
+            {
+                "text": "danswer test",
+                "bot_id": "B999",
+                "bot_profile": {"name": "DanswerBot Testing"},
+                "ts": "1.0",
+            },
+            None,
+        ),
+        # channel_join subtype: filtered as DISALLOWED
+        (
+            {"text": "joined", "subtype": "channel_join", "ts": "1.0"},
+            SlackMessageFilterReason.DISALLOWED,
+        ),
+        # channel_leave subtype: filtered as DISALLOWED
+        (
+            {"text": "left", "subtype": "channel_leave", "ts": "1.0"},
+            SlackMessageFilterReason.DISALLOWED,
+        ),
+        # pinned_item subtype: filtered as DISALLOWED
+        (
+            {"text": "pinned", "subtype": "pinned_item", "ts": "1.0"},
+            SlackMessageFilterReason.DISALLOWED,
+        ),
+        # Empty subtype: not filtered
+        (
+            {"text": "normal", "subtype": "", "ts": "1.0"},
+            None,
+        ),
+    ],
+    ids=[
+        "regular_user_message",
+        "bot_id_message",
+        "app_id_message",
+        "bot_and_app_id",
+        "danswerbot_testing_allowed",
+        "channel_join",
+        "channel_leave",
+        "pinned_item",
+        "empty_subtype",
+    ],
+)
+def test_default_msg_filter(
+    message: MessageType,
+    expected_reason: SlackMessageFilterReason | None,
+) -> None:
+    assert default_msg_filter(message) == expected_reason
+
+
+# -- _bot_inclusive_msg_filter tests --
+
+
+@pytest.mark.parametrize(
+    "message,expected_reason",
+    [
+        # Regular user message: not filtered
+        (
+            {"text": "hello", "user": "U123", "ts": "1.0"},
+            None,
+        ),
+        # Bot message: NOT filtered (this is the whole point)
+        (
+            {"text": "automated update", "bot_id": "B123", "ts": "1.0"},
+            None,
+        ),
+        # App message: NOT filtered
+        (
+            {"text": "app notification", "app_id": "A123", "ts": "1.0"},
+            None,
+        ),
+        # channel_join subtype: still filtered as DISALLOWED
+        (
+            {"text": "joined", "subtype": "channel_join", "ts": "1.0"},
+            SlackMessageFilterReason.DISALLOWED,
+        ),
+        # channel_leave subtype: still filtered as DISALLOWED
+        (
+            {"text": "left", "subtype": "channel_leave", "ts": "1.0"},
+            SlackMessageFilterReason.DISALLOWED,
+        ),
+    ],
+    ids=[
+        "regular_user_message",
+        "bot_message_allowed",
+        "app_message_allowed",
+        "channel_join_still_filtered",
+        "channel_leave_still_filtered",
+    ],
+)
+def test_bot_inclusive_msg_filter(
+    message: MessageType,
+    expected_reason: SlackMessageFilterReason | None,
+) -> None:
+    assert _bot_inclusive_msg_filter(message) == expected_reason
+
+
+# -- SlackConnector config tests --
+
+
+def test_default_filter_when_include_bot_messages_false() -> None:
+    """When include_bot_messages is False (default), the default filter is used."""
+    connector = SlackConnector(use_redis=False)
+    assert connector.msg_filter_func is default_msg_filter
+
+
+def test_bot_inclusive_filter_when_include_bot_messages_true() -> None:
+    """When include_bot_messages is True, the bot-inclusive filter is used."""
+    connector = SlackConnector(include_bot_messages=True, use_redis=False)
+    assert connector.msg_filter_func is _bot_inclusive_msg_filter
+
+
+def test_include_bot_messages_defaults_to_false() -> None:
+    """The include_bot_messages config defaults to False for backward compatibility."""
+    connector = SlackConnector(use_redis=False)
+    assert connector.include_bot_messages is False
diff --git a/web/src/components/admin/connectors/ConnectorTitle.tsx b/web/src/components/admin/connectors/ConnectorTitle.tsx
@@ -85,6 +85,9 @@ export const ConnectorTitle = ({
     if (typedConnector.connector_specific_config.channel_regex_enabled) {
       additionalMetadata.set("Channel Regex Enabled", "True");
     }
+    if (typedConnector.connector_specific_config.include_bot_messages) {
+      additionalMetadata.set("Include Bot Messages", "True");
+    }
   } else if (connector.source === "zulip") {
     const typedConnector = connector as Connector<ZulipConfig>;
     additionalMetadata.set(
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
@@ -992,6 +992,15 @@ export const connectorConfigs: Record<
 For example, specifying .*-support.* as a "channel" will cause the connector to include any channels with "-support" in the name.`,
         optional: true,
       },
+      {
+        type: "checkbox",
+        query: "Include bot messages?",
+        label: "Include Bot Messages",
+        name: "include_bot_messages",
+        description:
+          "If enabled, messages from bots and apps will be indexed. Useful for channels that are primarily bot-driven feeds (e.g. CRM updates, automated notes).",
+        optional: true,
+      },
     ],
   },
   slab: {
@@ -1892,6 +1901,7 @@ export interface SlackConfig {
   workspace: string;
   channels?: string[];
   channel_regex_enabled?: boolean;
+  include_bot_messages?: boolean;
 }
 
 export interface SlabConfig {
