diff --git a/backend/.gitignore b/backend/.gitignore index b75d7f4d75..38ac9f33e7 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -8,3 +8,4 @@ qdrant-data/ typesense-data/ .env vespa-app.zip +dynamic_config_storage/ diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index 058b62fbe0..6bd72f18b3 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -59,6 +59,7 @@ class DocumentSource(str, Enum): NOTION = "notion" ZULIP = "zulip" LINEAR = "linear" + HUBSPOT = "hubspot" class DocumentIndexType(str, Enum): diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 6de50524a0..a3df62c576 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -22,6 +22,7 @@ from danswer.connectors.slack.connector import SlackLoadConnector from danswer.connectors.slack.connector import SlackPollConnector from danswer.connectors.web.connector import WebConnector from danswer.connectors.zulip.connector import ZulipConnector +from danswer.connectors.hubspot.connector import HubSpotConnector class ConnectorMissingException(Exception): @@ -50,6 +51,7 @@ def identify_connector_class( DocumentSource.ZULIP: ZulipConnector, DocumentSource.GURU: GuruConnector, DocumentSource.LINEAR: LinearConnector, + DocumentSource.HUBSPOT: HubSpotConnector, } connector_by_source = connector_map.get(source, {}) diff --git a/backend/danswer/connectors/hubspot/__init__.py b/backend/danswer/connectors/hubspot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/danswer/connectors/hubspot/connector.py b/backend/danswer/connectors/hubspot/connector.py new file mode 100644 index 0000000000..0c8b84ae0e --- /dev/null +++ b/backend/danswer/connectors/hubspot/connector.py @@ -0,0 +1,138 @@ +import requests +import json +from typing import Any +from datetime import datetime +from hubspot import HubSpot +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.utils.logger import setup_logger + +HUBSPOT_BASE_URL = "https://app.hubspot.com/contacts/" +HUBSPOT_API_URL = "https://api.hubapi.com/integrations/v1/me" + +logger = setup_logger() + +class HubSpotConnector(LoadConnector, PollConnector): + def __init__(self, batch_size: int = INDEX_BATCH_SIZE, access_token: str | None = None) -> None: + self.batch_size = batch_size + self.access_token = access_token + self.portal_id = None + self.ticket_base_url = HUBSPOT_BASE_URL + + def get_portal_id(self) -> str: + headers = { + 'Authorization': f'Bearer {self.access_token}', + 'Content-Type': 'application/json' + } + + response = requests.get(HUBSPOT_API_URL, headers=headers) + if response.status_code != 200: + raise Exception("Error fetching portal ID") + + data = response.json() + return data["portalId"] + + def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + self.access_token = credentials["hubspot_access_token"] + + if self.access_token: + self.portal_id = self.get_portal_id() + self.ticket_base_url = f"{HUBSPOT_BASE_URL}{self.portal_id}/ticket/" + + return None + + def _process_tickets( + self, start: datetime | None = None, end: datetime | None = None + ) -> GenerateDocumentsOutput: + if self.access_token is None: + raise ConnectorMissingCredentialError("HubSpot") + + api_client = HubSpot(access_token=self.access_token) + all_tickets = api_client.crm.tickets.get_all(associations=['contacts', 'notes']) + + doc_batch: list[Document] = [] + + for ticket in all_tickets: + updated_at = ticket.updated_at.replace(tzinfo=None) + if start is not None and updated_at < start: + continue + if end is not None and updated_at > end: + continue + + title = ticket.properties["subject"] + link = self.ticket_base_url + ticket.id + content_text = title + "\n" + ticket.properties["content"] + + associated_emails = [] + associated_notes = [] + + if ticket.associations: + contacts = ticket.associations.get("contacts") + notes = ticket.associations.get("notes") + + if contacts: + for contact in contacts.results: + contact = api_client.crm.contacts.basic_api.get_by_id(contact_id=contact.id) + associated_emails.append(contact.properties["email"]) + + if notes: + for note in notes.results: + note = api_client.crm.objects.notes.basic_api.get_by_id(note_id=note.id, properties=["content", "hs_body_preview"]) + associated_notes.append(note.properties["hs_body_preview"]) + + associated_emails = " ,".join(associated_emails) + associated_notes = " ".join(associated_notes) + + content_text = f"{content_text}\n emails: {associated_emails} \n notes: {associated_notes}" + + doc_batch.append( + Document( + id=ticket.id, + sections=[Section(link=link, text=content_text)], + source=DocumentSource.HUBSPOT, + semantic_identifier=title, + metadata={}, + ) + ) + + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + + if doc_batch: + yield doc_batch + + def load_from_state(self) -> GenerateDocumentsOutput: + return self._process_tickets() + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + start_datetime = datetime.fromtimestamp(start) + end_datetime = datetime.fromtimestamp(end) + return self._process_tickets(start_datetime, end_datetime) + + + +if __name__ == "__main__": + import os + import time + test_connector = HubSpotConnector() + test_connector.load_credentials({ + "hubspot_access_token": os.environ["HUBSPOT_ACCESS_TOKEN"] + }) + all_docs = test_connector.load_from_state() + + current = time.time() + one_day_ago = current - 24 * 60 * 60 # 1 day + latest_docs = test_connector.poll_source(one_day_ago, current) + print(latest_docs) + + diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 0cc8effa90..64620cf19e 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -53,3 +53,4 @@ transformers==4.30.1 typesense==0.15.1 uvicorn==0.21.1 zulip==0.8.2 +hubspot-api-client==8.1.0 diff --git a/web/public/HubSpot.png b/web/public/HubSpot.png new file mode 100644 index 0000000000..e4112aecd0 Binary files /dev/null and b/web/public/HubSpot.png differ diff --git a/web/src/app/admin/connectors/hubspot/page.tsx b/web/src/app/admin/connectors/hubspot/page.tsx new file mode 100644 index 0000000000..04559e4320 --- /dev/null +++ b/web/src/app/admin/connectors/hubspot/page.tsx @@ -0,0 +1,223 @@ +"use client"; + +import * as Yup from "yup"; +import { HubSpotIcon, TrashIcon } from "@/components/icons/icons"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { + Credential, + ConnectorIndexingStatus, + HubSpotConfig, + HubSpotCredentialJson, +} from "@/lib/types"; +import useSWR, { useSWRConfig } from "swr"; +import { fetcher } from "@/lib/fetcher"; +import { LoadingAnimation } from "@/components/Loading"; +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { usePopup } from "@/components/admin/connectors/Popup"; +import { usePublicCredentials } from "@/lib/hooks"; + +const Main = () => { + const { popup, setPopup } = usePopup(); + + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + isValidating: isCredentialsValidating, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + isConnectorIndexingStatusesLoading || + isCredentialsLoading || + isCredentialsValidating + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const hubSpotConnectorIndexingStatuses: ConnectorIndexingStatus< + HubSpotConfig, + HubSpotCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "hubspot" + ); + const hubSpotCredential: Credential = credentialsData.filter( + (credential) => credential.credential_json?.hubspot_access_token + )[0]; + + return ( + <> + {popup} +

+ This connector allows you to sync all your HubSpot Tickets into Danswer. +

+ +

+ Step 1: Provide your Credentials +

+ + {hubSpotCredential ? ( + <> +
+

Existing Access Token:

+

+ {hubSpotCredential.credential_json?.hubspot_access_token} +

+ +
+ + ) : ( + <> +

+ To use the HubSpot connector, provide the HubSpot Access Token. +

+
+ + formBody={ + <> + + + } + validationSchema={Yup.object().shape({ + hubspot_access_token: Yup.string().required( + "Please enter your HubSpot Access Token" + ) + })} + initialValues={{ + hubspot_access_token: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> +
+ + )} + +

+ Step 2: Start indexing! +

+ {hubSpotCredential ? ( + !hubSpotConnectorIndexingStatuses.length ? ( + <> +

+ Click the button below to start indexing! We will pull the latest + tickets from HubSpot every 10 minutes. +

+
+ + nameBuilder={() => "HubSpotConnector"} + source="hubspot" + inputType="poll" + formBody={null} + validationSchema={Yup.object().shape({})} + initialValues={{}} + refreshFreq={10 * 60} // 10 minutes + onSubmit={async (isSuccess, responseJson) => { + if (isSuccess && responseJson) { + await linkCredential(responseJson.id, hubSpotCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> +
+ + ) : ( + <> +

+ HubSpot connector is setup! We are pulling the latest tickets from HubSpot + every 10 minutes. +

+ + connectorIndexingStatuses={hubSpotConnectorIndexingStatuses} + liveCredential={hubSpotCredential} + getCredential={(credential) => { + return ( +
+

{credential.credential_json.hubspot_access_token}

+
+ ); + }} + onCredentialLink={async (connectorId) => { + if (hubSpotCredential) { + await linkCredential(connectorId, hubSpotCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + /> + + ) + ) : ( + <> +

+ Please provide your access token in Step 1 first! Once done with + that, you can then start indexing all your HubSpot tickets. +

+ + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+
+ +

HubSpot

+
+
+
+ ); +} diff --git a/web/src/app/admin/layout.tsx b/web/src/app/admin/layout.tsx index c911b3b83f..58bffb98d0 100644 --- a/web/src/app/admin/layout.tsx +++ b/web/src/app/admin/layout.tsx @@ -19,6 +19,7 @@ import { LinearIcon, UsersIcon, ThumbsUpIcon, + HubSpotIcon, BookmarkIcon, CPUIcon, } from "@/components/icons/icons"; @@ -192,6 +193,15 @@ export default async function AdminLayout({ ), link: "/admin/connectors/file", }, + { + name: ( +
+ +
HubSpot
+
+ ), + link: "/admin/connectors/hubspot", + }, ], }, { diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index 4b5d2344fc..34321051bd 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -41,6 +41,7 @@ import confluenceSVG from "../../../public/Confluence.svg"; import guruIcon from "../../../public/Guru.svg"; import zulipIcon from "../../../public/Zulip.png"; import linearIcon from "../../../public/Linear.png"; +import hubSpotIcon from "../../../public/HubSpot.png"; interface IconProps { size?: number; @@ -421,3 +422,18 @@ export const GuruIcon = ({ Logo ); + +export const HubSpotIcon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => { + return ( +
+ Logo +
+ ); +}; diff --git a/web/src/components/search/Filters.tsx b/web/src/components/search/Filters.tsx index 14e4e9ad6b..c8be96c354 100644 --- a/web/src/components/search/Filters.tsx +++ b/web/src/components/search/Filters.tsx @@ -27,6 +27,7 @@ const sources: Source[] = [ { displayName: "Notion", internalName: "notion" }, { displayName: "Zulip", internalName: "zulip" }, { displayName: "Linear", internalName: "linear" }, + { displayName: "HubSpot", internalName: "hubspot" }, ]; interface SourceSelectorProps { diff --git a/web/src/components/source.tsx b/web/src/components/source.tsx index 683c6c77fa..6db1120bd6 100644 --- a/web/src/components/source.tsx +++ b/web/src/components/source.tsx @@ -14,6 +14,7 @@ import { SlabIcon, SlackIcon, ZulipIcon, + HubSpotIcon, } from "./icons/icons"; interface SourceMetadata { @@ -108,6 +109,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => { displayName: "Linear", adminPageLink: "/admin/connectors/linear", }; + case "hubspot": + return { + icon: HubSpotIcon, + displayName: "HubSpot", + adminPageLink: "/admin/connectors/hubspot", + }; default: throw new Error("Invalid source type"); } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 594f819d8c..cfaf565d13 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -21,6 +21,7 @@ export type ValidSources = | "guru" | "zulip" | "linear" + | "hubspot" | "file"; export type ValidInputTypes = "load_state" | "poll" | "event"; export type ValidStatuses = @@ -106,6 +107,8 @@ export interface ZulipConfig { export interface NotionConfig {} +export interface HubSpotConfig {} + export interface IndexAttemptSnapshot { status: ValidStatuses | null; num_docs_indexed: number; @@ -204,6 +207,10 @@ export interface LinearCredentialJson { linear_api_key: string; } +export interface HubSpotCredentialJson { + hubspot_access_token: string; +} + // DELETION export interface DeletionAttemptSnapshot {