diff options
| -rw-r--r-- | continuedev/src/continuedev/libs/constants/default_config.py | 4 | ||||
| -rw-r--r-- | continuedev/src/continuedev/plugins/context_providers/url.py | 71 |
2 files changed, 36 insertions, 39 deletions
diff --git a/continuedev/src/continuedev/libs/constants/default_config.py b/continuedev/src/continuedev/libs/constants/default_config.py index fc696dfc..a22f941d 100644 --- a/continuedev/src/continuedev/libs/constants/default_config.py +++ b/continuedev/src/continuedev/libs/constants/default_config.py @@ -133,8 +133,8 @@ config = ContinueConfig( SearchContextProvider(), DiffContextProvider(), URLContextProvider( - commonly_referenced_urls = [ - # Add any common urls you reference here so they appear in auto complete + preset_urls = [ + # Add any common urls you reference here so they appear in autocomplete ] ) ], diff --git a/continuedev/src/continuedev/plugins/context_providers/url.py b/continuedev/src/continuedev/plugins/context_providers/url.py index 1fc2c38f..5b67608d 100644 --- a/continuedev/src/continuedev/plugins/context_providers/url.py +++ b/continuedev/src/continuedev/plugins/context_providers/url.py @@ -1,5 +1,7 @@ -from typing import List, Optional +from typing import List +from bs4 import BeautifulSoup +import requests from .util import remove_meilisearch_disallowed_chars from ...core.main import ContextItem, ContextItemDescription, ContextItemId @@ -9,9 +11,12 @@ from ...core.context import ContextProvider class URLContextProvider(ContextProvider): title = "url" - # Allows users to provide a list of urls that will - commonly_referenced_urls: List[str] = [] - + # Allows users to provide a list of preset urls + preset_urls: List[str] = [] + + # Static items loaded from preset_urls + static_url_context_items: List[ContextItem] = [] + # There is only a single dynamic url context item, so it has a static id DYNAMIC_URL_CONTEXT_ITEM_ID = "url" @@ -30,48 +35,41 @@ class URLContextProvider(ContextProvider): ) ) ) - - # These are staticaly configured items, they will be regenerated - # each time they are accessed - @property - def static_url_context_items_from_config(self) -> List[ContextItem]: - return [ - ContextItem( - content=self._get_url_text_contents(url), - description=ContextItemDescription( - name="Static URL", - description=f"Contents of {url}", - id=ContextItemId( - provider_title=self.title, - item_id=remove_meilisearch_disallowed_chars(url) - ) + + def static_url_context_item_from_url(self, url: str) -> ContextItem: + content, title = self._get_url_text_contents_and_title(url) + return ContextItem( + content=content, + description=ContextItemDescription( + name=title, + description=f"Contents of {url}", + id=ContextItemId( + provider_title=self.title, + item_id=remove_meilisearch_disallowed_chars(url) ) ) - for url in self.commonly_referenced_urls - ] - - def _get_url_text_contents(self, url: str): - from bs4 import BeautifulSoup - import requests + ) + def _get_url_text_contents_and_title(self, url: str) -> (str, str): response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') - return soup.get_text() + title = url.replace( + "https://", "").replace("http://", "").replace("www.", "") + if soup.title is not None: + title = soup.title.string + return soup.get_text(), title async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]: - items = [self.DYNAMIC_CONTEXT_ITEM] - items.extend(self.static_url_context_items_from_config) + self.static_url_context_items = [ + self.static_url_context_item_from_url(url) for url in self.preset_urls] - return items + return [self.DYNAMIC_CONTEXT_ITEM] + self.static_url_context_items async def get_item(self, id: ContextItemId, query: str) -> ContextItem: - static_items = self.static_url_context_items_from_config - # Check if the item is a static item - matching_static_item = next((item for item in static_items if item.description.id.item_id == id.item_id), None) + matching_static_item = next( + (item for item in self.static_url_context_items if item.description.id.item_id == id.item_id), None) if matching_static_item: - matching_static_item.description.name = url.replace( - "https://", "").replace("http://", "") return matching_static_item # Check if the item is the dynamic item @@ -82,12 +80,11 @@ class URLContextProvider(ContextProvider): url = query.lstrip("url ").strip() if url is None or url == "": return None - content = self._get_url_text_contents(url) + content, title = self._get_url_text_contents_and_title(url) ctx_item = self.DYNAMIC_CONTEXT_ITEM.copy() ctx_item.content = content - ctx_item.description.name = url.replace( - "https://", "").replace("http://", "") + ctx_item.description.name = title ctx_item.description.id.item_id = remove_meilisearch_disallowed_chars( url) return ctx_item |