From f1309d12ac878c6ca033770f044d40880a3e6a4c Mon Sep 17 00:00:00 2001 From: Kirill Dubovitskiy Date: Thu, 10 Aug 2023 15:45:48 -0700 Subject: Draft fixing broken default configuration and introducing default urls that you reference often --- .../continuedev/libs/constants/default_config.py | 6 +- .../continuedev/plugins/context_providers/url.py | 75 +++++++++++++--------- 2 files changed, 51 insertions(+), 30 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/src/continuedev/libs/constants/default_config.py b/continuedev/src/continuedev/libs/constants/default_config.py index e2b033b7..fc696dfc 100644 --- a/continuedev/src/continuedev/libs/constants/default_config.py +++ b/continuedev/src/continuedev/libs/constants/default_config.py @@ -132,7 +132,11 @@ config = ContinueConfig( # ) SearchContextProvider(), DiffContextProvider(), - URLContextProvider() + URLContextProvider( + commonly_referenced_urls = [ + # Add any common urls you reference here so they appear in auto complete + ] + ) ], # Policies hold the main logic that decides which Step to take next diff --git a/continuedev/src/continuedev/plugins/context_providers/url.py b/continuedev/src/continuedev/plugins/context_providers/url.py index 32c1d69c..1fc2c38f 100644 --- a/continuedev/src/continuedev/plugins/context_providers/url.py +++ b/continuedev/src/continuedev/plugins/context_providers/url.py @@ -1,3 +1,4 @@ + from typing import List, Optional from .util import remove_meilisearch_disallowed_chars @@ -8,37 +9,46 @@ from ...core.context import ContextProvider class URLContextProvider(ContextProvider): title = "url" - url: Optional[str] = None - display_name: Optional[str] = None - URL_CONTEXT_ITEM_ID = "url" - - @property - def optional_url_item_id(self) -> str: - return remove_meilisearch_disallowed_chars(self.url) - - @property - def optional_url_item(self) -> ContextItem: - cp = self.BASE_CONTEXT_ITEM.copy() - if self.display_name: - cp.description.name = self.display_name - cp.description.description = f"Contents of {self.url}" - cp.description.id.item_id = self.optional_url_item_id - - return cp + # Allows users to provide a list of urls that will + commonly_referenced_urls: List[str] = [] + + # There is only a single dynamic url context item, so it has a static id + DYNAMIC_URL_CONTEXT_ITEM_ID = "url" + # This is a template dynamic item that will generate context item on demand + # when get item is called @property - def BASE_CONTEXT_ITEM(self): + def DYNAMIC_CONTEXT_ITEM(self): return ContextItem( content="", description=ContextItemDescription( - name="URL", + name="Dynamic URL", description="Reference the contents of a webpage (e.g. '@url https://www.w3schools.com/python/python_ref_functions.asp')", id=ContextItemId( provider_title=self.title, - item_id=self.URL_CONTEXT_ITEM_ID + item_id=self.DYNAMIC_URL_CONTEXT_ITEM_ID ) ) ) + + # These are staticaly configured items, they will be regenerated + # each time they are accessed + @property + def static_url_context_items_from_config(self) -> List[ContextItem]: + return [ + ContextItem( + content=self._get_url_text_contents(url), + description=ContextItemDescription( + name="Static URL", + description=f"Contents of {url}", + id=ContextItemId( + provider_title=self.title, + item_id=remove_meilisearch_disallowed_chars(url) + ) + ) + ) + for url in self.commonly_referenced_urls + ] def _get_url_text_contents(self, url: str): from bs4 import BeautifulSoup @@ -49,25 +59,32 @@ class URLContextProvider(ContextProvider): return soup.get_text() async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]: - items = [self.BASE_CONTEXT_ITEM] - if self.url: - items.append(self.optional_url_item) + items = [self.DYNAMIC_CONTEXT_ITEM] + items.extend(self.static_url_context_items_from_config) return items async def get_item(self, id: ContextItemId, query: str) -> ContextItem: - if id.item_id == self.optional_url_item_id: - item = self.optional_url_item - item.content = self._get_url_text_contents(self.url) - return item + static_items = self.static_url_context_items_from_config + + # Check if the item is a static item + matching_static_item = next((item for item in static_items if item.description.id.item_id == id.item_id), None) + if matching_static_item: + matching_static_item.description.name = url.replace( + "https://", "").replace("http://", "") + return matching_static_item - if not id.item_id == self.URL_CONTEXT_ITEM_ID: + # Check if the item is the dynamic item + if not id.item_id == self.DYNAMIC_URL_CONTEXT_ITEM_ID: raise Exception("Invalid item id") + # Generate the dynamic item url = query.lstrip("url ").strip() + if url is None or url == "": + return None content = self._get_url_text_contents(url) - ctx_item = self.BASE_CONTEXT_ITEM.copy() + ctx_item = self.DYNAMIC_CONTEXT_ITEM.copy() ctx_item.content = content ctx_item.description.name = url.replace( "https://", "").replace("http://", "") -- cgit v1.2.3-70-g09d2 From 3dabc4bd6c72e2d12afb059040ca75f606e47d9d Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 11 Aug 2023 18:42:03 -0700 Subject: feat: :recycle: load preset_urls at load_index --- .../continuedev/libs/constants/default_config.py | 4 +- .../continuedev/plugins/context_providers/url.py | 71 +++++++++++----------- 2 files changed, 36 insertions(+), 39 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/src/continuedev/libs/constants/default_config.py b/continuedev/src/continuedev/libs/constants/default_config.py index fc696dfc..a22f941d 100644 --- a/continuedev/src/continuedev/libs/constants/default_config.py +++ b/continuedev/src/continuedev/libs/constants/default_config.py @@ -133,8 +133,8 @@ config = ContinueConfig( SearchContextProvider(), DiffContextProvider(), URLContextProvider( - commonly_referenced_urls = [ - # Add any common urls you reference here so they appear in auto complete + preset_urls = [ + # Add any common urls you reference here so they appear in autocomplete ] ) ], diff --git a/continuedev/src/continuedev/plugins/context_providers/url.py b/continuedev/src/continuedev/plugins/context_providers/url.py index 1fc2c38f..5b67608d 100644 --- a/continuedev/src/continuedev/plugins/context_providers/url.py +++ b/continuedev/src/continuedev/plugins/context_providers/url.py @@ -1,5 +1,7 @@ -from typing import List, Optional +from typing import List +from bs4 import BeautifulSoup +import requests from .util import remove_meilisearch_disallowed_chars from ...core.main import ContextItem, ContextItemDescription, ContextItemId @@ -9,9 +11,12 @@ from ...core.context import ContextProvider class URLContextProvider(ContextProvider): title = "url" - # Allows users to provide a list of urls that will - commonly_referenced_urls: List[str] = [] - + # Allows users to provide a list of preset urls + preset_urls: List[str] = [] + + # Static items loaded from preset_urls + static_url_context_items: List[ContextItem] = [] + # There is only a single dynamic url context item, so it has a static id DYNAMIC_URL_CONTEXT_ITEM_ID = "url" @@ -30,48 +35,41 @@ class URLContextProvider(ContextProvider): ) ) ) - - # These are staticaly configured items, they will be regenerated - # each time they are accessed - @property - def static_url_context_items_from_config(self) -> List[ContextItem]: - return [ - ContextItem( - content=self._get_url_text_contents(url), - description=ContextItemDescription( - name="Static URL", - description=f"Contents of {url}", - id=ContextItemId( - provider_title=self.title, - item_id=remove_meilisearch_disallowed_chars(url) - ) + + def static_url_context_item_from_url(self, url: str) -> ContextItem: + content, title = self._get_url_text_contents_and_title(url) + return ContextItem( + content=content, + description=ContextItemDescription( + name=title, + description=f"Contents of {url}", + id=ContextItemId( + provider_title=self.title, + item_id=remove_meilisearch_disallowed_chars(url) ) ) - for url in self.commonly_referenced_urls - ] - - def _get_url_text_contents(self, url: str): - from bs4 import BeautifulSoup - import requests + ) + def _get_url_text_contents_and_title(self, url: str) -> (str, str): response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') - return soup.get_text() + title = url.replace( + "https://", "").replace("http://", "").replace("www.", "") + if soup.title is not None: + title = soup.title.string + return soup.get_text(), title async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]: - items = [self.DYNAMIC_CONTEXT_ITEM] - items.extend(self.static_url_context_items_from_config) + self.static_url_context_items = [ + self.static_url_context_item_from_url(url) for url in self.preset_urls] - return items + return [self.DYNAMIC_CONTEXT_ITEM] + self.static_url_context_items async def get_item(self, id: ContextItemId, query: str) -> ContextItem: - static_items = self.static_url_context_items_from_config - # Check if the item is a static item - matching_static_item = next((item for item in static_items if item.description.id.item_id == id.item_id), None) + matching_static_item = next( + (item for item in self.static_url_context_items if item.description.id.item_id == id.item_id), None) if matching_static_item: - matching_static_item.description.name = url.replace( - "https://", "").replace("http://", "") return matching_static_item # Check if the item is the dynamic item @@ -82,12 +80,11 @@ class URLContextProvider(ContextProvider): url = query.lstrip("url ").strip() if url is None or url == "": return None - content = self._get_url_text_contents(url) + content, title = self._get_url_text_contents_and_title(url) ctx_item = self.DYNAMIC_CONTEXT_ITEM.copy() ctx_item.content = content - ctx_item.description.name = url.replace( - "https://", "").replace("http://", "") + ctx_item.description.name = title ctx_item.description.id.item_id = remove_meilisearch_disallowed_chars( url) return ctx_item -- cgit v1.2.3-70-g09d2