Merge branch 'main' of https://github.com/continuedev/continue

2 files changed, 56 insertions, 38 deletions

diff --git a/continuedev/src/continuedev/libs/constants/default_config.py b/continuedev/src/continuedev/libs/constants/default_config.py
index e2b033b7..a22f941d 100644
--- a/continuedev/src/continuedev/libs/constants/default_config.py
+++ b/continuedev/src/continuedev/libs/constants/default_config.py
@@ -132,7 +132,11 @@ config = ContinueConfig(
         # )
         SearchContextProvider(),
         DiffContextProvider(),
-        URLContextProvider()
+        URLContextProvider(
+            preset_urls = [
+                # Add any common urls you reference here so they appear in autocomplete
+            ]
+        )
     ],
 
     # Policies hold the main logic that decides which Step to take next
diff --git a/continuedev/src/continuedev/plugins/context_providers/url.py b/continuedev/src/continuedev/plugins/context_providers/url.py
index 32c1d69c..5b67608d 100644
--- a/continuedev/src/continuedev/plugins/context_providers/url.py
+++ b/continuedev/src/continuedev/plugins/context_providers/url.py
@@ -1,4 +1,7 @@
-from typing import List, Optional
+
+from typing import List
+from bs4 import BeautifulSoup
+import requests
 
 from .util import remove_meilisearch_disallowed_chars
 from ...core.main import ContextItem, ContextItemDescription, ContextItemId
@@ -8,69 +11,80 @@ from ...core.context import ContextProvider
 class URLContextProvider(ContextProvider):
     title = "url"
 
-    url: Optional[str] = None
-    display_name: Optional[str] = None
-    URL_CONTEXT_ITEM_ID = "url"
-
-    @property
-    def optional_url_item_id(self) -> str:
-        return remove_meilisearch_disallowed_chars(self.url)
+    # Allows users to provide a list of preset urls
+    preset_urls: List[str] = []
 
-    @property
-    def optional_url_item(self) -> ContextItem:
-        cp = self.BASE_CONTEXT_ITEM.copy()
-        if self.display_name:
-            cp.description.name = self.display_name
-        cp.description.description = f"Contents of {self.url}"
-        cp.description.id.item_id = self.optional_url_item_id
+    # Static items loaded from preset_urls
+    static_url_context_items: List[ContextItem] = []
 
-        return cp
+    # There is only a single dynamic url context item, so it has a static id
+    DYNAMIC_URL_CONTEXT_ITEM_ID = "url"
 
+    # This is a template dynamic item that will generate context item on demand
+    # when get item is called
     @property
-    def BASE_CONTEXT_ITEM(self):
+    def DYNAMIC_CONTEXT_ITEM(self):
         return ContextItem(
             content="",
             description=ContextItemDescription(
-                name="URL",
+                name="Dynamic URL",
                 description="Reference the contents of a webpage (e.g. '@url https://www.w3schools.com/python/python_ref_functions.asp')",
                 id=ContextItemId(
                     provider_title=self.title,
-                    item_id=self.URL_CONTEXT_ITEM_ID
+                    item_id=self.DYNAMIC_URL_CONTEXT_ITEM_ID
                 )
             )
         )
 
-    def _get_url_text_contents(self, url: str):
-        from bs4 import BeautifulSoup
-        import requests
+    def static_url_context_item_from_url(self, url: str) -> ContextItem:
+        content, title = self._get_url_text_contents_and_title(url)
+        return ContextItem(
+            content=content,
+            description=ContextItemDescription(
+                name=title,
+                description=f"Contents of {url}",
+                id=ContextItemId(
+                    provider_title=self.title,
+                    item_id=remove_meilisearch_disallowed_chars(url)
+                )
+            )
+        )
 
+    def _get_url_text_contents_and_title(self, url: str) -> (str, str):
         response = requests.get(url)
         soup = BeautifulSoup(response.text, 'html.parser')
-        return soup.get_text()
+        title = url.replace(
+            "https://", "").replace("http://", "").replace("www.", "")
+        if soup.title is not None:
+            title = soup.title.string
+        return soup.get_text(), title
 
     async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]:
-        items = [self.BASE_CONTEXT_ITEM]
-        if self.url:
-            items.append(self.optional_url_item)
+        self.static_url_context_items = [
+            self.static_url_context_item_from_url(url) for url in self.preset_urls]
 
-        return items
+        return [self.DYNAMIC_CONTEXT_ITEM] + self.static_url_context_items
 
     async def get_item(self, id: ContextItemId, query: str) -> ContextItem:
-        if id.item_id == self.optional_url_item_id:
-            item = self.optional_url_item
-            item.content = self._get_url_text_contents(self.url)
-            return item
-
-        if not id.item_id == self.URL_CONTEXT_ITEM_ID:
+        # Check if the item is a static item
+        matching_static_item = next(
+            (item for item in self.static_url_context_items if item.description.id.item_id == id.item_id), None)
+        if matching_static_item:
+            return matching_static_item
+
+        # Check if the item is the dynamic item
+        if not id.item_id == self.DYNAMIC_URL_CONTEXT_ITEM_ID:
             raise Exception("Invalid item id")
 
+        # Generate the dynamic item
         url = query.lstrip("url ").strip()
-        content = self._get_url_text_contents(url)
+        if url is None or url == "":
+            return None
+        content, title = self._get_url_text_contents_and_title(url)
 
-        ctx_item = self.BASE_CONTEXT_ITEM.copy()
+        ctx_item = self.DYNAMIC_CONTEXT_ITEM.copy()
         ctx_item.content = content
-        ctx_item.description.name = url.replace(
-            "https://", "").replace("http://", "")
+        ctx_item.description.name = title
         ctx_item.description.id.item_id = remove_meilisearch_disallowed_chars(
             url)
         return ctx_item