Preview (#541)

* Strong typing (#533)

* refactor: :recycle: get rid of continuedev.src.continuedev structure

* refactor: :recycle: switching back to server folder

* feat: :sparkles: make config.py imports shorter

* feat: :bookmark: publish as pre-release vscode extension

* refactor: :recycle: refactor and add more completion params to ui

* build: :building_construction: download from preview S3

* fix: :bug: fix paths

* fix: :green_heart: package:pre-release

* ci: :green_heart: more time for tests

* fix: :green_heart: fix build scripts

* fix: :bug: fix import in run.py

* fix: :bookmark: update version to try again

* ci: 💚 Update package.json version [skip ci]

* refactor: :fire: don't check for old extensions version

* fix: :bug: small bug fixes

* fix: :bug: fix config.py import paths

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: platform-specific builds test #1

* feat: :green_heart: ship with binary

* fix: :green_heart: fix copy statement to include.exe for windows

* fix: :green_heart: cd extension before packaging

* chore: :loud_sound: count tokens generated

* fix: :green_heart: remove npm_config_arch

* fix: :green_heart: publish as pre-release!

* chore: :bookmark: update version

* perf: :green_heart: hardcode distro paths

* fix: :bug: fix yaml syntax error

* chore: :bookmark: update version

* fix: :green_heart: update permissions and version

* feat: :bug: kill old server if needed

* feat: :lipstick: update marketplace icon for pre-release

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: auto-reload for config.py

* feat: :wrench: update default config.py imports

* feat: :sparkles: codelens in config.py

* feat: :sparkles: select model param count from UI

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: more model options, ollama error handling

* perf: :zap: don't show server loading immediately

* fix: :bug: fixing small UI details

* ci: 💚 Update package.json version [skip ci]

* feat: :rocket: headers param on LLM class

* fix: :bug: fix headers for openai.;y

* feat: :sparkles: highlight code on cmd+shift+L

* ci: 💚 Update package.json version [skip ci]

* feat: :lipstick: sticky top bar in gui.tsx

* fix: :loud_sound: websocket logging and horizontal scrollbar

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: allow AzureOpenAI Service through GGML

* ci: 💚 Update package.json version [skip ci]

* fix: :bug: fix automigration

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: upload binaries in ci, download apple silicon

* chore: :fire: remove notes

* fix: :green_heart: use curl to download binary

* fix: :green_heart: set permissions on apple silicon binary

* fix: :green_heart: testing

* fix: :green_heart: cleanup file

* fix: :green_heart: fix preview.yaml

* fix: :green_heart: only upload once per binary

* fix: :green_heart: install rosetta

* ci: :green_heart: download binary after tests

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: prepare ci for merge to main

---------

Co-authored-by: GitHub Action <action@github.com>

1 files changed, 104 insertions, 0 deletions

diff --git a/server/continuedev/plugins/context_providers/url.py b/server/continuedev/plugins/context_providers/url.py
new file mode 100644
index 00000000..1ed7c18e
--- /dev/null
+++ b/server/continuedev/plugins/context_providers/url.py
@@ -0,0 +1,104 @@
+from typing import List
+
+import requests
+from bs4 import BeautifulSoup
+from pydantic import Field
+
+from ...core.context import ContextProvider
+from ...core.main import ContextItem, ContextItemDescription, ContextItemId
+from .util import remove_meilisearch_disallowed_chars
+
+
+class URLContextProvider(ContextProvider):
+    """Type '@url' to reference the contents of a URL. You can either reference preset URLs, or reference one dynamically by typing '@url https://example.com'. The text contents of the page will be fetched and used as context."""
+
+    title = "url"
+    display_title = "URL"
+    description = "Reference the contents of a webpage"
+    dynamic = True
+    requires_query = True
+
+    # Allows users to provide a list of preset urls
+    preset_urls: List[str] = Field(
+        [],
+        description="A list of preset URLs that you will be able to quickly reference by typing '@url'",
+    )
+
+    # Static items loaded from preset_urls
+    static_url_context_items: List[ContextItem] = []
+
+    # There is only a single dynamic url context item, so it has a static id
+    _DYNAMIC_URL_CONTEXT_ITEM_ID = "url"
+
+    # This is a template dynamic item that will generate context item on demand
+    # when get item is called
+    @property
+    def DYNAMIC_CONTEXT_ITEM(self):
+        return ContextItem(
+            content="",
+            description=ContextItemDescription(
+                name="Dynamic URL",
+                description="Reference the contents of a webpage (e.g. '@url https://www.w3schools.com/python/python_ref_functions.asp')",
+                id=ContextItemId(
+                    provider_title=self.title, item_id=self._DYNAMIC_URL_CONTEXT_ITEM_ID
+                ),
+            ),
+        )
+
+    def static_url_context_item_from_url(self, url: str) -> ContextItem:
+        content, title = self._get_url_text_contents_and_title(url)
+        return ContextItem(
+            content=content,
+            description=ContextItemDescription(
+                name=title,
+                description=f"Contents of {url}",
+                id=ContextItemId(
+                    provider_title=self.title,
+                    item_id=remove_meilisearch_disallowed_chars(url),
+                ),
+            ),
+        )
+
+    def _get_url_text_contents_and_title(self, url: str) -> (str, str):
+        response = requests.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+        title = url.replace("https://", "").replace("http://", "").replace("www.", "")
+        if soup.title is not None:
+            title = soup.title.string
+        return soup.get_text(), title
+
+    async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]:
+        self.static_url_context_items = [
+            self.static_url_context_item_from_url(url) for url in self.preset_urls
+        ]
+
+        return [self.DYNAMIC_CONTEXT_ITEM] + self.static_url_context_items
+
+    async def get_item(self, id: ContextItemId, query: str) -> ContextItem:
+        # Check if the item is a static item
+        matching_static_item = next(
+            (
+                item
+                for item in self.static_url_context_items
+                if item.description.id.item_id == id.item_id
+            ),
+            None,
+        )
+        if matching_static_item:
+            return matching_static_item
+
+        # Check if the item is the dynamic item
+        if not id.provider_title == self.title:
+            raise Exception("Invalid provider title for item")
+
+        # Generate the dynamic item
+        url = query.lstrip("url ").strip()
+        if url is None or url == "":
+            return None
+        content, title = self._get_url_text_contents_and_title(url)
+
+        ctx_item = self.DYNAMIC_CONTEXT_ITEM.copy()
+        ctx_item.content = content
+        ctx_item.description.name = title
+        ctx_item.description.id.item_id = remove_meilisearch_disallowed_chars(url)
+        return ctx_item