Preview (#541)

* Strong typing (#533)

* refactor: :recycle: get rid of continuedev.src.continuedev structure

* refactor: :recycle: switching back to server folder

* feat: :sparkles: make config.py imports shorter

* feat: :bookmark: publish as pre-release vscode extension

* refactor: :recycle: refactor and add more completion params to ui

* build: :building_construction: download from preview S3

* fix: :bug: fix paths

* fix: :green_heart: package:pre-release

* ci: :green_heart: more time for tests

* fix: :green_heart: fix build scripts

* fix: :bug: fix import in run.py

* fix: :bookmark: update version to try again

* ci: 💚 Update package.json version [skip ci]

* refactor: :fire: don't check for old extensions version

* fix: :bug: small bug fixes

* fix: :bug: fix config.py import paths

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: platform-specific builds test #1

* feat: :green_heart: ship with binary

* fix: :green_heart: fix copy statement to include.exe for windows

* fix: :green_heart: cd extension before packaging

* chore: :loud_sound: count tokens generated

* fix: :green_heart: remove npm_config_arch

* fix: :green_heart: publish as pre-release!

* chore: :bookmark: update version

* perf: :green_heart: hardcode distro paths

* fix: :bug: fix yaml syntax error

* chore: :bookmark: update version

* fix: :green_heart: update permissions and version

* feat: :bug: kill old server if needed

* feat: :lipstick: update marketplace icon for pre-release

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: auto-reload for config.py

* feat: :wrench: update default config.py imports

* feat: :sparkles: codelens in config.py

* feat: :sparkles: select model param count from UI

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: more model options, ollama error handling

* perf: :zap: don't show server loading immediately

* fix: :bug: fixing small UI details

* ci: 💚 Update package.json version [skip ci]

* feat: :rocket: headers param on LLM class

* fix: :bug: fix headers for openai.;y

* feat: :sparkles: highlight code on cmd+shift+L

* ci: 💚 Update package.json version [skip ci]

* feat: :lipstick: sticky top bar in gui.tsx

* fix: :loud_sound: websocket logging and horizontal scrollbar

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: allow AzureOpenAI Service through GGML

* ci: 💚 Update package.json version [skip ci]

* fix: :bug: fix automigration

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: upload binaries in ci, download apple silicon

* chore: :fire: remove notes

* fix: :green_heart: use curl to download binary

* fix: :green_heart: set permissions on apple silicon binary

* fix: :green_heart: testing

* fix: :green_heart: cleanup file

* fix: :green_heart: fix preview.yaml

* fix: :green_heart: only upload once per binary

* fix: :green_heart: install rosetta

* ci: :green_heart: download binary after tests

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: prepare ci for merge to main

---------

Co-authored-by: GitHub Action <action@github.com>

1 files changed, 136 insertions, 0 deletions

diff --git a/server/continuedev/plugins/context_providers/file.py b/server/continuedev/plugins/context_providers/file.py
new file mode 100644
index 00000000..4cfbcfdb
--- /dev/null
+++ b/server/continuedev/plugins/context_providers/file.py
@@ -0,0 +1,136 @@
+import asyncio
+import os
+from typing import List, Optional
+
+from ...core.context import ContextProvider
+from ...core.main import ContextItem, ContextItemDescription, ContextItemId
+from ...core.sdk import ContinueSDK
+from ...libs.util.filter_files import DEFAULT_IGNORE_PATTERNS
+from ...libs.util.logging import logger
+from .util import remove_meilisearch_disallowed_chars
+
+MAX_SIZE_IN_CHARS = 50_000
+
+
+async def get_file_contents(filepath: str, sdk: ContinueSDK) -> str:
+    try:
+        return (await sdk.ide.readFile(filepath))[:MAX_SIZE_IN_CHARS]
+    except Exception as _:
+        return None
+
+
+class FileContextProvider(ContextProvider):
+    """
+    The FileContextProvider is a ContextProvider that allows you to search files in the open workspace.
+    """
+
+    title = "file"
+    ignore_patterns: List[str] = DEFAULT_IGNORE_PATTERNS
+
+    display_title = "Files"
+    description = "Reference files in the current workspace"
+    dynamic = False
+
+    async def start(self, *args):
+        await super().start(*args)
+
+        async def on_file_saved(filepath: str, contents: str):
+            item = await self.get_context_item_for_filepath(filepath)
+            if item is None:
+                return
+            await self.update_documents([item], self.sdk.ide.workspace_directory)
+
+        async def on_files_created(filepaths: List[str]):
+            items = await asyncio.gather(
+                *[
+                    self.get_context_item_for_filepath(filepath)
+                    for filepath in filepaths
+                ]
+            )
+            items = [item for item in items if item is not None]
+            await self.update_documents(items, self.sdk.ide.workspace_directory)
+
+        async def on_files_deleted(filepaths: List[str]):
+            ids = [self.get_id_for_filepath(filepath) for filepath in filepaths]
+
+            await self.delete_documents(ids)
+
+        async def on_files_renamed(old_filepaths: List[str], new_filepaths: List[str]):
+            if self.sdk.ide.workspace_directory is None:
+                return
+
+            old_ids = [self.get_id_for_filepath(filepath) for filepath in old_filepaths]
+            new_docs = await asyncio.gather(
+                *[
+                    self.get_context_item_for_filepath(filepath)
+                    for filepath in new_filepaths
+                ]
+            )
+            new_docs = [doc for doc in new_docs if doc is not None]
+
+            await self.delete_documents(old_ids)
+            await self.update_documents(new_docs, self.sdk.ide.workspace_directory)
+
+        self.sdk.ide.subscribeToFileSaved(on_file_saved)
+        self.sdk.ide.subscribeToFilesCreated(on_files_created)
+        self.sdk.ide.subscribeToFilesDeleted(on_files_deleted)
+        self.sdk.ide.subscribeToFilesRenamed(on_files_renamed)
+
+    def get_id_for_filepath(self, absolute_filepath: str) -> str:
+        return remove_meilisearch_disallowed_chars(absolute_filepath)
+
+    async def get_context_item_for_filepath(
+        self, absolute_filepath: str
+    ) -> Optional[ContextItem]:
+        content = await get_file_contents(absolute_filepath, self.sdk)
+        if content is None:
+            return None
+
+        workspace_dir = self.sdk.ide.workspace_directory
+        if (
+            os.path.splitdrive(workspace_dir)[0]
+            != os.path.splitdrive(absolute_filepath)[0]
+        ):
+            workspace_dir = (
+                os.path.splitdrive(absolute_filepath)[0]
+                + os.path.splitdrive(workspace_dir)[1]
+            )
+
+        try:
+            relative_to_workspace = os.path.relpath(absolute_filepath, workspace_dir)
+        except Exception as e:
+            logger.warning(f"Error getting relative path: {e}")
+            return None
+
+        return ContextItem(
+            content=content[: min(2000, len(content))],
+            description=ContextItemDescription(
+                name=os.path.basename(absolute_filepath),
+                # We should add the full path to the ContextItem
+                # It warrants a data modeling discussion and has no immediate use case
+                description=relative_to_workspace,
+                id=ContextItemId(
+                    provider_title=self.title,
+                    item_id=self.get_id_for_filepath(absolute_filepath),
+                ),
+            ),
+        )
+
+    async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]:
+        contents = await self.sdk.ide.listDirectoryContents(workspace_dir, True)
+        if contents is None:
+            return []
+
+        absolute_filepaths: List[str] = []
+        for filepath in contents[:1000]:
+            absolute_filepaths.append(filepath)
+
+        items = await asyncio.gather(
+            *[
+                self.get_context_item_for_filepath(filepath)
+                for filepath in absolute_filepaths
+            ]
+        )
+        items = list(filter(lambda item: item is not None, items))
+
+        return items