From f09150617ed2454f3074bcf93f53aae5ae637d40 Mon Sep 17 00:00:00 2001 From: Nate Sesti <33237525+sestinj@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:37:27 -0700 Subject: Preview (#541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Strong typing (#533) * refactor: :recycle: get rid of continuedev.src.continuedev structure * refactor: :recycle: switching back to server folder * feat: :sparkles: make config.py imports shorter * feat: :bookmark: publish as pre-release vscode extension * refactor: :recycle: refactor and add more completion params to ui * build: :building_construction: download from preview S3 * fix: :bug: fix paths * fix: :green_heart: package:pre-release * ci: :green_heart: more time for tests * fix: :green_heart: fix build scripts * fix: :bug: fix import in run.py * fix: :bookmark: update version to try again * ci: 💚 Update package.json version [skip ci] * refactor: :fire: don't check for old extensions version * fix: :bug: small bug fixes * fix: :bug: fix config.py import paths * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: platform-specific builds test #1 * feat: :green_heart: ship with binary * fix: :green_heart: fix copy statement to include.exe for windows * fix: :green_heart: cd extension before packaging * chore: :loud_sound: count tokens generated * fix: :green_heart: remove npm_config_arch * fix: :green_heart: publish as pre-release! * chore: :bookmark: update version * perf: :green_heart: hardcode distro paths * fix: :bug: fix yaml syntax error * chore: :bookmark: update version * fix: :green_heart: update permissions and version * feat: :bug: kill old server if needed * feat: :lipstick: update marketplace icon for pre-release * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: auto-reload for config.py * feat: :wrench: update default config.py imports * feat: :sparkles: codelens in config.py * feat: :sparkles: select model param count from UI * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: more model options, ollama error handling * perf: :zap: don't show server loading immediately * fix: :bug: fixing small UI details * ci: 💚 Update package.json version [skip ci] * feat: :rocket: headers param on LLM class * fix: :bug: fix headers for openai.;y * feat: :sparkles: highlight code on cmd+shift+L * ci: 💚 Update package.json version [skip ci] * feat: :lipstick: sticky top bar in gui.tsx * fix: :loud_sound: websocket logging and horizontal scrollbar * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: allow AzureOpenAI Service through GGML * ci: 💚 Update package.json version [skip ci] * fix: :bug: fix automigration * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: upload binaries in ci, download apple silicon * chore: :fire: remove notes * fix: :green_heart: use curl to download binary * fix: :green_heart: set permissions on apple silicon binary * fix: :green_heart: testing * fix: :green_heart: cleanup file * fix: :green_heart: fix preview.yaml * fix: :green_heart: only upload once per binary * fix: :green_heart: install rosetta * ci: :green_heart: download binary after tests * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: prepare ci for merge to main --------- Co-authored-by: GitHub Action --- .../continuedev/plugins/context_providers/file.py | 136 +++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 server/continuedev/plugins/context_providers/file.py (limited to 'server/continuedev/plugins/context_providers/file.py') diff --git a/server/continuedev/plugins/context_providers/file.py b/server/continuedev/plugins/context_providers/file.py new file mode 100644 index 00000000..4cfbcfdb --- /dev/null +++ b/server/continuedev/plugins/context_providers/file.py @@ -0,0 +1,136 @@ +import asyncio +import os +from typing import List, Optional + +from ...core.context import ContextProvider +from ...core.main import ContextItem, ContextItemDescription, ContextItemId +from ...core.sdk import ContinueSDK +from ...libs.util.filter_files import DEFAULT_IGNORE_PATTERNS +from ...libs.util.logging import logger +from .util import remove_meilisearch_disallowed_chars + +MAX_SIZE_IN_CHARS = 50_000 + + +async def get_file_contents(filepath: str, sdk: ContinueSDK) -> str: + try: + return (await sdk.ide.readFile(filepath))[:MAX_SIZE_IN_CHARS] + except Exception as _: + return None + + +class FileContextProvider(ContextProvider): + """ + The FileContextProvider is a ContextProvider that allows you to search files in the open workspace. + """ + + title = "file" + ignore_patterns: List[str] = DEFAULT_IGNORE_PATTERNS + + display_title = "Files" + description = "Reference files in the current workspace" + dynamic = False + + async def start(self, *args): + await super().start(*args) + + async def on_file_saved(filepath: str, contents: str): + item = await self.get_context_item_for_filepath(filepath) + if item is None: + return + await self.update_documents([item], self.sdk.ide.workspace_directory) + + async def on_files_created(filepaths: List[str]): + items = await asyncio.gather( + *[ + self.get_context_item_for_filepath(filepath) + for filepath in filepaths + ] + ) + items = [item for item in items if item is not None] + await self.update_documents(items, self.sdk.ide.workspace_directory) + + async def on_files_deleted(filepaths: List[str]): + ids = [self.get_id_for_filepath(filepath) for filepath in filepaths] + + await self.delete_documents(ids) + + async def on_files_renamed(old_filepaths: List[str], new_filepaths: List[str]): + if self.sdk.ide.workspace_directory is None: + return + + old_ids = [self.get_id_for_filepath(filepath) for filepath in old_filepaths] + new_docs = await asyncio.gather( + *[ + self.get_context_item_for_filepath(filepath) + for filepath in new_filepaths + ] + ) + new_docs = [doc for doc in new_docs if doc is not None] + + await self.delete_documents(old_ids) + await self.update_documents(new_docs, self.sdk.ide.workspace_directory) + + self.sdk.ide.subscribeToFileSaved(on_file_saved) + self.sdk.ide.subscribeToFilesCreated(on_files_created) + self.sdk.ide.subscribeToFilesDeleted(on_files_deleted) + self.sdk.ide.subscribeToFilesRenamed(on_files_renamed) + + def get_id_for_filepath(self, absolute_filepath: str) -> str: + return remove_meilisearch_disallowed_chars(absolute_filepath) + + async def get_context_item_for_filepath( + self, absolute_filepath: str + ) -> Optional[ContextItem]: + content = await get_file_contents(absolute_filepath, self.sdk) + if content is None: + return None + + workspace_dir = self.sdk.ide.workspace_directory + if ( + os.path.splitdrive(workspace_dir)[0] + != os.path.splitdrive(absolute_filepath)[0] + ): + workspace_dir = ( + os.path.splitdrive(absolute_filepath)[0] + + os.path.splitdrive(workspace_dir)[1] + ) + + try: + relative_to_workspace = os.path.relpath(absolute_filepath, workspace_dir) + except Exception as e: + logger.warning(f"Error getting relative path: {e}") + return None + + return ContextItem( + content=content[: min(2000, len(content))], + description=ContextItemDescription( + name=os.path.basename(absolute_filepath), + # We should add the full path to the ContextItem + # It warrants a data modeling discussion and has no immediate use case + description=relative_to_workspace, + id=ContextItemId( + provider_title=self.title, + item_id=self.get_id_for_filepath(absolute_filepath), + ), + ), + ) + + async def provide_context_items(self, workspace_dir: str) -> List[ContextItem]: + contents = await self.sdk.ide.listDirectoryContents(workspace_dir, True) + if contents is None: + return [] + + absolute_filepaths: List[str] = [] + for filepath in contents[:1000]: + absolute_filepaths.append(filepath) + + items = await asyncio.gather( + *[ + self.get_context_item_for_filepath(filepath) + for filepath in absolute_filepaths + ] + ) + items = list(filter(lambda item: item is not None, items)) + + return items -- cgit v1.2.3-70-g09d2