From dac960348a938552de4a6fcfaff32e517e6ebcb1 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Sun, 23 Jul 2023 20:12:44 -0700
Subject: file context provider

---
 continuedev/src/continuedev/core/autopilot.py      |  6 ++-
 .../context_providers/file_context_provider.py     | 54 ++++++++++++++++++++++
 .../src/continuedev/libs/llm/proxy_server.py       |  3 +-
 3 files changed, 59 insertions(+), 4 deletions(-)
 create mode 100644 continuedev/src/continuedev/libs/context_providers/file_context_provider.py

(limited to 'continuedev/src')

diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py
index c0f95414..4dff2c6d 100644
--- a/continuedev/src/continuedev/core/autopilot.py
+++ b/continuedev/src/continuedev/core/autopilot.py
@@ -9,7 +9,8 @@ from pydantic import root_validator
 from ..models.filesystem import RangeInFileWithContents
 from ..models.filesystem_edit import FileEditWithFullContents
 from .observation import Observation, InternalErrorObservation
-from .context import ContextItem, ContextItemDescription, ContextItemId, ContextManager
+from .context import ContextManager
+from ..libs.context_providers.file_context_provider import FileContextProvider
 from ..libs.context_providers.highlighted_code_context_provider import HighlightedCodeContextProvider
 from ..server.ide_protocol import AbstractIdeProtocolServer
 from ..libs.util.queue import AsyncSubscriptionQueue
@@ -69,7 +70,8 @@ class Autopilot(ContinueBaseModel):
         # Load documents into the search index
         autopilot.context_manager = ContextManager(
             autopilot.continue_sdk.config.context_providers + [
-                HighlightedCodeContextProvider(ide=ide)
+                HighlightedCodeContextProvider(ide=ide),
+                FileContextProvider(workspace_dir=ide.workspace_directory)
             ])
         await autopilot.context_manager.load_index()
 
diff --git a/continuedev/src/continuedev/libs/context_providers/file_context_provider.py b/continuedev/src/continuedev/libs/context_providers/file_context_provider.py
new file mode 100644
index 00000000..c3ec351f
--- /dev/null
+++ b/continuedev/src/continuedev/libs/context_providers/file_context_provider.py
@@ -0,0 +1,54 @@
+import os
+from typing import List
+from ...core.main import ContextItem, ContextItemDescription, ContextItemId
+from ...core.context import ContextProvider
+from fnmatch import fnmatch
+
+
+def get_file_contents(filepath: str) -> str:
+    with open(filepath, "r") as f:
+        return f.read()
+
+
+class FileContextProvider(ContextProvider):
+    """
+    The FileContextProvider is a ContextProvider that allows you to search files in the open workspace.
+    """
+
+    title = "file"
+    workspace_dir: str
+    ignore_patterns: List[str] = list(map(lambda folder: f"**/{folder}", [
+        ".git",
+        ".vscode",
+        ".idea",
+        ".vs",
+        ".venv",
+        "env",
+        ".env",
+        "node_modules",
+        "dist",
+        "build",
+        "target",
+        "out",
+        "bin",
+    ]))
+
+    async def provide_context_items(self) -> List[ContextItem]:
+        filepaths = []
+        for root, dir_names, file_names in os.walk(self.workspace_dir):
+            dir_names[:] = [d for d in dir_names if not any(
+                fnmatch(d, pattern) for pattern in self.ignore_patterns)]
+            for file_name in file_names:
+                filepaths.append(os.path.join(root, file_name))
+
+        return [ContextItem(
+            content=get_file_contents(file),
+            description=ContextItemDescription(
+                name=f"File {os.path.basename(file)}",
+                description=file,
+                id=ContextItemId(
+                    provider_title=self.title,
+                    item_id=file
+                )
+            )
+        ) for file in filepaths]
diff --git a/continuedev/src/continuedev/libs/llm/proxy_server.py b/continuedev/src/continuedev/libs/llm/proxy_server.py
index bd50fe02..46ab19ee 100644
--- a/continuedev/src/continuedev/libs/llm/proxy_server.py
+++ b/continuedev/src/continuedev/libs/llm/proxy_server.py
@@ -1,5 +1,4 @@
 
-from functools import cached_property
 import json
 import traceback
 from typing import Any, Callable, Coroutine, Dict, Generator, List, Literal, Union
@@ -7,7 +6,7 @@ import aiohttp
 from ..util.telemetry import capture_event
 from ...core.main import ChatMessage
 from ..llm import LLM
-from ..util.count_tokens import DEFAULT_ARGS, DEFAULT_MAX_TOKENS, compile_chat_messages, CHAT_MODELS, count_tokens, format_chat_messages
+from ..util.count_tokens import DEFAULT_ARGS, compile_chat_messages, count_tokens, format_chat_messages
 import certifi
 import ssl
 
-- 
cgit v1.2.3-70-g09d2