diff options
Diffstat (limited to 'continuedev/src')
| -rw-r--r-- | continuedev/src/continuedev/core/autopilot.py | 8 | ||||
| -rw-r--r-- | continuedev/src/continuedev/core/config.py | 3 | ||||
| -rw-r--r-- | continuedev/src/continuedev/core/context.py | 9 | ||||
| -rw-r--r-- | continuedev/src/continuedev/core/sdk.py | 2 | ||||
| -rw-r--r-- | continuedev/src/continuedev/libs/llm/__init__.py | 14 | ||||
| -rw-r--r-- | continuedev/src/continuedev/libs/util/devdata.py | 67 | ||||
| -rw-r--r-- | continuedev/src/continuedev/libs/util/paths.py | 15 | ||||
| -rw-r--r-- | continuedev/src/continuedev/plugins/steps/chat.py | 8 | ||||
| -rw-r--r-- | continuedev/src/continuedev/plugins/steps/core/core.py | 7 | ||||
| -rw-r--r-- | continuedev/src/continuedev/server/ide.py | 3 |
10 files changed, 135 insertions, 1 deletions
diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index 8ac7241d..cbf9ad59 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -11,6 +11,7 @@ from openai import error as openai_errors from pydantic import root_validator from ..libs.util.create_async_task import create_async_task +from ..libs.util.devdata import dev_data_logger from ..libs.util.edit_config import edit_config_property from ..libs.util.logging import logger from ..libs.util.paths import getSavedContextGroupsPath @@ -356,6 +357,9 @@ class Autopilot(ContinueBaseModel): posthog_logger.capture_event( "step run", {"step_name": step.name, "params": step.dict()} ) + dev_data_logger.capture( + "step_run", {"step_name": step.name, "params": step.dict()} + ) if not is_future_step: # Check manual edits buffer, clear out if needed by creating a ManualEditStep @@ -532,6 +536,7 @@ class Autopilot(ContinueBaseModel): date_created=str(time.time()), workspace_directory=self.ide.workspace_directory, ) + dev_data_logger.capture("new_session", self.session_info.dict()) create_async_task( create_title(), @@ -602,6 +607,9 @@ class Autopilot(ContinueBaseModel): posthog_logger.capture_event( "select_context_group", {"title": id, "length": len(context_group)} ) + dev_data_logger.capture( + "select_context_group", {"title": id, "items": context_group} + ) async def delete_context_group(self, id: str): if id not in self._saved_context_groups: diff --git a/continuedev/src/continuedev/core/config.py b/continuedev/src/continuedev/core/config.py index 68b2b17d..b513e22a 100644 --- a/continuedev/src/continuedev/core/config.py +++ b/continuedev/src/continuedev/core/config.py @@ -53,8 +53,9 @@ class ContinueConfig(BaseModel): on_traceback: Optional[Step] = None system_message: Optional[str] = None policy_override: Optional[Policy] = None - context_providers: List[ContextProvider] = [] + user_token: Optional[str] = None + data_server_url: Optional[str] = "https://us-west1-autodebug.cloudfunctions.net" @validator("temperature", pre=True) def temperature_validator(cls, v): diff --git a/continuedev/src/continuedev/core/context.py b/continuedev/src/continuedev/core/context.py index bb2c43dc..47763d8b 100644 --- a/continuedev/src/continuedev/core/context.py +++ b/continuedev/src/continuedev/core/context.py @@ -7,6 +7,7 @@ from meilisearch_python_async import Client from pydantic import BaseModel from ..libs.util.create_async_task import create_async_task +from ..libs.util.devdata import dev_data_logger from ..libs.util.logging import logger from ..libs.util.telemetry import posthog_logger from ..server.meilisearch_server import poll_meilisearch_running @@ -327,6 +328,14 @@ class ContextManager: "query": query, }, ) + dev_data_logger.capture( + "select_context_item", + { + "provider_title": id.provider_title, + "item_id": id.item_id, + "query": query, + }, + ) await self.context_providers[id.provider_title].add_context_item(id, query) async def delete_context_with_ids(self, ids: List[str]): diff --git a/continuedev/src/continuedev/core/sdk.py b/continuedev/src/continuedev/core/sdk.py index b25d4566..ddcf6e55 100644 --- a/continuedev/src/continuedev/core/sdk.py +++ b/continuedev/src/continuedev/core/sdk.py @@ -3,6 +3,7 @@ import traceback from typing import Coroutine, List, Optional, Union from ..libs.llm import LLM +from ..libs.util.devdata import dev_data_logger from ..libs.util.logging import logger from ..libs.util.paths import getConfigFilePath from ..libs.util.telemetry import posthog_logger @@ -109,6 +110,7 @@ class ContinueSDK(AbstractContinueSDK): # When the config is loaded, setup posthog logger posthog_logger.setup(sdk.ide.unique_id, sdk.config.allow_anonymous_telemetry) + dev_data_logger.setup(sdk.config.user_token, sdk.config.data_server_url) return sdk diff --git a/continuedev/src/continuedev/libs/llm/__init__.py b/continuedev/src/continuedev/libs/llm/__init__.py index dcb0d2ef..85ca1969 100644 --- a/continuedev/src/continuedev/libs/llm/__init__.py +++ b/continuedev/src/continuedev/libs/llm/__init__.py @@ -3,6 +3,7 @@ from typing import Any, Callable, Coroutine, Dict, Generator, List, Optional, Un from pydantic import validator from ...core.main import ChatMessage +from ...libs.util.devdata import dev_data_logger from ...models.main import ContinueBaseModel from ..util.count_tokens import ( DEFAULT_ARGS, @@ -172,6 +173,10 @@ class LLM(ContinueBaseModel): completion += chunk self.write_log(f"Completion: \n\n{completion}") + dev_data_logger.capture( + "tokens_generated", + {"model": self.model, "tokens": self.count_tokens(completion)}, + ) async def complete( self, @@ -209,6 +214,11 @@ class LLM(ContinueBaseModel): completion = await self._complete(prompt=prompt, options=options) self.write_log(f"Completion: \n\n{completion}") + dev_data_logger.capture( + "tokens_generated", + {"model": self.model, "tokens": self.count_tokens(completion)}, + ) + return completion async def stream_chat( @@ -261,6 +271,10 @@ class LLM(ContinueBaseModel): completion += chunk self.write_log(f"Completion: \n\n{completion}") + dev_data_logger.capture( + "tokens_generated", + {"model": self.model, "tokens": self.count_tokens(completion)}, + ) def _stream_complete( self, prompt, options: CompletionOptions diff --git a/continuedev/src/continuedev/libs/util/devdata.py b/continuedev/src/continuedev/libs/util/devdata.py new file mode 100644 index 00000000..61b4351d --- /dev/null +++ b/continuedev/src/continuedev/libs/util/devdata.py @@ -0,0 +1,67 @@ +""" +This file contains mechanisms for logging development data to files, SQL databases, and other formats. +""" + + +import json +from datetime import datetime +from typing import Any, Dict + +import aiohttp + +from .create_async_task import create_async_task +from .logging import logger +from .paths import getDevDataFilePath + + +class DevDataLogger: + user_token: str = None + data_server_url: str = None + + def setup(self, user_token: str = None, data_server_url: str = None): + self.user_token = user_token + self.data_server_url = data_server_url + + def _to_data_server(self, table_name: str, data: Dict[str, Any]): + async def _async_helper(self, table_name: str, data: Dict[str, Any]): + if self.user_token is None or self.data_server_url is None: + return + + async with aiohttp.ClientSession() as session: + await session.post( + f"{self.data_server_url}/event", + headers={"Authorization": f"Bearer {self.user_token}"}, + json={ + "table_name": table_name, + "data": data, + "user_token": self.user_token, + }, + ) + + create_async_task( + _async_helper(self, table_name, data), + lambda e: logger.warning(f"Failed to send dev data: {e}"), + ) + + def _static_columns(self): + return { + "user_token": self.user_token or "NO_USER_TOKEN", + "timestamp": datetime.now().isoformat(), + } + + def _to_local(self, table_name: str, data: Dict[str, Any]): + filepath = getDevDataFilePath(table_name) + with open(filepath, "a") as f: + json_line = json.dumps(data) + f.write(f"{json_line}\n") + + def capture(self, table_name: str, data: Dict[str, Any]): + try: + data = {**self._static_columns(), **data} + self._to_data_server(table_name, data) + self._to_local(table_name, data) + except Exception as e: + logger.warning(f"Failed to capture dev data: {e}") + + +dev_data_logger = DevDataLogger() diff --git a/continuedev/src/continuedev/libs/util/paths.py b/continuedev/src/continuedev/libs/util/paths.py index 9f3117d0..216386c3 100644 --- a/continuedev/src/continuedev/libs/util/paths.py +++ b/continuedev/src/continuedev/libs/util/paths.py @@ -31,6 +31,21 @@ def getServerFolderPath(): return path +def getDevDataFolderPath(): + path = os.path.join(getGlobalFolderPath(), "dev_data") + os.makedirs(path, exist_ok=True) + return path + + +def getDevDataFilePath(table_name: str): + filepath = os.path.join(getDevDataFolderPath(), f"{table_name}.jsonl") + if not os.path.exists(filepath): + with open(filepath, "w") as f: + f.write("") + + return filepath + + def getMeilisearchExePath(): binary_name = "meilisearch.exe" if os.name == "nt" else "meilisearch" path = os.path.join(getServerFolderPath(), binary_name) diff --git a/continuedev/src/continuedev/plugins/steps/chat.py b/continuedev/src/continuedev/plugins/steps/chat.py index 15740057..05931bf1 100644 --- a/continuedev/src/continuedev/plugins/steps/chat.py +++ b/continuedev/src/continuedev/plugins/steps/chat.py @@ -13,6 +13,7 @@ from ...core.main import ChatMessage, FunctionCall, Models, Step, step_to_json_s from ...core.sdk import ContinueSDK from ...libs.llm.maybe_proxy_openai import MaybeProxyOpenAI from ...libs.llm.openai import OpenAI +from ...libs.util.devdata import dev_data_logger from ...libs.util.strings import remove_quotes_and_escapes from ...libs.util.telemetry import posthog_logger from .core.core import MessageStep @@ -94,6 +95,13 @@ class SimpleChatStep(Step): "provider": sdk.models.default.__class__.__name__, }, ) + dev_data_logger.capture( + "model_use", + { + "model": sdk.models.default.model, + "provider": sdk.models.default.__class__.__name__, + }, + ) async for chunk in generator: if sdk.current_step_was_deleted(): diff --git a/continuedev/src/continuedev/plugins/steps/core/core.py b/continuedev/src/continuedev/plugins/steps/core/core.py index ba67c932..7766a887 100644 --- a/continuedev/src/continuedev/plugins/steps/core/core.py +++ b/continuedev/src/continuedev/plugins/steps/core/core.py @@ -12,6 +12,7 @@ from ....core.observation import Observation, TextObservation, UserInputObservat from ....libs.llm import LLM from ....libs.llm.maybe_proxy_openai import MaybeProxyOpenAI from ....libs.util.count_tokens import DEFAULT_MAX_TOKENS +from ....libs.util.devdata import dev_data_logger from ....libs.util.strings import ( dedent_and_get_common_whitespace, remove_quotes_and_escapes, @@ -644,6 +645,10 @@ Please output the code to be inserted at the cursor in order to fulfill the user "model_use", {"model": model_to_use.model, "provider": model_to_use.__class__.__name__}, ) + dev_data_logger.capture( + "model_use", + {"model": model_to_use.model, "provider": model_to_use.__class__.__name__}, + ) try: async for chunk in generator: @@ -814,6 +819,8 @@ Please output the code to be inserted at the cursor in order to fulfill the user await self.stream_rif(rif, sdk) await sdk.ide.setSuggestionsLocked(rif.filepath, False) + self.name = "Generating summary" + class EditFileStep(Step): filepath: str diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index 1c88d2a0..fba455c4 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -13,6 +13,7 @@ from starlette.websockets import WebSocketDisconnect, WebSocketState from uvicorn.main import Server from ..libs.util.create_async_task import create_async_task +from ..libs.util.devdata import dev_data_logger from ..libs.util.logging import logger from ..libs.util.queue import AsyncSubscriptionQueue from ..libs.util.telemetry import posthog_logger @@ -346,9 +347,11 @@ class IdeProtocolServer(AbstractIdeProtocolServer): def onAcceptRejectSuggestion(self, accepted: bool): posthog_logger.capture_event("accept_reject_suggestion", {"accepted": accepted}) + dev_data_logger.capture("accept_reject_suggestion", {"accepted": accepted}) def onAcceptRejectDiff(self, accepted: bool): posthog_logger.capture_event("accept_reject_diff", {"accepted": accepted}) + dev_data_logger.capture("accept_reject_diff", {"accepted": accepted}) def onFileSystemUpdate(self, update: FileSystemEdit): # Access to Autopilot (so SessionManager) |