1 files changed, 66 insertions, 0 deletions
diff --git a/server/continuedev/libs/chroma/update.py b/server/continuedev/libs/chroma/update.py
new file mode 100644
index 00000000..7a1217f9
--- /dev/null
+++ b/server/continuedev/libs/chroma/update.py
@@ -0,0 +1,66 @@
+# import faiss
+import os
+import subprocess
+from typing import List
+
+from dotenv import load_dotenv
+from llama_index import Document, SimpleDirectoryReader
+
+load_dotenv()
+
+FILE_TYPES_TO_IGNORE = [".pyc", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico"]
+
+
+def filter_ignored_files(files: List[str], root_dir: str):
+    """Further filter files before indexing."""
+    for file in files:
+        if (
+            file.endswith(tuple(FILE_TYPES_TO_IGNORE))
+            or file.startswith(".git")
+            or file.startswith("archive")
+        ):
+            continue  # nice
+        yield root_dir + "/" + file
+
+
+def get_git_ignored_files(root_dir: str):
+    """Get the list of ignored files in a Git repository."""
+    try:
+        output = (
+            subprocess.check_output(
+                ["git", "ls-files", "--ignored", "--others", "--exclude-standard"],
+                cwd=root_dir,
+            )
+            .strip()
+            .decode()
+        )
+        return output.split("\n")
+    except subprocess.CalledProcessError:
+        return []
+
+
+def get_all_files(root_dir: str):
+    """Get a list of all files in a directory."""
+    for dir_path, _, file_names in os.walk(root_dir):
+        for file_name in file_names:
+            yield os.path.join(os.path.relpath(dir_path, root_dir), file_name)
+
+
+def get_input_files(root_dir: str):
+    """Get a list of all files in a Git repository that are not ignored."""
+    ignored_files = set(get_git_ignored_files(root_dir))
+    all_files = set(get_all_files(root_dir))
+    nonignored_files = all_files - ignored_files
+    return filter_ignored_files(nonignored_files, root_dir)
+
+
+def load_gpt_index_documents(root: str) -> List[Document]:
+    """Loads a list of GPTIndex Documents, respecting .gitignore files."""
+    # Get input files
+    input_files = get_input_files(root)
+    # Use SimpleDirectoryReader to load the files into Documents
+    return SimpleDirectoryReader(
+        root,
+        input_files=input_files,
+        file_metadata=lambda filename: {"filename": filename},
+    ).load_data()