refactor: refactor codebase to unify server module and update file paths (#142)

* Refactor project into a dedicated 'server' module and update all references accordingly

---------

Co-authored-by: Romain Courtois <romain@coderamp.io>

CONTRIBUTING.md

@@ -48,18 +48,18 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be
    pytest
    ```
 
-8. Run the app locally using Docker to test your changes (optional):
+8. Navigate to src folder
 
    1. Build the Docker image
 
         ``` bash
-        docker build -t gitingest .
+        cd src
         ```
 
-   2. Run the Docker container:
+   2. Run the local web server:
 
       ``` bash
-      docker run -d --name gitingest -p 8000:8000 gitingest
+      uvicorn server.main:app
       ```
 
    3. Open your browser and navigate to `http://localhost:8000` to see the app running.

Dockerfile

@@ -41,4 +41,4 @@ USER appuser
 
 EXPOSE 8000
 
-CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]

src/gitingest/cli.py

@@ -6,7 +6,7 @@ import asyncio
 
 import click
 
-from config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
+from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
 from gitingest.repository_ingest import ingest
 
 

src/gitingest/config.py

@@ -0,0 +1,11 @@
+""" Configuration file for the project. """
+
+from pathlib import Path
+
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB
+MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
+MAX_FILES = 10_000  # Maximum number of files to process
+MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # 500 MB
+
+OUTPUT_FILE_PATH = "digest.txt"
+TMP_BASE_PATH = Path("/tmp/gitingest")

src/gitingest/query_ingestion.py

@@ -6,7 +6,7 @@ from typing import Any
 
 import tiktoken
 
-from config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
+from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
 from gitingest.exceptions import (
     AlreadyVisitedError,
     InvalidNotebookError,

src/gitingest/query_parser.py

@@ -9,7 +9,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from urllib.parse import unquote, urlparse
 
-from config import MAX_FILE_SIZE, TMP_BASE_PATH
+from gitingest.config import MAX_FILE_SIZE, TMP_BASE_PATH
 from gitingest.exceptions import InvalidPatternError
 from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
 from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list
@@ -163,7 +163,7 @@ async def _parse_repo_source(source: str) -> ParsedQuery:
 
     _id = str(uuid.uuid4())
     slug = f"{user_name}-{repo_name}"
-    local_path = Path(TMP_BASE_PATH) / _id / slug
+    local_path = TMP_BASE_PATH / _id / slug
     url = f"https://{host}/{user_name}/{repo_name}"
 
     parsed = ParsedQuery(

src/gitingest/repository_ingest.py

@@ -4,7 +4,7 @@ import asyncio
 import inspect
 import shutil
 
-from config import TMP_BASE_PATH
+from gitingest.config import TMP_BASE_PATH
 from gitingest.query_ingestion import run_ingest_query
 from gitingest.query_parser import ParsedQuery, parse_query
 from gitingest.repository_clone import CloneConfig, clone_repo

src/routers/__init__.py

@@ -1,7 +0,0 @@
-""" This module contains the routers for the FastAPI application. """
-
-from routers.download import router as download
-from routers.dynamic import router as dynamic
-from routers.index import router as index
-
-__all__ = ["download", "dynamic", "index"]

src/server/main.py

@@ -10,10 +10,9 @@ from fastapi.staticfiles import StaticFiles
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
-from config import templates
-from routers import download, dynamic, index
-from server_utils import limiter
-from utils import lifespan, rate_limit_exception_handler
+from server.routers import download, dynamic, index
+from server.server_config import templates
+from server.server_utils import lifespan, limiter, rate_limit_exception_handler
 
 # Load environment variables from .env file
 load_dotenv()

src/server/query_processor.py

@@ -5,11 +5,11 @@ from functools import partial
 from fastapi import Request
 from starlette.templating import _TemplateResponse
 
-from config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates
 from gitingest.query_ingestion import run_ingest_query
 from gitingest.query_parser import ParsedQuery, parse_query
 from gitingest.repository_clone import CloneConfig, clone_repo
-from server_utils import Colors, log_slider_to_size
+from server.server_config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates
+from server.server_utils import Colors, log_slider_to_size
 
 
 async def process_query(

src/server/routers/__init__.py

@@ -0,0 +1,7 @@
+""" This module contains the routers for the FastAPI application. """
+
+from server.routers.download import router as download
+from server.routers.dynamic import router as dynamic
+from server.routers.index import router as index
+
+__all__ = ["download", "dynamic", "index"]

src/server/routers/download.py

@@ -3,7 +3,7 @@
 from fastapi import APIRouter, HTTPException
 from fastapi.responses import Response
 
-from config import TMP_BASE_PATH
+from gitingest.config import TMP_BASE_PATH
 
 router = APIRouter()
 

src/server/routers/dynamic.py

@@ -3,9 +3,9 @@
 from fastapi import APIRouter, Form, Request
 from fastapi.responses import HTMLResponse
 
-from config import templates
-from query_processor import process_query
-from server_utils import limiter
+from server.query_processor import process_query
+from server.server_config import templates
+from server.server_utils import limiter
 
 router = APIRouter()
 

src/server/routers/index.py

@@ -3,9 +3,9 @@
 from fastapi import APIRouter, Form, Request
 from fastapi.responses import HTMLResponse
 
-from config import EXAMPLE_REPOS, templates
-from query_processor import process_query
-from server_utils import limiter
+from server.query_processor import process_query
+from server.server_config import EXAMPLE_REPOS, templates
+from server.server_utils import limiter
 
 router = APIRouter()
 

src/server/server_config.py

@@ -1,19 +1,10 @@
-""" Configuration file for the project. """
-
-from pathlib import Path
+""" Configuration for the server. """
 
 from fastapi.templating import Jinja2Templates
 
-MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB
-MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
-MAX_FILES = 10_000  # Maximum number of files to process
-MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # 500 MB
-
 MAX_DISPLAY_SIZE: int = 300_000
 DELETE_REPO_AFTER: int = 60 * 60  # In seconds
 
-OUTPUT_FILE_PATH = "digest.txt"
-TMP_BASE_PATH = Path("/tmp/gitingest")
 
 EXAMPLE_REPOS: list[dict[str, str]] = [
     {"name": "Gitingest", "url": "https://github.com/cyclotruc/gitingest"},
@@ -23,4 +14,4 @@ EXAMPLE_REPOS: list[dict[str, str]] = [
     {"name": "ApiAnalytics", "url": "https://github.com/tom-draper/api-analytics"},
 ]
 
-templates = Jinja2Templates(directory="templates")
+templates = Jinja2Templates(directory="server/templates")

src/server/server_utils.py

@@ -1,6 +1,7 @@
-""" Utility functions for the FastAPI server. """
+""" Utility functions for the server. """
 
 import asyncio
+import math
 import shutil
 import time
 from contextlib import asynccontextmanager
@@ -8,10 +9,15 @@ from pathlib import Path
 
 from fastapi import FastAPI, Request
 from fastapi.responses import Response
-from slowapi import _rate_limit_exceeded_handler
+from slowapi import Limiter, _rate_limit_exceeded_handler
 from slowapi.errors import RateLimitExceeded
+from slowapi.util import get_remote_address
 
-from config import DELETE_REPO_AFTER, TMP_BASE_PATH
+from gitingest.config import TMP_BASE_PATH
+from server.server_config import DELETE_REPO_AFTER
+
+# Initialize a rate limiter
+limiter = Limiter(key_func=get_remote_address)
 
 
 async def rate_limit_exception_handler(request: Request, exc: Exception) -> Response:
@@ -136,3 +142,53 @@ async def _process_folder(folder: Path) -> None:
         shutil.rmtree(folder)
     except Exception as e:
         print(f"Error deleting {folder}: {e}")
+
+
+def log_slider_to_size(position: int) -> int:
+    """
+    Convert a slider position to a file size in bytes using a logarithmic scale.
+
+    Parameters
+    ----------
+    position : int
+        Slider position ranging from 0 to 500.
+
+    Returns
+    -------
+    int
+        File size in bytes corresponding to the slider position.
+    """
+    maxp = 500
+    minv = math.log(1)
+    maxv = math.log(102_400)
+    return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5))) * 1024
+
+
+## Color printing utility
+class Colors:
+    """ANSI color codes"""
+
+    BLACK = "\033[0;30m"
+    RED = "\033[0;31m"
+    GREEN = "\033[0;32m"
+    BROWN = "\033[0;33m"
+    BLUE = "\033[0;34m"
+    PURPLE = "\033[0;35m"
+    CYAN = "\033[0;36m"
+    LIGHT_GRAY = "\033[0;37m"
+    DARK_GRAY = "\033[1;30m"
+    LIGHT_RED = "\033[1;31m"
+    LIGHT_GREEN = "\033[1;32m"
+    YELLOW = "\033[1;33m"
+    LIGHT_BLUE = "\033[1;34m"
+    LIGHT_PURPLE = "\033[1;35m"
+    LIGHT_CYAN = "\033[1;36m"
+    WHITE = "\033[1;37m"
+    BOLD = "\033[1m"
+    FAINT = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    NEGATIVE = "\033[7m"
+    CROSSED = "\033[9m"
+    END = "\033[0m"

src/server_utils.py

@@ -1,59 +0,0 @@
-""" Utility functions for the server. """
-
-import math
-
-from slowapi import Limiter
-from slowapi.util import get_remote_address
-
-# Initialize a rate limiter
-limiter = Limiter(key_func=get_remote_address)
-
-
-def log_slider_to_size(position: int) -> int:
-    """
-    Convert a slider position to a file size in bytes using a logarithmic scale.
-
-    Parameters
-    ----------
-    position : int
-        Slider position ranging from 0 to 500.
-
-    Returns
-    -------
-    int
-        File size in bytes corresponding to the slider position.
-    """
-    maxp = 500
-    minv = math.log(1)
-    maxv = math.log(102_400)
-    return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5))) * 1024
-
-
-## Color printing utility
-class Colors:
-    """ANSI color codes"""
-
-    BLACK = "\033[0;30m"
-    RED = "\033[0;31m"
-    GREEN = "\033[0;32m"
-    BROWN = "\033[0;33m"
-    BLUE = "\033[0;34m"
-    PURPLE = "\033[0;35m"
-    CYAN = "\033[0;36m"
-    LIGHT_GRAY = "\033[0;37m"
-    DARK_GRAY = "\033[1;30m"
-    LIGHT_RED = "\033[1;31m"
-    LIGHT_GREEN = "\033[1;32m"
-    YELLOW = "\033[1;33m"
-    LIGHT_BLUE = "\033[1;34m"
-    LIGHT_PURPLE = "\033[1;35m"
-    LIGHT_CYAN = "\033[1;36m"
-    WHITE = "\033[1;37m"
-    BOLD = "\033[1m"
-    FAINT = "\033[2m"
-    ITALIC = "\033[3m"
-    UNDERLINE = "\033[4m"
-    BLINK = "\033[5m"
-    NEGATIVE = "\033[7m"
-    CROSSED = "\033[9m"
-    END = "\033[0m"

tests/test_cli.py

@@ -4,8 +4,8 @@ import os
 
 from click.testing import CliRunner
 
-from config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
 from gitingest.cli import main
+from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
 
 
 def test_cli_with_default_options():