feat!: migrate to OpenAI native tool calls and fix circular deps (#fuck-gemini)
- Fix circular dependencies in agent/tools - Migrate from custom JSON to OpenAI tool calls format - Add async streaming (step_stream, complete_stream) - Simplify prompt system and remove token counting - Add 5 new API endpoints (/health, /v1/models, /api/memory/*) - Add 3 new tools (get_torrent_by_index, add_torrent_by_index, set_language) - Fix all 500 tests and add coverage config (80% threshold) - Add comprehensive docs (README, pytest guide) BREAKING: LLM interface changed, memory injection via get_memory()
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
"""Filesystem operations."""
|
||||
|
||||
from .exceptions import FilesystemError, PathTraversalError
|
||||
from .file_manager import FileManager
|
||||
from .organizer import MediaOrganizer
|
||||
from .exceptions import FilesystemError, PathTraversalError
|
||||
|
||||
__all__ = [
|
||||
"FileManager",
|
||||
|
||||
@@ -3,19 +3,23 @@
|
||||
|
||||
class FilesystemError(Exception):
|
||||
"""Base exception for filesystem operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class PathTraversalError(FilesystemError):
|
||||
"""Raised when path traversal attack is detected."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class FileNotFoundError(FilesystemError):
|
||||
"""Raised when a file is not found."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class PermissionDeniedError(FilesystemError):
|
||||
"""Raised when permission is denied."""
|
||||
|
||||
pass
|
||||
|
||||
@@ -1,19 +1,22 @@
|
||||
"""File manager - Migrated from agent/tools/filesystem.py with domain logic extracted."""
|
||||
from typing import Dict, Any, List
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
"""File manager for filesystem operations."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .exceptions import FilesystemError, PathTraversalError
|
||||
from infrastructure.persistence.memory import Memory
|
||||
from infrastructure.persistence import get_memory
|
||||
|
||||
from .exceptions import PathTraversalError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FolderName(Enum):
|
||||
"""Types of folders that can be managed."""
|
||||
|
||||
DOWNLOAD = "download"
|
||||
TVSHOW = "tvshow"
|
||||
MOVIE = "movie"
|
||||
@@ -23,137 +26,116 @@ class FolderName(Enum):
|
||||
class FileManager:
|
||||
"""
|
||||
File manager for filesystem operations.
|
||||
|
||||
Handles folder configuration, listing, and file operations with security.
|
||||
|
||||
Handles folder configuration, listing, and file operations
|
||||
with security checks to prevent path traversal attacks.
|
||||
"""
|
||||
|
||||
def __init__(self, memory: Memory):
|
||||
|
||||
def set_folder_path(self, folder_name: str, path_value: str) -> dict[str, Any]:
|
||||
"""
|
||||
Initialize file manager.
|
||||
|
||||
Set a folder path in the configuration.
|
||||
|
||||
Validates that the path exists, is a directory, and is readable.
|
||||
|
||||
Args:
|
||||
memory: Memory instance for folder configuration
|
||||
"""
|
||||
self.memory = memory
|
||||
|
||||
def set_folder_path(self, folder_name: str, path_value: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Set a folder path in the configuration with validation.
|
||||
|
||||
Args:
|
||||
folder_name: Name of folder to set (download, tvshow, movie, torrent)
|
||||
path_value: Absolute path to the folder
|
||||
|
||||
folder_name: Name of folder (download, tvshow, movie, torrent).
|
||||
path_value: Absolute path to the folder.
|
||||
|
||||
Returns:
|
||||
Dict with status or error information
|
||||
Dict with status or error information.
|
||||
"""
|
||||
try:
|
||||
# Validate folder name
|
||||
self._validate_folder_name(folder_name)
|
||||
|
||||
# Convert to Path object for better handling
|
||||
path_obj = Path(path_value).resolve()
|
||||
|
||||
# Validate path exists and is a directory
|
||||
|
||||
if not path_obj.exists():
|
||||
logger.warning(f"Path does not exist: {path_value}")
|
||||
return {
|
||||
"error": "invalid_path",
|
||||
"message": f"Path does not exist: {path_value}"
|
||||
"message": f"Path does not exist: {path_value}",
|
||||
}
|
||||
|
||||
|
||||
if not path_obj.is_dir():
|
||||
logger.warning(f"Path is not a directory: {path_value}")
|
||||
return {
|
||||
"error": "invalid_path",
|
||||
"message": f"Path is not a directory: {path_value}"
|
||||
"message": f"Path is not a directory: {path_value}",
|
||||
}
|
||||
|
||||
# Check if path is readable
|
||||
|
||||
if not os.access(path_obj, os.R_OK):
|
||||
logger.warning(f"Path is not readable: {path_value}")
|
||||
return {
|
||||
"error": "permission_denied",
|
||||
"message": f"Path is not readable: {path_value}"
|
||||
"message": f"Path is not readable: {path_value}",
|
||||
}
|
||||
|
||||
# Store in memory
|
||||
config = self.memory.get("config", {})
|
||||
config[f"{folder_name}_folder"] = str(path_obj)
|
||||
self.memory.set("config", config)
|
||||
|
||||
|
||||
memory = get_memory()
|
||||
memory.ltm.set_config(f"{folder_name}_folder", str(path_obj))
|
||||
memory.save()
|
||||
|
||||
logger.info(f"Set {folder_name}_folder to: {path_obj}")
|
||||
return {
|
||||
"status": "ok",
|
||||
"folder_name": folder_name,
|
||||
"path": str(path_obj)
|
||||
}
|
||||
|
||||
return {"status": "ok", "folder_name": folder_name, "path": str(path_obj)}
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"Validation error: {e}")
|
||||
return {"error": "validation_failed", "message": str(e)}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error setting path: {e}", exc_info=True)
|
||||
return {"error": "internal_error", "message": "Failed to set path"}
|
||||
|
||||
def list_folder(self, folder_type: str, path: str = ".") -> Dict[str, Any]:
|
||||
|
||||
def list_folder(self, folder_type: str, path: str = ".") -> dict[str, Any]:
|
||||
"""
|
||||
List contents of a folder with security checks.
|
||||
|
||||
List contents of a configured folder.
|
||||
|
||||
Includes security checks to prevent path traversal.
|
||||
|
||||
Args:
|
||||
folder_type: Type of folder to list (download, tvshow, movie, torrent)
|
||||
path: Relative path within the folder (default: ".")
|
||||
|
||||
folder_type: Type of folder (download, tvshow, movie, torrent).
|
||||
path: Relative path within the folder (default: root).
|
||||
|
||||
Returns:
|
||||
Dict with folder contents or error information
|
||||
Dict with folder contents or error information.
|
||||
"""
|
||||
try:
|
||||
# Validate folder type
|
||||
self._validate_folder_name(folder_type)
|
||||
|
||||
# Sanitize the path
|
||||
safe_path = self._sanitize_path(path)
|
||||
|
||||
# Get root folder from config
|
||||
|
||||
memory = get_memory()
|
||||
folder_key = f"{folder_type}_folder"
|
||||
config = self.memory.get("config", {})
|
||||
|
||||
if folder_key not in config or not config[folder_key]:
|
||||
folder_path = memory.ltm.get_config(folder_key)
|
||||
|
||||
if not folder_path:
|
||||
logger.warning(f"Folder not configured: {folder_type}")
|
||||
return {
|
||||
"error": "folder_not_set",
|
||||
"message": f"{folder_type.capitalize()} folder not set in config."
|
||||
"message": f"{folder_type.capitalize()} folder not configured.",
|
||||
}
|
||||
|
||||
root = Path(config[folder_key])
|
||||
|
||||
root = Path(folder_path)
|
||||
target = root / safe_path
|
||||
|
||||
# Security check: ensure target is within root
|
||||
|
||||
if not self._is_safe_path(root, target):
|
||||
logger.warning(f"Path traversal attempt detected: {path}")
|
||||
logger.warning(f"Path traversal attempt: {path}")
|
||||
return {
|
||||
"error": "forbidden",
|
||||
"message": "Access denied: path outside allowed directory"
|
||||
"message": "Access denied: path outside allowed directory",
|
||||
}
|
||||
|
||||
# Check if target exists
|
||||
|
||||
if not target.exists():
|
||||
logger.warning(f"Path does not exist: {target}")
|
||||
return {
|
||||
"error": "not_found",
|
||||
"message": f"Path does not exist: {safe_path}"
|
||||
"message": f"Path does not exist: {safe_path}",
|
||||
}
|
||||
|
||||
# Check if target is a directory
|
||||
|
||||
if not target.is_dir():
|
||||
logger.warning(f"Path is not a directory: {target}")
|
||||
return {
|
||||
"error": "not_a_directory",
|
||||
"message": f"Path is not a directory: {safe_path}"
|
||||
"message": f"Path is not a directory: {safe_path}",
|
||||
}
|
||||
|
||||
# List directory contents
|
||||
|
||||
try:
|
||||
entries = [entry.name for entry in target.iterdir()]
|
||||
logger.debug(f"Listed {len(entries)} entries in {target}")
|
||||
@@ -162,147 +144,163 @@ class FileManager:
|
||||
"folder_type": folder_type,
|
||||
"path": safe_path,
|
||||
"entries": sorted(entries),
|
||||
"count": len(entries)
|
||||
"count": len(entries),
|
||||
}
|
||||
except PermissionError:
|
||||
logger.warning(f"Permission denied accessing: {target}")
|
||||
logger.warning(f"Permission denied: {target}")
|
||||
return {
|
||||
"error": "permission_denied",
|
||||
"message": f"Permission denied accessing: {safe_path}"
|
||||
"message": f"Permission denied: {safe_path}",
|
||||
}
|
||||
|
||||
|
||||
except PathTraversalError as e:
|
||||
logger.warning(f"Path traversal attempt: {e}")
|
||||
return {
|
||||
"error": "forbidden",
|
||||
"message": str(e)
|
||||
}
|
||||
|
||||
return {"error": "forbidden", "message": str(e)}
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"Validation error: {e}")
|
||||
return {"error": "validation_failed", "message": str(e)}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error listing folder: {e}", exc_info=True)
|
||||
return {"error": "internal_error", "message": "Failed to list folder"}
|
||||
|
||||
def move_file(self, source: str, destination: str) -> Dict[str, Any]:
|
||||
|
||||
def move_file(self, source: str, destination: str) -> dict[str, Any]:
|
||||
"""
|
||||
Move a file from one location to another with safety checks.
|
||||
|
||||
Move a file from one location to another.
|
||||
|
||||
Includes validation and verification after move.
|
||||
|
||||
Args:
|
||||
source: Source file path
|
||||
destination: Destination file path
|
||||
|
||||
source: Source file path.
|
||||
destination: Destination file path.
|
||||
|
||||
Returns:
|
||||
Dict with status or error information
|
||||
Dict with status or error information.
|
||||
"""
|
||||
try:
|
||||
# Convert to Path objects
|
||||
source_path = Path(source).resolve()
|
||||
dest_path = Path(destination).resolve()
|
||||
|
||||
logger.info(f"Moving file from {source_path} to {dest_path}")
|
||||
|
||||
# Validate source
|
||||
|
||||
logger.info(f"Moving file: {source_path} -> {dest_path}")
|
||||
|
||||
if not source_path.exists():
|
||||
return {
|
||||
"error": "source_not_found",
|
||||
"message": f"Source file does not exist: {source}"
|
||||
"message": f"Source does not exist: {source}",
|
||||
}
|
||||
|
||||
|
||||
if not source_path.is_file():
|
||||
return {
|
||||
"error": "source_not_file",
|
||||
"message": f"Source is not a file: {source}"
|
||||
"message": f"Source is not a file: {source}",
|
||||
}
|
||||
|
||||
# Get source file size for verification
|
||||
|
||||
source_size = source_path.stat().st_size
|
||||
|
||||
# Validate destination
|
||||
dest_parent = dest_path.parent
|
||||
|
||||
if not dest_parent.exists():
|
||||
return {
|
||||
"error": "destination_dir_not_found",
|
||||
"message": f"Destination directory does not exist: {dest_parent}"
|
||||
"message": f"Destination directory does not exist: {dest_parent}",
|
||||
}
|
||||
|
||||
|
||||
if dest_path.exists():
|
||||
return {
|
||||
"error": "destination_exists",
|
||||
"message": f"Destination file already exists: {destination}"
|
||||
"message": f"Destination already exists: {destination}",
|
||||
}
|
||||
|
||||
# Perform move
|
||||
|
||||
shutil.move(str(source_path), str(dest_path))
|
||||
|
||||
# Verify
|
||||
|
||||
# Verify move
|
||||
if not dest_path.exists():
|
||||
return {
|
||||
"error": "move_verification_failed",
|
||||
"message": "File was not moved successfully"
|
||||
"message": "File was not moved successfully",
|
||||
}
|
||||
|
||||
|
||||
dest_size = dest_path.stat().st_size
|
||||
if dest_size != source_size:
|
||||
return {
|
||||
"error": "size_mismatch",
|
||||
"message": f"File size mismatch after move"
|
||||
"message": "File size mismatch after move",
|
||||
}
|
||||
|
||||
logger.info(f"File successfully moved: {dest_path.name}")
|
||||
|
||||
logger.info(f"File moved successfully: {dest_path.name}")
|
||||
return {
|
||||
"status": "ok",
|
||||
"source": str(source_path),
|
||||
"destination": str(dest_path),
|
||||
"filename": dest_path.name,
|
||||
"size": dest_size
|
||||
"size": dest_size,
|
||||
}
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving file: {e}", exc_info=True)
|
||||
return {
|
||||
"error": "move_failed",
|
||||
"message": str(e)
|
||||
}
|
||||
|
||||
return {"error": "move_failed", "message": str(e)}
|
||||
|
||||
def _validate_folder_name(self, folder_name: str) -> bool:
|
||||
"""Validate folder name against allowed values."""
|
||||
"""
|
||||
Validate folder name against allowed values.
|
||||
|
||||
Args:
|
||||
folder_name: Name to validate.
|
||||
|
||||
Returns:
|
||||
True if valid.
|
||||
|
||||
Raises:
|
||||
ValueError: If folder name is invalid.
|
||||
"""
|
||||
valid_names = [fn.value for fn in FolderName]
|
||||
if folder_name not in valid_names:
|
||||
raise ValueError(
|
||||
f"Invalid folder_name '{folder_name}'. Must be one of: {', '.join(valid_names)}"
|
||||
f"Invalid folder_name '{folder_name}'. "
|
||||
f"Must be one of: {', '.join(valid_names)}"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _sanitize_path(self, path: str) -> str:
|
||||
"""Sanitize path to prevent path traversal attacks."""
|
||||
# Normalize path
|
||||
"""
|
||||
Sanitize path to prevent path traversal attacks.
|
||||
|
||||
Args:
|
||||
path: Path to sanitize.
|
||||
|
||||
Returns:
|
||||
Sanitized path.
|
||||
|
||||
Raises:
|
||||
PathTraversalError: If path contains traversal attempts.
|
||||
"""
|
||||
normalized = os.path.normpath(path)
|
||||
|
||||
# Check for absolute paths
|
||||
|
||||
if os.path.isabs(normalized):
|
||||
raise PathTraversalError("Absolute paths are not allowed")
|
||||
|
||||
# Check for parent directory references
|
||||
|
||||
if normalized.startswith("..") or "/.." in normalized or "\\.." in normalized:
|
||||
raise PathTraversalError("Parent directory references are not allowed")
|
||||
|
||||
# Check for null bytes
|
||||
raise PathTraversalError("Parent directory references not allowed")
|
||||
|
||||
if "\x00" in normalized:
|
||||
raise PathTraversalError("Null bytes in path are not allowed")
|
||||
|
||||
raise PathTraversalError("Null bytes in path not allowed")
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _is_safe_path(self, base_path: Path, target_path: Path) -> bool:
|
||||
"""Check if target path is within base path (prevents path traversal)."""
|
||||
"""
|
||||
Check if target path is within base path.
|
||||
|
||||
Args:
|
||||
base_path: The allowed base directory.
|
||||
target_path: The path to check.
|
||||
|
||||
Returns:
|
||||
True if target is within base, False otherwise.
|
||||
"""
|
||||
try:
|
||||
# Resolve both paths to absolute paths
|
||||
base_resolved = base_path.resolve()
|
||||
target_resolved = target_path.resolve()
|
||||
|
||||
# Check if target is relative to base
|
||||
target_resolved.relative_to(base_resolved)
|
||||
return True
|
||||
except (ValueError, OSError):
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
"""Media organizer - Organizes movies and TV shows into proper folder structures."""
|
||||
from pathlib import Path
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
from domain.movies.entities import Movie
|
||||
from domain.tv_shows.entities import TVShow, Episode
|
||||
from domain.shared.value_objects import FilePath
|
||||
from domain.tv_shows.entities import Episode, TVShow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,101 +12,99 @@ logger = logging.getLogger(__name__)
|
||||
class MediaOrganizer:
|
||||
"""
|
||||
Organizes media files into proper folder structures.
|
||||
|
||||
|
||||
This service knows how to organize movies and TV shows according to
|
||||
common media server conventions (Plex, Jellyfin, etc.).
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, movie_folder: Path, tvshow_folder: Path):
|
||||
"""
|
||||
Initialize media organizer.
|
||||
|
||||
|
||||
Args:
|
||||
movie_folder: Root folder for movies
|
||||
tvshow_folder: Root folder for TV shows
|
||||
"""
|
||||
self.movie_folder = movie_folder
|
||||
self.tvshow_folder = tvshow_folder
|
||||
|
||||
|
||||
def get_movie_destination(self, movie: Movie, filename: str) -> Path:
|
||||
"""
|
||||
Get the destination path for a movie file.
|
||||
|
||||
|
||||
Structure: /movies/Movie Title (Year)/Movie.Title.Year.Quality.ext
|
||||
|
||||
|
||||
Args:
|
||||
movie: Movie entity
|
||||
filename: Original filename (to extract extension)
|
||||
|
||||
|
||||
Returns:
|
||||
Full destination path
|
||||
"""
|
||||
# Create movie folder
|
||||
folder_name = movie.get_folder_name()
|
||||
movie_dir = self.movie_folder / folder_name
|
||||
|
||||
|
||||
# Get extension from original filename
|
||||
extension = Path(filename).suffix
|
||||
|
||||
|
||||
# Create new filename
|
||||
new_filename = movie.get_filename() + extension
|
||||
|
||||
|
||||
return movie_dir / new_filename
|
||||
|
||||
|
||||
def get_episode_destination(
|
||||
self,
|
||||
show: TVShow,
|
||||
episode: Episode,
|
||||
filename: str
|
||||
self, show: TVShow, episode: Episode, filename: str
|
||||
) -> Path:
|
||||
"""
|
||||
Get the destination path for a TV show episode file.
|
||||
|
||||
|
||||
Structure: /tvshows/Show.Name/Season 01/S01E05.Episode.Title.ext
|
||||
|
||||
|
||||
Args:
|
||||
show: TVShow entity
|
||||
episode: Episode entity
|
||||
filename: Original filename (to extract extension)
|
||||
|
||||
|
||||
Returns:
|
||||
Full destination path
|
||||
"""
|
||||
# Create show folder
|
||||
show_folder_name = show.get_folder_name()
|
||||
show_dir = self.tvshow_folder / show_folder_name
|
||||
|
||||
|
||||
# Create season folder
|
||||
from domain.tv_shows.entities import Season
|
||||
|
||||
season = Season(
|
||||
show_imdb_id=show.imdb_id,
|
||||
season_number=episode.season_number,
|
||||
episode_count=0 # Not needed for folder name
|
||||
episode_count=0, # Not needed for folder name
|
||||
)
|
||||
season_folder_name = season.get_folder_name()
|
||||
season_dir = show_dir / season_folder_name
|
||||
|
||||
|
||||
# Get extension from original filename
|
||||
extension = Path(filename).suffix
|
||||
|
||||
|
||||
# Create new filename
|
||||
new_filename = episode.get_filename() + extension
|
||||
|
||||
|
||||
return season_dir / new_filename
|
||||
|
||||
|
||||
def create_movie_directory(self, movie: Movie) -> bool:
|
||||
"""
|
||||
Create the directory structure for a movie.
|
||||
|
||||
|
||||
Args:
|
||||
movie: Movie entity
|
||||
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
folder_name = movie.get_folder_name()
|
||||
movie_dir = self.movie_folder / folder_name
|
||||
|
||||
|
||||
try:
|
||||
movie_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created movie directory: {movie_dir}")
|
||||
@@ -115,32 +112,32 @@ class MediaOrganizer:
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create movie directory: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def create_episode_directory(self, show: TVShow, season_number: int) -> bool:
|
||||
"""
|
||||
Create the directory structure for a TV show season.
|
||||
|
||||
|
||||
Args:
|
||||
show: TVShow entity
|
||||
season_number: Season number
|
||||
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
from domain.tv_shows.entities import Season
|
||||
from domain.tv_shows.value_objects import SeasonNumber
|
||||
|
||||
|
||||
show_folder_name = show.get_folder_name()
|
||||
show_dir = self.tvshow_folder / show_folder_name
|
||||
|
||||
|
||||
season = Season(
|
||||
show_imdb_id=show.imdb_id,
|
||||
season_number=SeasonNumber(season_number),
|
||||
episode_count=0
|
||||
episode_count=0,
|
||||
)
|
||||
season_folder_name = season.get_folder_name()
|
||||
season_dir = show_dir / season_folder_name
|
||||
|
||||
|
||||
try:
|
||||
season_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created season directory: {season_dir}")
|
||||
|
||||
Reference in New Issue
Block a user